Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -413,7 +413,9 @@ protected boolean removeEldestEntry(Map.Entry<String, Map<String, String>> eldes
BigQueryConnectionProperty.newBuilder()
.setName(ENABLE_HTAPI_PROPERTY_NAME)
.setDescription(
"Enables or disables Read API usage in the Driver. Disabled by default.")
"Enables or disables Read API usage in the Driver. Disabled by default."
+ " With EnableQueryResultsArrowFormat enabled, this applies only"
+ " when LargeResultTable or LargeResultDataset is set.")
.setDefaultValue(String.valueOf(DEFAULT_ENABLE_HTAPI_VALUE))
.build(),
BigQueryConnectionProperty.newBuilder()
Expand Down Expand Up @@ -453,7 +455,12 @@ protected boolean removeEldestEntry(Map.Entry<String, Map<String, String>> eldes
.setDescription(
"Reads query results in the Apache Arrow format: the first page of"
+ " jobs.query and the query job's Storage Read API stream (Preview)."
+ " Disabled by default.")
+ " A result with more than one page and at least"
+ " HighThroughputMinTableSize rows is read from the job stream,"
+ " whatever EnableHighThroughputAPI and"
+ " HighThroughputActivationRatio are set to, unless"
+ " LargeResultTable or LargeResultDataset is set. Disabled by"
+ " default.")
.setDefaultValue(
String.valueOf(DEFAULT_ENABLE_QUERY_RESULTS_ARROW_FORMAT_VALUE))
.build(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -147,8 +147,6 @@ public class BigQueryStatement extends BigQueryNoOpsStatement {

private BlockingQueue<BigQueryFieldValueListWrapper> bigQueryFieldValueListWrapperBlockingQueue;

private BlockingQueue<BigQueryArrowBatchWrapper> arrowBatchWrapperBlockingQueue;

// Variables Required for the ReferenceQueue implementation
static ReferenceQueue<BigQueryArrowResultSet> referenceQueueArrowRs = new ReferenceQueue<>();
static ReferenceQueue<BigQueryJsonResultSet> referenceQueueJsonRs = new ReferenceQueue<>();
Expand Down Expand Up @@ -549,7 +547,6 @@ private void closeStatementResources() throws SQLException {
tmp.close();
}
this.batchQueries.clear();
this.arrowBatchWrapperBlockingQueue = null;
this.bigQueryFieldValueListWrapperBlockingQueue = null;
this.currentUpdateCount = -1;
this.currentJobIdIndex = -1;
Expand Down Expand Up @@ -923,10 +920,8 @@ ResultSet processArrowResultSet(TableResult results, Job job) throws SQLExceptio
LOG.finer("++enter++");

// set the resultset
long totalRows = getEffectiveTotalRows(results);
JobId currentJobId = results.getJobId();
TableId destinationTable = getDestinationTable(currentJobId);
Schema schema = results.getSchema();
Future<?> populateBufferWorker = null;
try {
String parent = String.format("projects/%s", destinationTable.getProject());
Expand All @@ -953,45 +948,91 @@ ResultSet processArrowResultSet(TableResult results, Job job) throws SQLExceptio
.setMaxStreamCount(1);

ReadSession readSession = getReadSession(builder.build());
this.arrowBatchWrapperBlockingQueue = new LinkedBlockingDeque<>(getArrowBufferSize());
BlockingQueue<BigQueryArrowBatchWrapper> queue =
new LinkedBlockingDeque<>(getArrowBufferSize());
// deserialize and populate the buffer async, so that the client isn't blocked
populateBufferWorker =
populateArrowBufferedQueue(
readSession, this.arrowBatchWrapperBlockingQueue, getBigQueryReadClient());

BigQueryArrowResultSet arrowResultSet =
BigQueryArrowResultSet.of(
schema,
getArrowSchema(readSession),
totalRows,
this,
this.arrowBatchWrapperBlockingQueue,
populateBufferWorker,
this.bigQuery,
job);
arrowResultSetFinalizers.add(
new BigQueryResultSetFinalizers.ArrowResultSetFinalizer(
arrowResultSet, referenceQueueArrowRs, populateBufferWorker));
arrowResultSet.setJobId(currentJobId);
arrowResultSet.setQueryId(results.getQueryId());
if (job == null && currentJobId == null) {
arrowResultSet.setQueryStatistics(results.extractQueryStatistics());
}
return arrowResultSet;
populateArrowBufferedQueue(readSession, queue, getBigQueryReadClient());

return newArrowResultSet(
results, job, getArrowSchema(readSession), queue, populateBufferWorker);

} catch (Exception | OutOfMemoryError ex) {
if (populateBufferWorker != null) {
populateBufferWorker.cancel(true);
}
if (ex instanceof OutOfMemoryError || ex instanceof RejectedExecutionException) {
throw new BigQueryJdbcException(
"Failed to execute query: Unable to allocate background threads to process the query results. Connection-scoped thread pool limit of 100 threads was reached or system is out of memory.",
ex);
throw arrowResultSetException(ex);
}
}

/**
* Reads a query job's results from its {@code _default} stream, starting at row 0. No read
* session is created and the job is not looked up. The first page in {@code results} is not
* returned: it supplies the column types and the row count. The Arrow schema arrives in the
* stream's first message.
*/
private ResultSet processJobStreamResultSet(TableResult results, Job job) throws SQLException {
LOG.finer("++enter++");
Future<?> populateBufferWorker = null;
try {
BlockingQueue<BigQueryArrowBatchWrapper> queue =
new LinkedBlockingDeque<>(getArrowBufferSize());
populateBufferWorker =
populateArrowBufferedQueueFromJobStream(
results.getJobId(), 0, queue, getBigQueryReadClient());

return newArrowResultSet(results, job, null, queue, populateBufferWorker);
} catch (Exception | OutOfMemoryError ex) {
if (populateBufferWorker != null) {
populateBufferWorker.cancel(true);
}
throw new BigQueryJdbcException(ex.getMessage(), ex);
throw arrowResultSetException(ex);
}
}

/**
* Builds the result set over {@code queue}, which {@code reader} fills. Without an {@code
* arrowSchema}, the result set takes it from the first message in the queue.
*/
private BigQueryArrowResultSet newArrowResultSet(
TableResult results,
Job job,
ArrowSchema arrowSchema,
BlockingQueue<BigQueryArrowBatchWrapper> queue,
Future<?> reader)
throws SQLException {
JobId currentJobId = results.getJobId();
BigQueryArrowResultSet arrowResultSet =
BigQueryArrowResultSet.of(
results.getSchema(),
arrowSchema,
getEffectiveTotalRows(results),
this,
queue,
reader,
this.bigQuery,
job);
arrowResultSetFinalizers.add(
new BigQueryResultSetFinalizers.ArrowResultSetFinalizer(
arrowResultSet, referenceQueueArrowRs, reader));
arrowResultSet.setJobId(currentJobId);
arrowResultSet.setQueryId(results.getQueryId());
if (job == null && currentJobId == null) {
arrowResultSet.setQueryStatistics(results.extractQueryStatistics());
}
return arrowResultSet;
}

/** The exception a statement throws when it can't set up an Arrow result set. */
private static BigQueryJdbcException arrowResultSetException(Throwable ex) {
if (ex instanceof OutOfMemoryError || ex instanceof RejectedExecutionException) {
return new BigQueryJdbcException(
"Failed to execute query: Unable to allocate background threads to process the query results. Connection-scoped thread pool limit of 100 threads was reached or system is out of memory.",
ex);
}
return new BigQueryJdbcException(ex);
}
Comment thread
keshavdandeva marked this conversation as resolved.
Resolved

private TableReadOptions buildTableReadOptions() {
if (!isEnableTimestampPicos()) {
return null;
Expand Down Expand Up @@ -1027,10 +1068,7 @@ ArrowSerializationOptions buildArrowSerializationOptions() {
*/
@VisibleForTesting
static String jobStreamName(JobId jobId) throws BigQueryJdbcException {
if (jobId == null
|| Strings.isNullOrEmpty(jobId.getProject())
|| Strings.isNullOrEmpty(jobId.getLocation())
|| Strings.isNullOrEmpty(jobId.getJob())) {
if (!isCompleteJobReference(jobId)) {
throw new BigQueryJdbcException(
"A job stream needs the job's project, location and job ID, got: " + jobId);
}
Expand All @@ -1039,6 +1077,14 @@ static String jobStreamName(JobId jobId) throws BigQueryJdbcException {
jobId.getProject(), jobId.getLocation(), jobId.getJob());
}

/** A job stream can be named only from a reference that has a project, location and job ID. */
private static boolean isCompleteJobReference(JobId jobId) {
return jobId != null
&& !Strings.isNullOrEmpty(jobId.getProject())
&& !Strings.isNullOrEmpty(jobId.getLocation())
&& !Strings.isNullOrEmpty(jobId.getJob());
}

/** Asynchronously reads results and populates an arrow record queue */
@InternalApi
Future<?> populateArrowBufferedQueue(
Expand Down Expand Up @@ -1212,7 +1258,14 @@ void processQueryResponse(String query, TableResult results, Job job) throws SQL
LOG.fine("Processing query response. Query: %s", query);

ResultSet resultSet = null;
if (jobId != null && useReadAPI(results)) {
if (isJobStreamAllowed()) {
// The job stream or JSON. A read session is never used where the job stream is allowed.
if (useJobStream(results)) {
LOG.info("Using the job stream to read the data.");
resultSet = processJobStreamResultSet(results, job);
this.currentExecutionBuilder.setQueryApiType(QueryApiType.QUERY_API_TYPE_READ_API);
}
} else if (jobId != null && useReadAPI(results)) {
try {
LOG.info("Using ReadAPI to read the data.");
resultSet = processArrowResultSet(results, job);
Expand Down Expand Up @@ -1253,6 +1306,49 @@ private boolean isPermissionDeniedException(Throwable t) {
return false;
}

/**
* True when a result may be read from its job's stream: {@code EnableQueryResultsArrowFormat} is
* on and the user set no destination table. A job with a user-set destination table has no
* stream, so it keeps the High-Throughput API rules.
*/
private boolean isJobStreamAllowed() {
return querySettings.isEnableQueryResultsArrowFormat() && !isLargeResultsEnabled();
}

/**
* Whether a SELECT result is read from the query job's stream. The result must have more pages,
* and return at least {@code HighThroughputMinTableSize} rows and more rows than its first page
* holds. Rows are counted after {@link #setMaxRows(int)}. Any other result is read as JSON,
* starting with the page already fetched.
*/
@VisibleForTesting
boolean useJobStream(TableResult results) {
if (!isJobStreamAllowed()) {
return false;
}
long rows = getEffectiveTotalRows(results);
Long rowsInPage = results.getRowsInPage();
int threshold = querySettings.getHighThroughputMinTableSize();
LOG.fine(
"Job stream rule. More pages: %s, rows to return: %d, rows in the first page: %s,"
+ " threshold: %d.",
results.hasNextPage(), rows, rowsInPage, threshold);
if (!results.hasNextPage() || rows < threshold) {
return false;
}
if (rowsInPage != null && rows <= rowsInPage) {
// setMaxRows stops within the first page, and that page is already here.
return false;
}
if (!isCompleteJobReference(results.getJobId())) {
LOG.warning(
"Reading the results as JSON: the job reference %s can't name a job stream.",
results.getJobId());
return false;
}
return true;
}

// The read Ratio should be met
// AND the User must not have disabled the Read API
@VisibleForTesting
Expand Down
Loading
Loading