Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import io.opentelemetry.context.Scope;
import java.io.IOException;
import java.math.BigDecimal;
import java.nio.channels.Channels;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.time.LocalDate;
Expand Down Expand Up @@ -226,16 +227,14 @@ private void deserializeArrowBatch(ArrowRecordBatch batch) throws SQLException {
// Clear vectorSchemaRoot before populating a new batch
vectorSchemaRoot.clear();
}
org.apache.arrow.vector.ipc.message.ArrowRecordBatch deserializedBatch =
// Read straight from the ByteString instead of copying it into a heap byte[] first.
try (org.apache.arrow.vector.ipc.message.ArrowRecordBatch deserializedBatch =
MessageSerializer.deserializeRecordBatch(
new ReadChannel(
new ByteArrayReadableSeekableByteChannel(
batch.getSerializedRecordBatch().toByteArray())),
allocator);

vectorLoader.load(deserializedBatch);
// Release buffers from batch (they are still held in the vectors in root).
deserializedBatch.close();
new ReadChannel(Channels.newChannel(batch.getSerializedRecordBatch().newInput())),
allocator)) {
Comment thread
keshavdandeva marked this conversation as resolved.
Resolved
// The vectors keep their own references to the buffers after the batch is closed.
vectorLoader.load(deserializedBatch);
}
} catch (RuntimeException | IOException ex) {
throw new BigQueryJdbcException(ex);
}
Expand Down Expand Up @@ -346,6 +345,9 @@ Object getObjectInternal(int columnIndex) throws SQLException {
LOG.severe(ex.getMessage(), ex);
throw ex;
}
if (this.afterLast || this.nestedRowIndex < this.fromIndex) {
throw new BigQueryJdbcException("The ResultSet is not positioned on a row.");
}
// For Arrays the first column is Index, ref:
// https://docs.oracle.com/javase/7/docs/api/java/sql/Array.html#getResultSet()
if (columnIndex == 1) {
Expand All @@ -356,6 +358,9 @@ Object getObjectInternal(int columnIndex) throws SQLException {
value = this.currentNestedBatch.getNestedRecords().get(this.nestedRowIndex);
}
} else {
if (this.afterLast || this.vectorSchemaRoot == null || this.currentBatchRowIndex < 0) {
throw new BigQueryJdbcException("The ResultSet is not positioned on a row.");
}
Comment thread
keshavdandeva marked this conversation as resolved.
Resolved
// get the current column
// SQL index to Java Index
FieldVector currentColumn = this.vectorSchemaRoot.getVector(columnIndex - 1);
Expand Down Expand Up @@ -511,9 +516,9 @@ public boolean isFirst() throws SQLException {
LOG.finestTrace("isFirst");
checkClosed();
if (this.isNested) {
return this.nestedRowIndex == this.fromIndex;
return !this.afterLast && this.nestedRowIndex == this.fromIndex;
} else {
return this.rowCount == 1;
return !this.afterLast && this.rowCount == 1;
}
}

Expand All @@ -522,9 +527,9 @@ public boolean isLast() throws SQLException {
LOG.finestTrace("isLast");
checkClosed();
if (this.isNested) {
return this.nestedRowIndex == this.toIndexExclusive - 1;
return !this.afterLast && this.nestedRowIndex == this.toIndexExclusive - 1;
} else {
return this.rowCount == this.totalRows;
return !this.afterLast && this.rowCount == this.totalRows;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -907,7 +907,7 @@ ResultSet processArrowResultSet(TableResult results, Job job) throws SQLExceptio
LOG.finer("++enter++");

// set the resultset
long totalRows = (getMaxRows() > 0) ? getMaxRows() : results.getTotalRows();
long totalRows = getEffectiveTotalRows(results);
JobId currentJobId = results.getJobId();
TableId destinationTable = getDestinationTable(currentJobId);
Schema schema = results.getSchema();
Expand Down Expand Up @@ -1198,11 +1198,20 @@ private boolean meetsReadRatio(TableResult results) {
return totalRows / pageSize > querySettings.getHighThroughputActivationRatio();
}

/**
* Number of rows the result set returns: the query's total, capped by {@link #setMaxRows(int)}.
*/
@VisibleForTesting
long getEffectiveTotalRows(TableResult results) {
long totalRows = results.getTotalRows();
return getMaxRows() > 0 ? Math.min(getMaxRows(), totalRows) : totalRows;
}

BigQueryJsonResultSet processJsonResultSet(TableResult results, Job job) throws SQLException {
List<Future<?>> taskList = new ArrayList<>();

Schema schema = results.getSchema();
long totalRows = (getMaxRows() > 0) ? getMaxRows() : results.getTotalRows();
long totalRows = getEffectiveTotalRows(results);
this.bigQueryFieldValueListWrapperBlockingQueue = new LinkedBlockingDeque<>(getBufferSize());
BlockingQueue<Tuple<TableResult, Boolean>> rpcResponseQueue =
new LinkedBlockingDeque<>(getPageCacheSize(getBufferSize(), schema));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -812,4 +812,25 @@ public void testTimestampPicosAccessorsAcrossJvmZones(String jvmZone)
TimeZone.setDefault(originalDefault);
}
}

@Test
public void testGetterWhenNotOnRowThrows() throws SQLException {
assertThrows(SQLException.class, () -> bigQueryArrowResultSet.getString(4));
assertThat(bigQueryArrowResultSet.next()).isTrue();
assertThat(bigQueryArrowResultSet.getString(4)).isEqualTo("text1");
assertThat(bigQueryArrowResultSet.next()).isFalse();
assertThat(bigQueryArrowResultSet.isFirst()).isFalse();
assertThat(bigQueryArrowResultSet.isLast()).isFalse();
assertThrows(SQLException.class, () -> bigQueryArrowResultSet.getString(4));

// Nested result set (Array.getResultSet) over [10, 20].
assertThrows(SQLException.class, () -> bigQueryArrowResultSetNested.getObject(2));
assertThat(bigQueryArrowResultSetNested.next()).isTrue();
assertThat(bigQueryArrowResultSetNested.getObject(2)).isEqualTo(10L);
assertThat(bigQueryArrowResultSetNested.next()).isTrue();
assertThat(bigQueryArrowResultSetNested.next()).isFalse();
assertThat(bigQueryArrowResultSetNested.isFirst()).isFalse();
assertThat(bigQueryArrowResultSetNested.isLast()).isFalse();
assertThrows(SQLException.class, () -> bigQueryArrowResultSetNested.getObject(2));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -450,6 +450,19 @@ public void testSetMaxRowsJson() throws SQLException {
assertEquals(10, bigQueryStatement.getMaxRows());
}

@Test
public void testGetEffectiveTotalRowsIsCappedByMaxRows() throws SQLException {
TableResult result = mock(TableResult.class);
doReturn(10L).when(result).getTotalRows();

assertEquals(10, bigQueryStatement.getEffectiveTotalRows(result));
// A limit larger than the result must not inflate the total (isLast() relies on it).
bigQueryStatement.setMaxRows(1000);
assertEquals(10, bigQueryStatement.getEffectiveTotalRows(result));
bigQueryStatement.setMaxRows(5);
assertEquals(5, bigQueryStatement.getEffectiveTotalRows(result));
}

@Test
public void setQueryTimeoutTest() throws Exception {
bigQueryStatement.setQueryTimeout(3);
Expand Down
Loading