Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -34,17 +34,21 @@
import com.google.protobuf.Descriptors.DescriptorValidationException;
import java.io.IOException;
import java.util.concurrent.Phaser;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.logging.Level;
import javax.annotation.concurrent.GuardedBy;

class BigQueryJdbcBulkInsertWriter {
private static final BigQueryJdbcCustomLogger LOG =
new BigQueryJdbcCustomLogger(BigQueryJdbcBulkInsertWriter.class.getName());
private JsonStreamWriter jsonStreamWriter;
JsonStreamWriter jsonStreamWriter;
// Written once during initialize() and read by background gRPC completion callbacks.
private volatile String streamName;
private final Phaser openRequestCount = new Phaser(1);
private final AtomicBoolean writerClosed = new AtomicBoolean(false);
private final Object streamLock = new Object();

@GuardedBy("lock")
@GuardedBy("streamLock")
private RuntimeException error = null;

void initialize(TableName parentTable, BigQueryWriteClient client, RetrySettings retrySettings)
Expand All @@ -59,15 +63,21 @@ void initialize(TableName parentTable, BigQueryWriteClient client, RetrySettings
.setWriteStream(stream)
.build();
WriteStream writeStream = client.createWriteStream(createWriteStreamRequest);
this.streamName = writeStream.getName();

JsonStreamWriter.Builder jsonStreamWriterBuilder =
JsonStreamWriter.newBuilder(writeStream.getName(), writeStream.getTableSchema(), client);
JsonStreamWriter.newBuilder(this.streamName, writeStream.getTableSchema(), client);

if (retrySettings != null) {
jsonStreamWriterBuilder.setRetrySettings(retrySettings);
}

this.jsonStreamWriter = jsonStreamWriterBuilder.build();
this.jsonStreamWriter = buildJsonStreamWriter(jsonStreamWriterBuilder);
}

JsonStreamWriter buildJsonStreamWriter(JsonStreamWriter.Builder builder)
throws DescriptorValidationException, IOException, InterruptedException {
return builder.build();
}

void append(JsonArray data, long offset) throws DescriptorValidationException, IOException {
Expand All @@ -80,17 +90,29 @@ void append(JsonArray data, long offset) throws DescriptorValidationException, I
}
}

ApiFuture<AppendRowsResponse> future = jsonStreamWriter.append(data, offset);
// Register before submitting so a synchronously completed future cannot drop the party count
// to zero (which permanently terminates the Phaser) before registration occurs.
openRequestCount.register();
ApiFuture<AppendRowsResponse> future;
try {
future = jsonStreamWriter.append(data, offset);
} catch (Throwable t) {
openRequestCount.arriveAndDeregister();
throw t;
}
ApiFutures.addCallback(
future, new AppendCompleteCallback(this), MoreExecutors.directExecutor());
Comment on lines +95 to 104

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

Moving the ApiFutures.addCallback call inside the try block ensures that if any exception occurs during callback registration, the phaser is correctly deregistered in the catch block. This prevents a potential hang during cleanup where the main thread waits indefinitely for the leaked phaser party. Catching Throwable here is appropriate as it is at a Future boundary to ensure all failures are captured.

    openRequestCount.register();
    try {
      ApiFuture<AppendRowsResponse> future = jsonStreamWriter.append(data, offset);
      ApiFutures.addCallback(
          future, new AppendCompleteCallback(this), MoreExecutors.directExecutor());
    } catch (Throwable t) {
      openRequestCount.arriveAndDeregister();
      throw t;
    }
References
  1. Catching Throwable instead of Exception is acceptable at Future boundaries or during stream cleanup in asynchronous operations to ensure all failures are captured and propagated.

openRequestCount.register();
}

long cleanup(BigQueryWriteClient client) {
long cleanup(BigQueryWriteClient client) throws InterruptedException {
LOG.finer("++enter++");
LOG.fine("Cleaning up bulk insert writer for stream: %s", jsonStreamWriter.getStreamName());
openRequestCount.arriveAndAwaitAdvance();
jsonStreamWriter.close();
String currentStreamName = getStreamName();
LOG.fine("Cleaning up bulk insert writer for stream: %s", currentStreamName);
try {
openRequestCount.awaitAdvanceInterruptibly(openRequestCount.arrive());
} finally {
closeStreamWriter();
}

synchronized (this.streamLock) {
if (this.error != null) {
Expand All @@ -99,14 +121,32 @@ long cleanup(BigQueryWriteClient client) {
}

// Finalize the stream.
FinalizeWriteStreamResponse finalizeResponse =
client.finalizeWriteStream(jsonStreamWriter.getStreamName());
LOG.finer("Rows written: " + finalizeResponse.getRowCount());
FinalizeWriteStreamResponse finalizeResponse = client.finalizeWriteStream(currentStreamName);
LOG.finer("Rows written: %d", finalizeResponse.getRowCount());
return finalizeResponse.getRowCount();
}

void closeQuietly() {
if (this.jsonStreamWriter != null && this.writerClosed.compareAndSet(false, true)) {
try {
this.jsonStreamWriter.close();
} catch (RuntimeException ex) {
LOG.warning(ex, "Failed to close JsonStreamWriter during error cleanup.");
}
}
}

private void closeStreamWriter() {
if (this.jsonStreamWriter != null && this.writerClosed.compareAndSet(false, true)) {
this.jsonStreamWriter.close();
}
}

String getStreamName() {
return jsonStreamWriter.getStreamName();
if (this.streamName != null) {
return this.streamName;
}
return this.jsonStreamWriter != null ? this.jsonStreamWriter.getStreamName() : null;
}

static class AppendCompleteCallback implements ApiFutureCallback<AppendRowsResponse> {
Expand All @@ -122,7 +162,7 @@ public void onSuccess(AppendRowsResponse response) {
}

public void onFailure(Throwable throwable) {
parent.LOG.warning(
LOG.warning(
"Append failed for stream %s: %s", parent.getStreamName(), throwable.getMessage());
synchronized (this.parent.streamLock) {
if (this.parent.error == null) {
Expand Down
Loading
Loading