Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,14 @@ private enum PoolState {
@GuardedBy("poolLock")
private int consecutiveFailures = 0;

@GuardedBy("poolLock")
@Nullable
private Status lastCloseStatus = null;

@GuardedBy("poolLock")
@Nullable
private Metadata lastCloseTrailers = null;

/**
* When the client fallback to a non-session AFE session creation will return unimplemented
* errors. In which case the requests should fallback to classic client instead of waiting for an
Expand Down Expand Up @@ -598,6 +606,8 @@ private void onSessionReady(SessionHandle handle, OpenSessionResponse ignored) {
new Object[] {handle.getSession().getLogName(), handle.getSession().getState()});

consecutiveFailures = 0;
lastCloseStatus = null;
lastCloseTrailers = null;
consecutiveUnimplementedFailures = 0;

if (poolState != PoolState.STARTED) {
Expand Down Expand Up @@ -729,13 +739,15 @@ private void onSessionClose(
// rather than session establishment failures and should not fail pending vRPCs.
if (prevState == SessionState.STARTING) {
consecutiveFailures++;
lastCloseStatus = status;
lastCloseTrailers = trailers;
Comment thread
mutianf marked this conversation as resolved.
Resolved
if (status.getCode() == Status.Code.UNIMPLEMENTED) {
consecutiveUnimplementedFailures++;
} else {
consecutiveUnimplementedFailures = 0;
}
// TODO: decide if max consecutive failures should be capped per client
if (consecutiveFailures >= getMaxConsecutiveFailures(configManager)) {
if (isFailingWithNoEstablishedSessions()) {
toBeClosed = popClosableRpcs();
}
}
Expand All @@ -757,14 +769,7 @@ private void onSessionClose(

if (!toBeClosed.isEmpty()) {
// vRPCs failing with consecutive failures should fail
VRpcResult result =
VRpcResult.createRejectedError(
Status.fromCode(status.getCode())
.withDescription(
String.format(
"Session failed with consecutive failures. Most recent server status: %s,"
+ " metadata: %s.",
status, trailers)));
VRpcResult result = createConsecutiveFailureResult(status, trailers);
for (PendingVRpc<?, ?> vrpc : toBeClosed) {
try {
vrpc.cancelWithResult(result);
Expand All @@ -775,6 +780,30 @@ private void onSessionClose(
}
}

@GuardedBy("poolLock")
private boolean hasEstablishedSession() {
return sessions.getStats().getReadyCount() + sessions.getStats().getInUseCount() > 0;
}

@GuardedBy("poolLock")
private boolean isFailingWithNoEstablishedSessions() {
return consecutiveFailures >= getMaxConsecutiveFailures(configManager)
&& lastCloseStatus != null
&& !hasEstablishedSession();
}

private static VRpcResult createConsecutiveFailureResult(
Status status, @Nullable Metadata trailers) {
Status errorStatus =
status.isOk() ? Status.UNAVAILABLE : Status.fromCode(status.getCode());
return VRpcResult.createRejectedError(
errorStatus.withDescription(
String.format(
"Session failed with consecutive failures. Most recent server status: %s,"
+ " metadata: %s.",
status, trailers)));
}

@GuardedBy("poolLock")
private void tryDrainPendingRpcs() {
while (!pendingRpcs.isEmpty()) {
Expand Down Expand Up @@ -895,7 +924,7 @@ public boolean hasSession() {
return false;
}
try {
return sessions.getStats().getReadyCount() + sessions.getStats().getInUseCount() > 0;
return hasEstablishedSession();
} finally {
poolLock.unlock();
}
Expand Down Expand Up @@ -970,6 +999,10 @@ public void start(ReqT req, VRpcCallContext ctx, VRpcListener<RespT> listener) {
});
return;
}
if (isFailingWithNoEstablishedSessions()) {
cancelWithResult(createConsecutiveFailureResult(lastCloseStatus, lastCloseTrailers));
return;
}
// Only arm the deadline monitor after we've committed to queueing; otherwise the
// fast-fail early return above would leak a timer that fires later and emits a phantom
// tracer.onAttemptFinish on the Active state's stale listener.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1448,6 +1448,112 @@ void configUpdateToEmptyClosesSessionsAndScalesBackUpWhenEnabled() throws Except
assertThat(sessionPool.hasSession()).isTrue();
}

@Test
@SuppressWarnings("GuardedBy")
void consecutiveFailuresFastFailsDuringBudgetLockout() throws Exception {
com.google.rpc.Status errorStatus =
com.google.rpc.Status.newBuilder()
.setCode(Code.INTERNAL_VALUE)
.setMessage("fake internal error")
.build();
OpenFakeSessionRequest request =
OpenFakeSessionRequest.newBuilder()
.setStreamError(
StreamError.newBuilder()
.setStatus(errorStatus)
.setDelay(Durations.fromMillis(0))
.build())
.build();
sessionPool.start(request, new Metadata());

// Wait until all initial SessionCreationBudget slots have been exhausted and closed on the
// client
ReentrantLock poolLock = extractPoolLock(sessionPool);
Field consecutiveFailuresField = SessionPoolImpl.class.getDeclaredField("consecutiveFailures");
consecutiveFailuresField.setAccessible(true);
long deadline = System.currentTimeMillis() + 5000;
while (System.currentTimeMillis() < deadline) {
poolLock.lock();
try {
if (sessionPool.sessions.getAllSessions().isEmpty()
&& consecutiveFailuresField.getInt(sessionPool) >= 10) {
break;
}
} finally {
poolLock.unlock();
}
Thread.sleep(10);
}

// A new call arriving during the 60s budget lockout must fail immediately instead of queueing
UnaryResponseFuture<SessionFakeScriptedResponse> f = new UnaryResponseFuture<>();
VRpc<SessionFakeScriptedRequest, SessionFakeScriptedResponse> rpc =
sessionPool.newCall(FakeDescriptor.SCRIPTED);
rpc.start(
SessionFakeScriptedRequest.getDefaultInstance(),
VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer),
f);

ExecutionException ex =
assertThrows(ExecutionException.class, () -> f.get(1, TimeUnit.SECONDS));
assertThat(ex.getCause())
.hasMessageThat()
.contains("INTERNAL: Session failed with consecutive failures.");
}

@Test
@SuppressWarnings("GuardedBy")
void consecutiveFailuresPreservesPendingRpcsWhilePoolHasInUseSessions() throws Exception {
sessionPool.start(OpenFakeSessionRequest.getDefaultInstance(), new Metadata());
waitForOpenRequestCount(5, Duration.ofSeconds(5));
assertPoolServesVRpc(sessionPool, "initial started pool");

ReentrantLock poolLock = extractPoolLock(sessionPool);
Field consecutiveFailuresField = SessionPoolImpl.class.getDeclaredField("consecutiveFailures");
consecutiveFailuresField.setAccessible(true);

UnaryResponseFuture<SessionFakeScriptedResponse> checkedOutFuture = new UnaryResponseFuture<>();
UnaryResponseFuture<SessionFakeScriptedResponse> pendingFuture = new UnaryResponseFuture<>();
VRpc<SessionFakeScriptedRequest, SessionFakeScriptedResponse> checkedOutRpc;
poolLock.lock();
try {
// Mark 4 of the 5 ready sessions closing, then check out the remaining 1 ready session via
// newCall() so readyCount == 0 and inUseCount == 1.
List<SessionHandle> handles = new ArrayList<>(sessionPool.sessions.getAllSessions());
for (int i = 1; i < handles.size(); i++) {
handles.get(i).onSessionClosing();
}
checkedOutRpc = sessionPool.newCall(FakeDescriptor.SCRIPTED);
assertThat(sessionPool.sessions.getStats().getReadyCount()).isEqualTo(0);
assertThat(sessionPool.sessions.getStats().getInUseCount()).isEqualTo(1);

// Simulate consecutive STARTING failures reaching threshold while inUseCount == 1.
Field lastCloseStatusField = SessionPoolImpl.class.getDeclaredField("lastCloseStatus");
lastCloseStatusField.setAccessible(true);
lastCloseStatusField.set(sessionPool, Status.INTERNAL.withDescription("fake internal error"));
consecutiveFailuresField.setInt(sessionPool, 10);

// A new PendingVRpc started while inUseCount > 0 must queue instead of failing fast.
VRpc<SessionFakeScriptedRequest, SessionFakeScriptedResponse> pendingRpc =
sessionPool.newCall(FakeDescriptor.SCRIPTED);
pendingRpc.start(
SessionFakeScriptedRequest.newBuilder().setTag(0).build(),
VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer),
pendingFuture);
assertThat(pendingFuture.isDone()).isFalse();
} finally {
poolLock.unlock();
}

// Complete the checked-out RPC so its session returns to the pool and drains pendingRpc.
checkedOutRpc.start(
SessionFakeScriptedRequest.newBuilder().setTag(0).build(),
VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer),
checkedOutFuture);
assertThat(checkedOutFuture.get(5, TimeUnit.SECONDS)).isNotNull();
assertThat(pendingFuture.get(5, TimeUnit.SECONDS)).isNotNull();
}

private void waitForOpenRequestCount(int minCount, Duration timeout) throws InterruptedException {
long deadline = System.currentTimeMillis() + timeout.toMillis();
while (System.currentTimeMillis() < deadline) {
Expand Down
Loading