Repository navigation
feat(storage): add http.client.request.duration metric for HTTP requests #14744
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: feat/storage-rpc-call-duration
Are you sure you want to change the base?
Changes from all commits
2590f0e
510824a
88cef70
57e750a
9a1c760
759422a
df4a503
d015a76
610bd86
baa4283
d6b4def
a92031d
f71f98a
f8d3db1
dfae3dd
d36f02b
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,277 @@ | ||
| /* | ||
| * Copyright 2026 Google LLC | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package com.google.cloud.storage; | ||
|
|
||
| import com.google.api.client.http.GenericUrl; | ||
| import com.google.api.client.http.HttpExecuteInterceptor; | ||
| import com.google.api.client.http.HttpIOExceptionHandler; | ||
| import com.google.api.client.http.HttpRequest; | ||
| import com.google.api.client.http.HttpRequestInitializer; | ||
| import com.google.api.client.http.HttpResponseInterceptor; | ||
| import com.google.api.client.http.HttpUnsuccessfulResponseHandler; | ||
| import com.google.api.core.ApiClock; | ||
| import com.google.common.collect.ImmutableSet; | ||
| import java.io.IOException; | ||
| import java.util.concurrent.atomic.AtomicBoolean; | ||
| import java.util.concurrent.atomic.AtomicLong; | ||
| import org.jspecify.annotations.NullMarked; | ||
| import org.jspecify.annotations.Nullable; | ||
|
|
||
| /** | ||
| * {@link HttpRequestInitializer} that instruments outgoing HTTP requests to record OpenTelemetry | ||
| * client metrics. | ||
| * | ||
| * <p>Note: {@code google-http-client} invokes {@link HttpResponseInterceptor#interceptResponse} | ||
| * inside {@link HttpRequest#execute()} when the HTTP response headers arrive, so download durations | ||
| * recorded here end at response headers and exclude streaming body transfer (a follow-up is tracked | ||
| * for recording when the response body is closed). End-to-end download duration including body | ||
| * transfer is captured by {@code gcp.client.request.duration}. | ||
| */ | ||
| @NullMarked | ||
| final class StorageHttpMetricsInterceptor implements HttpRequestInitializer { | ||
|
|
||
| private static final ImmutableSet<String> OBJECT_SUB_RESOURCES = | ||
| ImmutableSet.of("acl", "compose", "restore", "watchAll"); | ||
|
|
||
| @Nullable private final HttpRequestInitializer delegate; | ||
| @Nullable private final StorageClientMetrics metrics; | ||
| private final ApiClock clock; | ||
| @Nullable private final String defaultServerAddress; | ||
|
|
||
| StorageHttpMetricsInterceptor( | ||
| @Nullable HttpRequestInitializer delegate, | ||
| @Nullable StorageClientMetrics metrics, | ||
| ApiClock clock, | ||
| @Nullable String host) { | ||
| this.delegate = delegate; | ||
| this.metrics = metrics; | ||
| this.clock = clock; | ||
| this.defaultServerAddress = OpenTelemetryBootstrappingUtils.extractServerAddress(host); | ||
| } | ||
|
|
||
| @Override | ||
| public void initialize(HttpRequest request) throws IOException { | ||
| if (delegate != null) { | ||
| delegate.initialize(request); | ||
| } | ||
| if (metrics == null || metrics.getHttpClientRequestDuration() == null) { | ||
| return; | ||
| } | ||
|
|
||
| HttpExecuteInterceptor existingExec = request.getInterceptor(); | ||
| HttpResponseInterceptor existingResp = request.getResponseInterceptor(); | ||
| HttpUnsuccessfulResponseHandler existingUnsuccessfulHandler = | ||
| request.getUnsuccessfulResponseHandler(); | ||
| HttpIOExceptionHandler existingIoHandler = request.getIOExceptionHandler(); | ||
|
|
||
| AtomicLong startTimeNanos = new AtomicLong(); | ||
| AtomicBoolean recorded = new AtomicBoolean(true); | ||
|
|
||
| request.setInterceptor( | ||
| req -> { | ||
| if (existingExec != null) { | ||
| existingExec.intercept(req); | ||
| } | ||
| startTimeNanos.set(clock.nanoTime()); | ||
| recorded.set(false); | ||
| }); | ||
|
|
||
| if (existingUnsuccessfulHandler != null) { | ||
| request.setUnsuccessfulResponseHandler( | ||
| (req, response, supportsRetry) -> { | ||
| long endNanos = clock.nanoTime(); | ||
| boolean handled = | ||
| existingUnsuccessfulHandler.handleResponse(req, response, supportsRetry); | ||
| if (handled && recorded.compareAndSet(false, true)) { | ||
| int statusCode = response.getStatusCode(); | ||
| String errorType = | ||
| statusCode >= 400 | ||
| ? StorageErrorTypeMapper.mapHttpResponseCode(statusCode) | ||
| : StorageErrorTypeMapper.STATUS_OK; | ||
| recordDuration(req, startTimeNanos.get(), endNanos, (long) statusCode, errorType); | ||
| } | ||
| return handled; | ||
| }); | ||
| } | ||
|
|
||
| request.setResponseInterceptor( | ||
| response -> { | ||
| if (existingResp != null) { | ||
| existingResp.interceptResponse(response); | ||
| } | ||
| if (recorded.compareAndSet(false, true)) { | ||
|
nidhiii-27 marked this conversation as resolved.
Resolved
|
||
| int statusCode = response.getStatusCode(); | ||
| String errorType = | ||
| statusCode >= 400 | ||
| ? StorageErrorTypeMapper.mapHttpResponseCode(statusCode) | ||
| : StorageErrorTypeMapper.STATUS_OK; | ||
| recordDuration(request, startTimeNanos.get(), (long) statusCode, errorType); | ||
| } | ||
| }); | ||
|
|
||
| // HttpIOExceptionHandler does not receive the thrown IOException instance from | ||
| // google-http-client, so transport-level IOExceptions default to STATUS_CONNECTION_ERROR here | ||
| // (specific subtypes like TIMEOUT, DNS_FAILURE, and TLS_FAILURE are mapped in Retrying). | ||
| request.setIOExceptionHandler( | ||
|
nidhiii-27 marked this conversation as resolved.
Resolved
|
||
| (req, supportsRetry) -> { | ||
| if (recorded.compareAndSet(false, true)) { | ||
| recordDuration( | ||
| req, | ||
| startTimeNanos.get(), | ||
| StorageMetricsContext.HTTP_STATUS_UNSET, | ||
| StorageErrorTypeMapper.STATUS_CONNECTION_ERROR); | ||
| } | ||
| return existingIoHandler != null | ||
| && existingIoHandler.handleIOException(req, supportsRetry); | ||
| }); | ||
| } | ||
|
|
||
| private void recordDuration( | ||
| HttpRequest request, long startNanos, long statusCode, String errorType) { | ||
| recordDuration(request, startNanos, clock.nanoTime(), statusCode, errorType); | ||
| } | ||
|
|
||
| private void recordDuration( | ||
| HttpRequest request, long startNanos, long endNanos, long statusCode, String errorType) { | ||
| if (metrics == null) { | ||
| return; | ||
| } | ||
| double durationSeconds = Math.max(0.0, (endNanos - startNanos) / 1_000_000_000.0); | ||
| String method = request.getRequestMethod() != null ? request.getRequestMethod() : "UNKNOWN"; | ||
| String urlTemplate = extractUrlTemplate(request.getUrl()); | ||
| String serverAddress = resolveServerAddress(request.getUrl(), defaultServerAddress); | ||
|
|
||
| StorageMetricsContext.recordHttpRequestDuration( | ||
| metrics, durationSeconds, method, statusCode, urlTemplate, serverAddress, errorType); | ||
| } | ||
|
|
||
| @Nullable | ||
| static String resolveServerAddress( | ||
| @Nullable GenericUrl url, @Nullable String defaultServerAddress) { | ||
| String host = url != null ? url.getHost() : null; | ||
| if (host == null || host.isEmpty()) { | ||
| return defaultServerAddress; | ||
| } | ||
| if (host.endsWith(".storage.googleapis.com") && !host.equals("storage.googleapis.com")) { | ||
| return "storage.googleapis.com"; | ||
| } | ||
| if (defaultServerAddress != null | ||
| && !defaultServerAddress.isEmpty() | ||
| && host.endsWith("." + defaultServerAddress)) { | ||
| return defaultServerAddress; | ||
| } | ||
| return host; | ||
| } | ||
|
|
||
| static String extractUrlTemplate(@Nullable GenericUrl url) { | ||
| if (url == null) { | ||
| return "/"; | ||
| } | ||
| String path = url.getRawPath(); | ||
| if (path == null || path.isEmpty()) { | ||
| return "/"; | ||
| } | ||
| String host = url.getHost(); | ||
| if (host != null | ||
| && host.endsWith(".storage.googleapis.com") | ||
| && !host.equals("storage.googleapis.com")) { | ||
| return "/".equals(path) ? "/" : "/{object}"; | ||
| } | ||
| if (path.length() > 1 && path.endsWith("/")) { | ||
| path = path.substring(0, path.length() - 1); | ||
| } | ||
| if (path.startsWith("/upload/storage/v1/b/")) { | ||
| return "/upload/storage/v1/b/{bucket}/o"; | ||
| } | ||
| if (path.startsWith("/download/storage/v1/b/")) { | ||
| return "/download/storage/v1/b/{bucket}/o/{object}"; | ||
| } | ||
| if (path.startsWith("/batch/storage/v1")) { | ||
| return "/batch/storage/v1"; | ||
| } | ||
| if (path.startsWith("/storage/v1/")) { | ||
| return extractV1UrlTemplate(path.substring("/storage/v1/".length())); | ||
| } | ||
| String trimmed = path.startsWith("/") ? path.substring(1) : path; | ||
| if (trimmed.isEmpty()) { | ||
| return "/"; | ||
| } | ||
| String[] parts = trimmed.split("/", 2); | ||
| return parts.length == 1 ? "/{bucket}" : "/{bucket}/{object}"; | ||
| } | ||
|
|
||
| private static String extractV1UrlTemplate(String rest) { | ||
| if ("b".equals(rest)) { | ||
| return "/storage/v1/b"; | ||
| } | ||
| String[] segments = rest.split("/"); | ||
| if (segments.length >= 2 && "b".equals(segments[0])) { | ||
|
nidhiii-27 marked this conversation as resolved.
Resolved
|
||
| if (segments.length == 2) { | ||
| return "/storage/v1/b/{bucket}"; | ||
| } | ||
| if ("o".equals(segments[2])) { | ||
| return extractObjectUrlTemplate(segments); | ||
| } | ||
| if ("iam".equals(segments[2]) && segments.length > 3) { | ||
| return "/storage/v1/b/{bucket}/iam/" + segments[3]; | ||
| } | ||
| if (segments.length >= 4) { | ||
| if ("acl".equals(segments[2]) || "defaultObjectAcl".equals(segments[2])) { | ||
| return "/storage/v1/b/{bucket}/" + segments[2] + "/{entity}"; | ||
| } | ||
| if ("notificationConfigs".equals(segments[2]) || "notifications".equals(segments[2])) { | ||
| return "/storage/v1/b/{bucket}/" + segments[2] + "/{notification}"; | ||
| } | ||
| } | ||
| return "/storage/v1/b/{bucket}/" + segments[2]; | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. My earlier comment was imprecise, sorry: only raw ids should not leak; Python keeps
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done Co-authored by AI Agent |
||
| } | ||
| if (segments.length >= 2 && "projects".equals(segments[0])) { | ||
| segments[1] = "{project}"; | ||
| if (segments.length >= 4 && "hmacKeys".equals(segments[2])) { | ||
| segments[3] = "{access_id}"; | ||
| } | ||
| return "/storage/v1/" + String.join("/", segments); | ||
| } | ||
| return "/storage/v1"; | ||
| } | ||
|
|
||
| private static String extractObjectUrlTemplate(String[] segments) { | ||
| if (segments.length == 3) { | ||
| return "/storage/v1/b/{bucket}/o"; | ||
| } | ||
| if (segments.length == 4) { | ||
| return "/storage/v1/b/{bucket}/o/{object}"; | ||
| } | ||
| String action = segments[4]; | ||
| if (("copyTo".equals(action) || "rewriteTo".equals(action)) | ||
| && segments.length >= 8 | ||
| && "b".equals(segments[5]) | ||
| && "o".equals(segments[7])) { | ||
| return "/storage/v1/b/{bucket}/o/{object}/" + action + "/b/{bucket}/o/{object}"; | ||
| } | ||
| if ("moveTo".equals(action) && segments.length >= 7 && "o".equals(segments[5])) { | ||
| return "/storage/v1/b/{bucket}/o/{object}/moveTo/o/{object}"; | ||
| } | ||
| if ("acl".equals(action) && segments.length >= 6) { | ||
| return "/storage/v1/b/{bucket}/o/{object}/acl/{entity}"; | ||
| } | ||
| if (OBJECT_SUB_RESOURCES.contains(action)) { | ||
| return "/storage/v1/b/{bucket}/o/{object}/" + action; | ||
| } | ||
| return "/storage/v1/b/{bucket}/o/{object}"; | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Can we take the end time before delegating to the existing handler here? Otherwise the 401 point includes the token refresh round trip.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Done
Co-authored by AI Agent