diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java index 57d9748cca..5d602cbe52 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java @@ -112,6 +112,7 @@ import com.google.cloud.bigtable.data.v2.stub.metrics.RpcMeasureConstants; import com.google.cloud.bigtable.data.v2.stub.metrics.StatsHeadersServerStreamingCallable; import com.google.cloud.bigtable.data.v2.stub.metrics.StatsHeadersUnaryCallable; +import com.google.cloud.bigtable.data.v2.stub.metrics.TargetTracerInterceptor; import com.google.cloud.bigtable.data.v2.stub.metrics.TracedBatcherUnaryCallable; import com.google.cloud.bigtable.data.v2.stub.mutaterows.BulkMutateRowsUserFacingCallable; import com.google.cloud.bigtable.data.v2.stub.mutaterows.MutateRowsAttemptResult; @@ -275,7 +276,9 @@ public static ClientContext createClientContext(EnhancedBigtableStubSettings set } managedChannelBuilder.intercept(errorCountPerConnectionMetricTracker.getInterceptor()); - + if(settings.getIsDirectpath()) { + managedChannelBuilder.intercept(new TargetTracerInterceptor()); + } if (oldChannelConfigurator != null) { managedChannelBuilder = oldChannelConfigurator.apply(managedChannelBuilder); } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettings.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettings.java index 339df1dbaf..d282a68939 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettings.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettings.java @@ -100,6 +100,7 @@ public class EnhancedBigtableStubSettings extends StubSettings IDEMPOTENT_RETRY_CODES = ImmutableSet.of(Code.DEADLINE_EXCEEDED, Code.UNAVAILABLE); @@ -233,6 +234,8 @@ public class EnhancedBigtableStubSettings extends StubSettings diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracer.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracer.java index 3445514f7b..3a77bfbc81 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracer.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracer.java @@ -86,4 +86,8 @@ public void setLocations(String zone, String cluster) { public void grpcChannelQueuedLatencies(long queuedTimeMs) { // noop } + + public void addTarget(String target) { + // noop + } } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsConstants.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsConstants.java index d85300828b..b6e04a94a2 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsConstants.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsConstants.java @@ -24,6 +24,7 @@ import io.opentelemetry.sdk.metrics.InstrumentSelector; import io.opentelemetry.sdk.metrics.InstrumentType; import io.opentelemetry.sdk.metrics.View; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -48,6 +49,7 @@ public class BuiltinMetricsConstants { public static final AttributeKey CLIENT_NAME_KEY = AttributeKey.stringKey("client_name"); static final AttributeKey METHOD_KEY = AttributeKey.stringKey("method"); static final AttributeKey STATUS_KEY = AttributeKey.stringKey("status"); + static AttributeKey TARGET_KEY = AttributeKey.stringKey("target"); static final AttributeKey CLIENT_UID_KEY = AttributeKey.stringKey("client_uid"); // Metric names @@ -107,7 +109,8 @@ public class BuiltinMetricsConstants { CLUSTER_ID_KEY, ZONE_ID_KEY, METHOD_KEY, - CLIENT_NAME_KEY); + CLIENT_NAME_KEY, + TARGET_KEY); static void defineView( ImmutableMap.Builder viewMap, diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracer.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracer.java index abd214d760..b0450fa78b 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracer.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracer.java @@ -22,13 +22,16 @@ import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.STATUS_KEY; import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.STREAMING_KEY; import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.TABLE_ID_KEY; +import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.TARGET_KEY; import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.ZONE_ID_KEY; import com.google.api.gax.retrying.ServerStreamingAttemptException; import com.google.api.gax.tracing.SpanName; import com.google.cloud.bigtable.Version; import com.google.common.base.Stopwatch; +import com.google.common.base.Strings; import com.google.common.math.IntMath; +import io.grpc.CallOptions; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.metrics.DoubleHistogram; import io.opentelemetry.api.metrics.LongCounter; @@ -45,7 +48,6 @@ * bigtable.googleapis.com/client namespace */ class BuiltinMetricsTracer extends BigtableTracer { - private static final String NAME = "java-bigtable/" + Version.VERSION; private final OperationType operationType; private final SpanName spanName; @@ -85,6 +87,8 @@ class BuiltinMetricsTracer extends BigtableTracer { private Long serverLatencies = null; + private String target_endpoint = "unspecified"; + // OpenCensus (and server) histogram buckets use [start, end), however OpenTelemetry uses (start, // end]. To work around this, we measure all the latencies in nanoseconds and convert them // to milliseconds and use DoubleHistogram. This should minimize the chance of a data @@ -175,6 +179,12 @@ public void attemptSucceeded() { recordAttemptCompletion(null); } + public void addTarget(String target) { + if (!Strings.isNullOrEmpty(target)) { + this.target_endpoint = target; + } + } + @Override public void attemptCancelled() { recordAttemptCompletion(new CancellationException()); @@ -338,7 +348,6 @@ private void recordAttemptCompletion(@Nullable Throwable status) { } String statusStr = Util.extractStatus(status); - Attributes attributes = baseAttributes .toBuilder() @@ -352,9 +361,8 @@ private void recordAttemptCompletion(@Nullable Throwable status) { .build(); clientBlockingLatenciesHistogram.record(convertToMs(totalClientBlockingTime.get()), attributes); - attemptLatenciesHistogram.record( - convertToMs(attemptTimer.elapsed(TimeUnit.NANOSECONDS)), attributes); + convertToMs(attemptTimer.elapsed(TimeUnit.NANOSECONDS)), attributes.toBuilder().put(TARGET_KEY,target_endpoint).build()); if (serverLatencies != null) { serverLatenciesHistogram.record(serverLatencies, attributes); diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/CompositeTracer.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/CompositeTracer.java index 774c6d9f22..1e36fb723f 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/CompositeTracer.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/CompositeTracer.java @@ -17,6 +17,7 @@ import com.google.api.gax.tracing.ApiTracer; import com.google.common.collect.ImmutableList; +import io.opentelemetry.api.internal.StringUtils; import java.util.ArrayList; import java.util.List; import javax.annotation.Nullable; @@ -225,4 +226,12 @@ public void grpcChannelQueuedLatencies(long queuedTimeMs) { tracer.grpcChannelQueuedLatencies(queuedTimeMs); } } + + public void addTarget(String target) { + if (StringUtils.isNullOrEmpty(target)) { + for (BigtableTracer tracer : bigtableTracers) { + tracer.addTarget(target); + } + } + } } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/TargetTracerInterceptor.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/TargetTracerInterceptor.java new file mode 100644 index 0000000000..5a70586e6a --- /dev/null +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/TargetTracerInterceptor.java @@ -0,0 +1,73 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.stub.metrics; + +import com.google.api.gax.grpc.GrpcCallContext; +import com.google.api.gax.tracing.ApiTracer; +import io.grpc.CallOptions; +import io.grpc.Channel; +import io.grpc.ClientCall; +import io.grpc.ClientInterceptor; +import io.grpc.ForwardingClientCall; +import io.grpc.ForwardingClientCallListener; +import io.grpc.Grpc; +import io.grpc.Metadata; +import io.grpc.MethodDescriptor; +import io.grpc.Status; +import java.net.SocketAddress; +import java.util.concurrent.atomic.LongAdder; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** An interceptor extracts remote target and appends metric. */ +public class TargetTracerInterceptor implements ClientInterceptor { + private static final Logger LOG = + Logger.getLogger(TargetTracerInterceptor.class.toString()); + + @Override + public ClientCall interceptCall( + MethodDescriptor methodDescriptor, CallOptions callOptions, Channel channel) { + ClientCall clientCall = channel.newCall(methodDescriptor,callOptions); + return new ForwardingClientCall.SimpleForwardingClientCall(clientCall) { + @Override + public void start(Listener responseListener, Metadata headers) { + super.start( + new ForwardingClientCallListener.SimpleForwardingClientCallListener( + responseListener) { + @Override + public void onHeaders(Metadata headers) { + // Connection accounting is non-critical, so we log the exception, but let normal + // processing proceed. + try { + SocketAddress remoteAddr = + clientCall.getAttributes().get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR); + ApiTracer bigtableTracer = callOptions.getOption(GrpcCallContext.TRACER_KEY); + if(bigtableTracer instanceof BuiltinMetricsTracer) { + ((BuiltinMetricsTracer)bigtableTracer).addTarget(String.valueOf(remoteAddr)); + } + } catch (Throwable t) { + LOG.log( + Level.WARNING, "Unexpected error while updating target label", t); + } + super.onHeaders(headers); + } + }, + headers); + } + }; + } + +} diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/Util.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/Util.java index 4c3fd7a42d..0eca939901 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/Util.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/Util.java @@ -64,7 +64,7 @@ public class Util { private static final Pattern SERVER_TIMING_HEADER_PATTERN = Pattern.compile(".*dur=(?\\d+)"); static final Metadata.Key LOCATION_METADATA_KEY = Metadata.Key.of("x-goog-ext-425905942-bin", Metadata.BINARY_BYTE_MARSHALLER); - + /** Convert an exception into a value that can be used to create an OpenCensus tag value. */ static String extractStatus(@Nullable Throwable error) { final String statusString; @@ -209,6 +209,7 @@ static void recordMetricsFromMetadata( if (responseParams != null && latency == null) { latency = 0L; } + // Record gfe metrics tracer.recordGfeMetadata(latency, throwable); } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/FakeServiceBuilder.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/FakeServiceBuilder.java index 5edcca2f07..eeb3e9a4d0 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/FakeServiceBuilder.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/FakeServiceBuilder.java @@ -31,6 +31,7 @@ public class FakeServiceBuilder { private final List services = new ArrayList<>(); private final List transportFilters = new ArrayList<>(); + private int serverPort = 0; public static FakeServiceBuilder create(BindableService... services) { return new FakeServiceBuilder(services); } @@ -56,6 +57,9 @@ public FakeServiceBuilder addTransportFilter(ServerTransportFilter transportFilt return this; } + public int getServerPort() { + return serverPort; + } public Server start() throws IOException { IOException lastError = null; @@ -79,6 +83,7 @@ private Server startWithoutRetries() throws IOException { port = ss.getLocalPort(); } ServerBuilder builder = ServerBuilder.forPort(port); + serverPort = port; interceptors.forEach(builder::intercept); services.forEach(builder::addService); transportFilters.forEach(builder::addTransportFilter); diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTestUtils.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTestUtils.java index 2ea4f99bdc..120f93cd44 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTestUtils.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTestUtils.java @@ -124,6 +124,7 @@ public static void verifyAttributes(MetricData metricData, Attributes attributes metricData.getHistogramData().getPoints().stream() .filter(pd -> pd.getAttributes().equals(attributes)) .collect(Collectors.toList()); + assertThat(hd).isNotEmpty(); break; case LONG_SUM: diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java index e3304acdbf..1a50e68478 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java @@ -28,6 +28,7 @@ import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.STATUS_KEY; import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.STREAMING_KEY; import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.TABLE_ID_KEY; +import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.TARGET_KEY; import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.ZONE_ID_KEY; import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsTestUtils.getAggregatedValue; import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsTestUtils.getMetricData; @@ -97,6 +98,7 @@ import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader; import java.nio.charset.Charset; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -129,6 +131,7 @@ public class BuiltinMetricsTracerTest { private static final long SERVER_LATENCY = 100; private static final long APPLICATION_LATENCY = 200; private static final long SLEEP_VARIABILITY = 15; + private static final String TARGET_ENDPOINT_VALUE_FORMAT = "localhost/127.0.0.1:%s"; private static final String CLIENT_NAME = "java-bigtable/" + Version.VERSION; private static final long CHANNEL_BLOCKING_LATENCY = 75; @@ -265,8 +268,10 @@ public void sendMessage(ReqT message) { if (oldConfigurator != null) { builder = oldConfigurator.apply(builder); } + builder.intercept(new TargetTracerInterceptor()); return builder.intercept(clientInterceptor); }); + stubSettingsBuilder.setTransportChannelProvider(channelProvider.build()); EnhancedBigtableStubSettings stubSettings = stubSettingsBuilder.build(); @@ -514,6 +519,7 @@ public void testMutateRowAttemptsTagValues() { .put(METHOD_KEY, "Bigtable.MutateRow") .put(CLIENT_NAME_KEY, CLIENT_NAME) .put(STREAMING_KEY, false) + .put(TARGET_KEY, "unspecified") .build(); Attributes expected2 = @@ -526,6 +532,7 @@ public void testMutateRowAttemptsTagValues() { .put(METHOD_KEY, "Bigtable.MutateRow") .put(CLIENT_NAME_KEY, CLIENT_NAME) .put(STREAMING_KEY, false) + .put(TARGET_KEY, String.format(TARGET_ENDPOINT_VALUE_FORMAT,String.valueOf(server.getPort()))) .build(); verifyAttributes(metricData, expected1); @@ -555,6 +562,7 @@ public void testMutateRowsPartialError() throws InterruptedException { .put(METHOD_KEY, "Bigtable.MutateRows") .put(CLIENT_NAME_KEY, CLIENT_NAME) .put(STREAMING_KEY, false) + .put(TARGET_KEY, String.format(TARGET_ENDPOINT_VALUE_FORMAT,String.valueOf(server.getPort()))) .build(); verifyAttributes(metricData, expected); @@ -605,6 +613,7 @@ public void testReadRowsAttemptsTagValues() { .put(METHOD_KEY, "Bigtable.ReadRows") .put(CLIENT_NAME_KEY, CLIENT_NAME) .put(STREAMING_KEY, true) + .put(TARGET_KEY, "unspecified") .build(); Attributes expected2 = @@ -617,6 +626,7 @@ public void testReadRowsAttemptsTagValues() { .put(METHOD_KEY, "Bigtable.ReadRows") .put(CLIENT_NAME_KEY, CLIENT_NAME) .put(STREAMING_KEY, true) + .put(TARGET_KEY, String.format(TARGET_ENDPOINT_VALUE_FORMAT,String.valueOf(server.getPort()))) .build(); verifyAttributes(metricData, expected1); @@ -718,12 +728,24 @@ public void testPermanentFailure() { .put(STREAMING_KEY, true) .put(METHOD_KEY, "Bigtable.ReadRows") .put(CLIENT_NAME_KEY, CLIENT_NAME) + .put(TARGET_KEY, "unspecified") .build(); verifyAttributes(attemptLatency, expected); MetricData opLatency = getMetricData(metricReader, OPERATION_LATENCIES_NAME); - verifyAttributes(opLatency, expected); + Attributes expectedOperationLatencyAttributes = + baseAttributes + .toBuilder() + .put(STATUS_KEY, "NOT_FOUND") + .put(TABLE_ID_KEY, BAD_TABLE_ID) + .put(CLUSTER_ID_KEY, "unspecified") + .put(ZONE_ID_KEY, "global") + .put(STREAMING_KEY, true) + .put(METHOD_KEY, "Bigtable.ReadRows") + .put(CLIENT_NAME_KEY, CLIENT_NAME) + .build(); + verifyAttributes(opLatency, expectedOperationLatencyAttributes); } private static class FakeService extends BigtableGrpc.BigtableImplBase {