fix: publish batch flow control metrics

google-cloud-bigtable/clirr-ignored-differences.xml

@@ -499,6 +499,18 @@
         <className>com/google/cloud/bigtable/gaxx/grpc/BigtableChannelPoolSettings$Builder</className>
         <method>com.google.cloud.bigtable.gaxx.grpc.BigtableChannelPoolSettings$Builder setLoadBalancingStrategy(com.google.cloud.bigtable.gaxx.grpc.BigtableChannelPoolSettings$LoadBalancingStrategy)</method>
     </difference>
+    <difference>
+        <!-- InternalApi was udpated -->
+        <differenceType>6001</differenceType>
+        <className>com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsConstants</className>
+        <field>*</field>
+    </difference>
+    <difference>
+        <!-- InternalApi was udpated -->
+        <differenceType>7002</differenceType>
+        <className>com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsConstants</className>
+        <method>*</method>
+    </difference>
     <difference>
         <!-- InternalApi was updated -->
         <differenceType>7004</differenceType>

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataClientFactory.java

@@ -111,7 +111,9 @@ public BigtableDataClient createDefault() {
           sharedClientContext.getClientContext().toBuilder()
               .setTracerFactory(
                   EnhancedBigtableStub.createBigtableTracerFactory(
-                      defaultSettings.getStubSettings(), sharedClientContext.getOpenTelemetry()))
+                      defaultSettings.getStubSettings(),
+                      sharedClientContext.getOpenTelemetry(),
+                      sharedClientContext.getInternalOpenTelemtry()))
               .build();
 
       return BigtableDataClient.createWithClientContext(
@@ -140,7 +142,9 @@ public BigtableDataClient createForAppProfile(@Nonnull String appProfileId) thro
         sharedClientContext.getClientContext().toBuilder()
             .setTracerFactory(
                 EnhancedBigtableStub.createBigtableTracerFactory(
-                    settings.getStubSettings(), sharedClientContext.getOpenTelemetry()))
+                    settings.getStubSettings(),
+                    sharedClientContext.getOpenTelemetry(),
+                    sharedClientContext.getInternalOpenTelemtry()))
             .build();
     return BigtableDataClient.createWithClientContext(
         settings, sharedClientContext.withClientContext(clientContext));
@@ -168,7 +172,9 @@ public BigtableDataClient createForInstance(@Nonnull String projectId, @Nonnull
         sharedClientContext.getClientContext().toBuilder()
             .setTracerFactory(
                 EnhancedBigtableStub.createBigtableTracerFactory(
-                    settings.getStubSettings(), sharedClientContext.getOpenTelemetry()))
+                    settings.getStubSettings(),
+                    sharedClientContext.getOpenTelemetry(),
+                    sharedClientContext.getInternalOpenTelemtry()))
             .build();
 
     return BigtableDataClient.createWithClientContext(
@@ -197,7 +203,9 @@ public BigtableDataClient createForInstance(
         sharedClientContext.getClientContext().toBuilder()
             .setTracerFactory(
                 EnhancedBigtableStub.createBigtableTracerFactory(
-                    settings.getStubSettings(), sharedClientContext.getOpenTelemetry()))
+                    settings.getStubSettings(),
+                    sharedClientContext.getOpenTelemetry(),
+                    sharedClientContext.getInternalOpenTelemtry()))
             .build();
     return BigtableDataClient.createWithClientContext(
         settings, sharedClientContext.withClientContext(clientContext));

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/BigtableClientContext.java

@@ -215,6 +215,10 @@ public OpenTelemetry getOpenTelemetry() {
     return this.openTelemetry;
   }
 
+  public OpenTelemetry getInternalOpenTelemtry() {
+    return this.internalOpenTelemetry;
+  }
+
   public ClientContext getClientContext() {
     return this.clientContext;
   }

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java

@@ -204,9 +204,10 @@ public static EnhancedBigtableStub create(EnhancedBigtableStubSettings settings)
       throws IOException {
     BigtableClientContext bigtableClientContext = createBigtableClientContext(settings);
     OpenTelemetry openTelemetry = bigtableClientContext.getOpenTelemetry();
+    OpenTelemetry internalOtel = bigtableClientContext.getInternalOpenTelemtry();
     ClientContext contextWithTracer =
         bigtableClientContext.getClientContext().toBuilder()
-            .setTracerFactory(createBigtableTracerFactory(settings, openTelemetry))
+            .setTracerFactory(createBigtableTracerFactory(settings, openTelemetry, internalOtel))
             .build();
     bigtableClientContext = bigtableClientContext.withClientContext(contextWithTracer);
     return new EnhancedBigtableStub(settings, bigtableClientContext);
@@ -225,18 +226,21 @@ public static BigtableClientContext createBigtableClientContext(
   }
 
   public static ApiTracerFactory createBigtableTracerFactory(
-      EnhancedBigtableStubSettings settings, @Nullable OpenTelemetry openTelemetry)
+      EnhancedBigtableStubSettings settings,
+      @Nullable OpenTelemetry openTelemetry,
+      @Nullable OpenTelemetry internalOtel)
       throws IOException {
     return createBigtableTracerFactory(
-        settings, Tags.getTagger(), Stats.getStatsRecorder(), openTelemetry);
+        settings, Tags.getTagger(), Stats.getStatsRecorder(), openTelemetry, internalOtel);
   }
 
   @VisibleForTesting
   public static ApiTracerFactory createBigtableTracerFactory(
       EnhancedBigtableStubSettings settings,
       Tagger tagger,
       StatsRecorder stats,
-      @Nullable OpenTelemetry openTelemetry)
+      @Nullable OpenTelemetry openTelemetry,
+      @Nullable OpenTelemetry internalOtel)
       throws IOException {
     String projectId = settings.getProjectId();
     String instanceId = settings.getInstanceId();
@@ -268,12 +272,13 @@ public static ApiTracerFactory createBigtableTracerFactory(
         .add(MetricsTracerFactory.create(tagger, stats, attributes))
         // Add user configured tracer
         .add(settings.getTracerFactory());
-    BuiltinMetricsTracerFactory builtinMetricsTracerFactory =
-        openTelemetry != null
-            ? BuiltinMetricsTracerFactory.create(openTelemetry, createBuiltinAttributes(settings))
-            : null;
-    if (builtinMetricsTracerFactory != null) {
-      tracerFactories.add(builtinMetricsTracerFactory);
+    if (openTelemetry != null) {
+      tracerFactories.add(
+          BuiltinMetricsTracerFactory.create(openTelemetry, createBuiltinAttributes(settings)));
+    }
+    if (internalOtel != null) {
+      tracerFactories.add(
+          BuiltinMetricsTracerFactory.create(internalOtel, createBuiltinAttributes(settings)));
     }
     return new CompositeTracerFactory(tracerFactories.build());
   }

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableCloudMonitoringExporter.java

@@ -23,7 +23,6 @@
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.FIRST_RESPONSE_LATENCIES_NAME;
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.METER_NAME;
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.OPERATION_LATENCIES_NAME;
-import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.PER_CONNECTION_ERROR_COUNT_NAME;
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.REMAINING_DEADLINE_NAME;
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.RETRY_COUNT_NAME;
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.SERVER_LATENCIES_NAME;
@@ -343,11 +342,6 @@ public Map<ProjectName, List<TimeSeries>> convert(Collection<MetricData> metricD
   }
 
   static class InternalTimeSeriesConverter implements TimeSeriesConverter {
-    private static final ImmutableList<String> APPLICATION_METRICS =
-        ImmutableSet.of(PER_CONNECTION_ERROR_COUNT_NAME).stream()
-            .map(m -> METER_NAME + m)
-            .collect(ImmutableList.toImmutableList());
-
     private final Supplier<MonitoredResource> monitoredResource;
 
     InternalTimeSeriesConverter(Supplier<MonitoredResource> monitoredResource) {

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableExporterUtils.java

@@ -25,12 +25,13 @@
 import static com.google.api.MetricDescriptor.ValueType.DISTRIBUTION;
 import static com.google.api.MetricDescriptor.ValueType.DOUBLE;
 import static com.google.api.MetricDescriptor.ValueType.INT64;
+import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.APP_PROFILE_KEY;
+import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.BIGTABLE_CLIENT_METRICS;
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.BIGTABLE_PROJECT_ID_KEY;
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.CLIENT_UID_KEY;
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.CLUSTER_ID_KEY;
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.GRPC_METRICS;
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.INSTANCE_ID_KEY;
-import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.INTERNAL_METRICS;
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.METER_NAME;
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.TABLE_ID_KEY;
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.ZONE_ID_KEY;
@@ -99,6 +100,14 @@ class BigtableExporterUtils {
       ImmutableSet.of(
           BIGTABLE_PROJECT_ID_KEY, INSTANCE_ID_KEY, TABLE_ID_KEY, CLUSTER_ID_KEY, ZONE_ID_KEY);
 
+  // These labels are defined on the bigtable_client monitored resource. For connection level
+  // metrics, they are hard coded from the settings when we create the internal otel. For per
+  // request metrics, we update the values of these fields with the values from metrics label.
+  // This is needed for BigtableDataClientFactory when the otel instance is created with the
+  // shared settings and the clients are created with different instances / app profile ids.
+  private static final Set<AttributeKey<String>> BIGTABLE_CLIENT_RESOURCE_LABEL =
+      ImmutableSet.of(BIGTABLE_PROJECT_ID_KEY, INSTANCE_ID_KEY, APP_PROFILE_KEY);
+
   private static final Map<GCPPlatformDetector.SupportedPlatform, String> SUPPORTED_PLATFORM_MAP =
       ImmutableMap.of(
           GCPPlatformDetector.SupportedPlatform.GOOGLE_COMPUTE_ENGINE, "gcp_compute_engine",
@@ -317,8 +326,10 @@ private static Optional<TimeSeries> createInternalMetricsTimeSeries(
     // To unify these:
     // - the useless views should be removed
     // - internal metrics should use relative metric names w/o the prefix
-    if (INTERNAL_METRICS.contains(metricData.getName())) {
-      metricBuilder = newApplicationMetricBuilder(metricData.getName(), pointData.getAttributes());
+    if (BIGTABLE_CLIENT_METRICS.contains(metricData.getName())) {
+      metricBuilder =
+          newApplicationMetricBuilder(
+              metricData.getName(), pointData.getAttributes(), applicationResource);
     } else if (GRPC_METRICS.containsKey(metricData.getName())) {
       metricBuilder = newGrpcMetricBuilder(metricData.getName(), pointData.getAttributes());
     } else {
@@ -343,10 +354,16 @@ private static Optional<TimeSeries> createInternalMetricsTimeSeries(
   }
 
   private static Metric.Builder newApplicationMetricBuilder(
-      String metricName, Attributes attributes) {
+      String metricName, Attributes attributes, MonitoredResource applicationResource) {
+    MonitoredResource.Builder updatedResource = applicationResource.toBuilder();
     // TODO: unify handling of metric prefixes
     Metric.Builder metricBuilder = Metric.newBuilder().setType(metricName);
     for (Map.Entry<AttributeKey<?>, Object> e : attributes.asMap().entrySet()) {
+      AttributeKey<?> key = e.getKey();
+      if (BIGTABLE_CLIENT_RESOURCE_LABEL.contains(key)) {
+        updatedResource.putLabels(key.getKey(), String.valueOf(e.getValue()));
+      }
+
       metricBuilder.putLabels(e.getKey().getKey(), String.valueOf(e.getValue()));
     }
     return metricBuilder;

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsConstants.java

@@ -57,6 +57,7 @@ public class BuiltinMetricsConstants {
   static final AttributeKey<String> TRANSPORT_REGION = AttributeKey.stringKey("transport_region");
   static final AttributeKey<String> TRANSPORT_ZONE = AttributeKey.stringKey("transport_zone");
   static final AttributeKey<String> TRANSPORT_SUBZONE = AttributeKey.stringKey("transport_subzone");
+  static final AttributeKey<String> LB_POLICY_KEY = AttributeKey.stringKey("lb_policy");
 
   // gRPC attribute keys
   // Note that these attributes keys from transformed from
@@ -167,8 +168,13 @@ public class BuiltinMetricsConstants {
                   GRPC_TARGET_KEY.getKey()))
           .build();
 
-  public static final Set<String> INTERNAL_METRICS =
-      ImmutableSet.of(PER_CONNECTION_ERROR_COUNT_NAME, OUTSTANDING_RPCS_PER_CHANNEL_NAME).stream()
+  public static final Set<String> BIGTABLE_CLIENT_METRICS =
+      ImmutableSet.of(
+              PER_CONNECTION_ERROR_COUNT_NAME,
+              OUTSTANDING_RPCS_PER_CHANNEL_NAME,
+              BATCH_WRITE_FLOW_CONTROL_FACTOR_NAME,
+              BATCH_WRITE_FLOW_CONTROL_TARGET_QPS_NAME)
+          .stream()
           .map(m -> METER_NAME + m)
           .collect(ImmutableSet.toImmutableSet());
   // End allow list of metrics that will be exported
@@ -246,8 +252,6 @@ static void defineView(
             .build();
     Set<String> attributesFilter =
         ImmutableSet.<String>builder()
-            .addAll(
-                COMMON_ATTRIBUTES.stream().map(AttributeKey::getKey).collect(Collectors.toSet()))
             .addAll(attributes.stream().map(AttributeKey::getKey).collect(Collectors.toSet()))
             .build();
     ViewBuilder viewBuilder =
@@ -259,7 +263,7 @@ static void defineView(
   }
 
   // uses cloud.BigtableClient schema
-  public static Map<InstrumentSelector, View> getInternalViews() {
+  public static Map<InstrumentSelector, View> getBigtableClientViews() {
     ImmutableMap.Builder<InstrumentSelector, View> views = ImmutableMap.builder();
     defineView(
         views,
@@ -277,12 +281,43 @@ public static Map<InstrumentSelector, View> getInternalViews() {
         InstrumentType.HISTOGRAM,
         "1",
         ImmutableSet.<AttributeKey>builder()
-            .add(BIGTABLE_PROJECT_ID_KEY, INSTANCE_ID_KEY, APP_PROFILE_KEY, CLIENT_NAME_KEY)
+            .add(
+                BIGTABLE_PROJECT_ID_KEY,
+                INSTANCE_ID_KEY,
+                APP_PROFILE_KEY,
+                TRANSPORT_TYPE,
+                STREAMING_KEY,
+                LB_POLICY_KEY)
+            .build());
+    defineView(
+        views,
+        BATCH_WRITE_FLOW_CONTROL_TARGET_QPS_NAME,
+        null,
+        InstrumentType.GAUGE,
+        "1",
+        ImmutableSet.<AttributeKey>builder()
+            .add(BIGTABLE_PROJECT_ID_KEY, INSTANCE_ID_KEY, APP_PROFILE_KEY, METHOD_KEY)
+            .build());
+    defineView(
+        views,
+        BATCH_WRITE_FLOW_CONTROL_FACTOR_NAME,
+        AGGREGATION_BATCH_WRITE_FLOW_CONTROL_FACTOR_HISTOGRAM,
+        InstrumentType.HISTOGRAM,
+        "1",
+        ImmutableSet.<AttributeKey>builder()
+            .add(
+                BIGTABLE_PROJECT_ID_KEY,
+                INSTANCE_ID_KEY,
+                APP_PROFILE_KEY,
+                STATUS_KEY,
+                APPLIED_KEY,
+                METHOD_KEY)
             .build());
     return views.build();
   }
 
-  public static Map<InstrumentSelector, View> getAllViews() {
+  // uses cloud.BigtableTable schema
+  public static Map<InstrumentSelector, View> getBigtableTableViews() {
     ImmutableMap.Builder<InstrumentSelector, View> views = ImmutableMap.builder();
 
     defineView(
@@ -373,23 +408,6 @@ public static Map<InstrumentSelector, View> getAllViews() {
             .addAll(COMMON_ATTRIBUTES)
             .add(STREAMING_KEY, STATUS_KEY)
             .build());
-    defineView(
-        views,
-        BATCH_WRITE_FLOW_CONTROL_TARGET_QPS_NAME,
-        null,
-        InstrumentType.GAUGE,
-        "1",
-        ImmutableSet.<AttributeKey>builder().addAll(COMMON_ATTRIBUTES).build());
-    defineView(
-        views,
-        BATCH_WRITE_FLOW_CONTROL_FACTOR_NAME,
-        AGGREGATION_BATCH_WRITE_FLOW_CONTROL_FACTOR_HISTOGRAM,
-        InstrumentType.HISTOGRAM,
-        "1",
-        ImmutableSet.<AttributeKey>builder()
-            .addAll(COMMON_ATTRIBUTES)
-            .add(STATUS_KEY, APPLIED_KEY)
-            .build());
     return views.build();
   }
 }

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsView.java

@@ -136,7 +136,7 @@ static void registerBuiltinMetricsWithUniverseDomain(
             executorService);
 
     for (Map.Entry<InstrumentSelector, View> entry :
-        BuiltinMetricsConstants.getAllViews().entrySet()) {
+        BuiltinMetricsConstants.getBigtableTableViews().entrySet()) {
       builder.registerView(entry.getKey(), entry.getValue());
     }
     PeriodicMetricReaderBuilder readerBuilder = PeriodicMetricReader.builder(publicExporter);

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/Util.java

@@ -265,7 +265,7 @@ public static OpenTelemetrySdk newInternalOpentelemetry(
     SdkMeterProviderBuilder meterProviderBuilder = SdkMeterProvider.builder();
 
     for (Map.Entry<InstrumentSelector, View> e :
-        BuiltinMetricsConstants.getInternalViews().entrySet()) {
+        BuiltinMetricsConstants.getBigtableClientViews().entrySet()) {
       meterProviderBuilder.registerView(e.getKey(), e.getValue());
     }
 

google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerCallableTest.java

@@ -136,6 +136,7 @@ public void sendHeaders(Metadata headers) {
                     settings.getStubSettings(),
                     Tags.getTagger(),
                     localStats.getStatsRecorder(),
+                    null,
                     null))
             .build();
     attempts = settings.getStubSettings().readRowsSettings().getRetrySettings().getMaxAttempts();
@@ -163,6 +164,7 @@ public void sendHeaders(Metadata headers) {
                     noHeaderSettings.getStubSettings(),
                     Tags.getTagger(),
                     localStats.getStatsRecorder(),
+                    null,
                     null))
             .build();
     noHeaderStub =