From fe5b6214deead283299393d86199e14cab3856de Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 9 Jul 2025 10:27:20 -0400 Subject: [PATCH 01/31] Add metrics collector hooks in the right places --- src/tabular-api-surface.ts | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/tabular-api-surface.ts b/src/tabular-api-surface.ts index 462658e92..10eedf31d 100644 --- a/src/tabular-api-surface.ts +++ b/src/tabular-api-surface.ts @@ -342,6 +342,12 @@ Please use the format 'prezzy' or '${instance.name}/tables/prezzy'.`); [], ); + const metricsCollector = + this.bigtable._metricsConfigManager.createOperation( + MethodName.MUTATE_ROWS, + StreamingState.STREAMING, + this, + ); /* The following line of code sets the timeout if it was provided while creating the client. This will be used to determine if the client should @@ -388,6 +394,7 @@ Please use the format 'prezzy' or '${instance.name}/tables/prezzy'.`); // Return if the error happened before a request was made if (numRequestsMade === 0) { callback(err); + metricsCollector.onOperationComplete(err ? err.code : 0); return; } @@ -399,9 +406,11 @@ Please use the format 'prezzy' or '${instance.name}/tables/prezzy'.`); options.gaxOptions?.retry?.backoffSettings || DEFAULT_BACKOFF_SETTINGS; const nextDelay = getNextDelay(numRequestsMade, backOffSettings); + metricsCollector.onAttemptComplete(err ? err.code : 0); setTimeout(makeNextBatchRequest, nextDelay); return; } + metricsCollector.onOperationComplete(err ? err.code : 0); // If there's no more pending mutations, set the error // to null @@ -431,7 +440,9 @@ Please use the format 'prezzy' or '${instance.name}/tables/prezzy'.`); callback(err); }; + metricsCollector.onOperationStart(); const makeNextBatchRequest = () => { + metricsCollector.onAttemptStart(); const entryBatch = entries.filter((entry: Entry, index: number) => { return pendingEntryIndices.has(index); }); @@ -469,14 +480,16 @@ Please use the format 'prezzy' or '${instance.name}/tables/prezzy'.`); options.gaxOptions, ); - this.bigtable - .request({ + const requestStream = + this.bigtable.request({ client: 'BigtableClient', method: 'mutateRows', reqOpts, gaxOpts: options.gaxOptions, retryOpts, - }) + }); + metricsCollector.handleStatusAndMetadata(requestStream); + requestStream .on('error', (err: ServiceError) => { onBatchResponse(err); }) @@ -498,6 +511,7 @@ Please use the format 'prezzy' or '${instance.name}/tables/prezzy'.`); // eslint-disable-next-line @typescript-eslint/no-explicit-any (errorDetails as any).entry = originalEntry; mutationErrorsByEntryIndex.set(originalEntriesIndex, errorDetails); + metricsCollector.onResponse(); }); }) .on('end', onBatchResponse); From f74cee8c0d1e42baf3e2d0d6888cfdb8dbd2fb3c Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 9 Jul 2025 11:44:44 -0400 Subject: [PATCH 02/31] Move readrows tests over --- .../client-side-metrics-all-methods.ts | 672 ++++++++++++++++++ 1 file changed, 672 insertions(+) create mode 100644 system-test/client-side-metrics-all-methods.ts diff --git a/system-test/client-side-metrics-all-methods.ts b/system-test/client-side-metrics-all-methods.ts new file mode 100644 index 000000000..587f56bd6 --- /dev/null +++ b/system-test/client-side-metrics-all-methods.ts @@ -0,0 +1,672 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {after, before, describe, it} from 'mocha'; +import * as mocha from 'mocha'; +import { + CloudMonitoringExporter, + ExportResult, +} from '../src/client-side-metrics/exporter'; +import {ResourceMetrics} from '@opentelemetry/sdk-metrics'; +import * as assert from 'assert'; +import {GCPMetricsHandler} from '../src/client-side-metrics/gcp-metrics-handler'; +import * as proxyquire from 'proxyquire'; +import {Bigtable} from '../src'; +import {Row} from '../src/row'; +import {setupBigtable} from './client-side-metrics-setup-table'; +import {TestMetricsHandler} from '../test-common/test-metrics-handler'; +import { + OnAttemptCompleteData, + OnOperationCompleteData, +} from '../src/client-side-metrics/metrics-handler'; +import {ClientOptions, ServiceError} from 'google-gax'; +import {ClientSideMetricsConfigManager} from '../src/client-side-metrics/metrics-config-manager'; +import {MetricServiceClient} from '@google-cloud/monitoring'; + +const SECOND_PROJECT_ID = 'cfdb-sdk-node-tests'; + +function getFakeBigtable( + projectId: string, + metricsHandlerClass: typeof GCPMetricsHandler | typeof TestMetricsHandler, + apiEndpoint?: string, +) { + const metricHandler = new metricsHandlerClass({ + apiEndpoint, + } as unknown as ClientOptions & {value: string}); + const newClient = new Bigtable({ + projectId, + apiEndpoint, + }); + newClient._metricsConfigManager = new ClientSideMetricsConfigManager([ + metricHandler, + ]); + return newClient; +} + +function getHandlerFromExporter(Exporter: typeof CloudMonitoringExporter) { + return proxyquire('../src/client-side-metrics/gcp-metrics-handler.js', { + './exporter': { + CloudMonitoringExporter: Exporter, + }, + }).GCPMetricsHandler; +} + +function readRowsAssertionCheck( + projectId: string, + requestsHandled: (OnOperationCompleteData | OnAttemptCompleteData)[] = [], + method: string, + streaming: string, +) { + assert.strictEqual(requestsHandled.length, 4); + const firstRequest = requestsHandled[0] as any; + // We would expect these parameters to be different every time so delete + // them from the comparison after checking they exist. + assert(firstRequest.attemptLatency); + assert(firstRequest.serverLatency); + delete firstRequest.attemptLatency; + delete firstRequest.serverLatency; + delete firstRequest.metricsCollectorData.appProfileId; + assert.deepStrictEqual(firstRequest, { + connectivityErrorCount: 0, + streaming, + status: '0', + client_name: 'nodejs-bigtable', + metricsCollectorData: { + instanceId: 'emulator-test-instance', + table: 'my-table', + cluster: 'fake-cluster3', + zone: 'us-west1-c', + method, + }, + projectId, + }); + const secondRequest = requestsHandled[1] as any; + // We would expect these parameters to be different every time so delete + // them from the comparison after checking they exist. + assert(secondRequest.operationLatency); + assert(secondRequest.firstResponseLatency); + assert(secondRequest.applicationLatencies); + delete secondRequest.operationLatency; + delete secondRequest.firstResponseLatency; + delete secondRequest.applicationLatencies; + delete secondRequest.metricsCollectorData.appProfileId; + assert.deepStrictEqual(secondRequest, { + status: '0', + streaming, + client_name: 'nodejs-bigtable', + metricsCollectorData: { + instanceId: 'emulator-test-instance', + cluster: 'fake-cluster3', + zone: 'us-west1-c', + method, + table: 'my-table', + }, + projectId, + retryCount: 0, + }); + // We would expect these parameters to be different every time so delete + // them from the comparison after checking they exist. + const thirdRequest = requestsHandled[2] as any; + assert(thirdRequest.attemptLatency); + assert(thirdRequest.serverLatency); + delete thirdRequest.attemptLatency; + delete thirdRequest.serverLatency; + delete thirdRequest.metricsCollectorData.appProfileId; + assert.deepStrictEqual(thirdRequest, { + connectivityErrorCount: 0, + streaming, + status: '0', + client_name: 'nodejs-bigtable', + metricsCollectorData: { + instanceId: 'emulator-test-instance', + table: 'my-table2', + cluster: 'fake-cluster3', + zone: 'us-west1-c', + method, + }, + projectId, + }); + const fourthRequest = requestsHandled[3] as any; + // We would expect these parameters to be different every time so delete + // them from the comparison after checking they exist. + assert(fourthRequest.operationLatency); + assert(fourthRequest.firstResponseLatency); + assert(fourthRequest.applicationLatencies); + delete fourthRequest.operationLatency; + delete fourthRequest.firstResponseLatency; + delete fourthRequest.applicationLatencies; + delete fourthRequest.metricsCollectorData.appProfileId; + assert.deepStrictEqual(fourthRequest, { + status: '0', + streaming, + client_name: 'nodejs-bigtable', + metricsCollectorData: { + instanceId: 'emulator-test-instance', + cluster: 'fake-cluster3', + zone: 'us-west1-c', + method, + table: 'my-table2', + }, + projectId, + retryCount: 0, + }); +} + +function checkMultiRowCall( + projectId: string, + requestsHandled: (OnOperationCompleteData | OnAttemptCompleteData)[] = [], +) { + readRowsAssertionCheck( + projectId, + requestsHandled, + 'Bigtable.ReadRows', + 'true', + ); +} + +function checkSingleRowCall( + projectId: string, + requestsHandled: (OnOperationCompleteData | OnAttemptCompleteData)[] = [], +) { + readRowsAssertionCheck( + projectId, + requestsHandled, + 'Bigtable.ReadRow', + 'false', + ); +} + +/** + * Checks if metrics have been published to Google Cloud Monitoring. + * + * This asynchronous function queries Google Cloud Monitoring to verify + * that the expected metrics from the Bigtable client library have been + * successfully published. It constructs a `MetricServiceClient` to + * interact with the Cloud Monitoring API and retrieves time series data + * for a predefined set of metrics. The test passes if time series data + * is found for each of the specified metrics within a defined time + * interval. + * + * @param {string} projectId The Google Cloud project ID where metrics are + * expected to be published. + * @throws {Error} If no time series data is found for any of the specified + * metrics, indicating that the metrics were not successfully published to + * Cloud Monitoring. + */ +async function checkForPublishedMetrics(projectId: string) { + const monitoringClient = new MetricServiceClient(); // Correct instantiation + const now = Math.floor(Date.now() / 1000); + const filters = [ + 'metric.type="bigtable.googleapis.com/client/attempt_latencies"', + 'metric.type="bigtable.googleapis.com/client/operation_latencies"', + 'metric.type="bigtable.googleapis.com/client/retry_count"', + 'metric.type="bigtable.googleapis.com/client/server_latencies"', + 'metric.type="bigtable.googleapis.com/client/first_response_latencies"', + ]; + for (let i = 0; i < filters.length; i++) { + const filter = filters[i]; + const [series] = await monitoringClient.listTimeSeries({ + name: `projects/${projectId}`, + interval: { + endTime: { + seconds: now, + nanos: 0, + }, + startTime: { + seconds: now - 1000 * 60 * 60 * 24, + nanos: 0, + }, + }, + filter, + }); + assert(series.length > 0); + } +} + +describe('Bigtable/ClientSideMetrics', () => { + const instanceId1 = 'emulator-test-instance'; + const instanceId2 = 'emulator-test-instance2'; + const tableId1 = 'my-table'; + const tableId2 = 'my-table2'; + const columnFamilyId = 'cf1'; + let defaultProjectId: string; + + before(async () => { + const bigtable = new Bigtable(); + for (const instanceId of [instanceId1, instanceId2]) { + await setupBigtable(bigtable, columnFamilyId, instanceId, [ + tableId1, + tableId2, + ]); + } + defaultProjectId = await new Promise((resolve, reject) => { + bigtable.getProjectId_((err: Error | null, projectId?: string) => { + if (err) { + reject(err); + } else { + resolve(projectId as string); + } + }); + }); + }); + + after(async () => { + const bigtable = new Bigtable(); + try { + // If the instance has been deleted already by another source, we don't + // want this after hook to block the continuous integration pipeline. + const instance = bigtable.instance(instanceId1); + await instance.delete({}); + } catch (e) { + console.warn('The instance has been deleted already'); + } + try { + // If the instance has been deleted already by another source, we don't + // want this after hook to block the continuous integration pipeline. + const instance = bigtable.instance(instanceId2); + await instance.delete({}); + } catch (e) { + console.warn('The instance has been deleted already'); + } + }); + + describe('Bigtable/ClientSideMetricsToGCM', () => { + // This test suite ensures that for each test all the export calls are + // successful even when multiple instances and tables are created. + async function mockBigtable( + projectId: string, + done: mocha.Done, + apiEndpoint?: string, + ) { + /* + The exporter is called every x seconds, but we only want to test the value + it receives once. Since done cannot be called multiple times in mocha, + exported variable ensures we only test the value export receives one time. + */ + let exported = false; + /* + We need to create a timeout here because if we don't then mocha shuts down + the test as it is sleeping before the GCPMetricsHandler has a chance to + export the data. + */ + const timeout = setTimeout(() => { + if (!exported) { + done( + new Error( + 'The exporters have not completed yet and the timeout is over', + ), + ); + } + }, 120000); + + class TestExporter extends CloudMonitoringExporter { + constructor(options: ClientOptions) { + super(options); + } + + async export( + metrics: ResourceMetrics, + resultCallback: (result: ExportResult) => void, + ): Promise { + try { + await super.export(metrics, (result: ExportResult) => { + if (!exported) { + exported = true; + try { + clearTimeout(timeout); + // The test passes when the code is 0 because that means the + // result from calling export was successful. + assert.strictEqual(result.code, 0); + resultCallback({code: 0}); + void checkForPublishedMetrics(projectId).then(() => { + done(); + }); + } catch (error) { + // The code here isn't 0 so we report the original error to the mocha test runner. + done(result); + done(error); + } + } else { + resultCallback({code: 0}); + } + }); + } catch (error) { + done(error); + } + } + } + + return getFakeBigtable( + projectId, + getHandlerFromExporter(TestExporter), + apiEndpoint, + ); + } + + it('should send the metrics to Google Cloud Monitoring for a ReadRows call', done => { + (async () => { + try { + const bigtable = await mockBigtable(defaultProjectId, done); + for (const instanceId of [instanceId1, instanceId2]) { + await setupBigtable(bigtable, columnFamilyId, instanceId, [ + tableId1, + tableId2, + ]); + const instance = bigtable.instance(instanceId); + const table = instance.table(tableId1); + await table.getRows(); + const table2 = instance.table(tableId2); + await table2.getRows(); + } + } catch (e) { + done(new Error('An error occurred while running the script')); + done(e); + } + })().catch(err => { + throw err; + }); + }); + it('should send the metrics to Google Cloud Monitoring for a custom endpoint', done => { + (async () => { + try { + const bigtable = await mockBigtable( + defaultProjectId, + done, + 'bogus-endpoint', + ); + const instance = bigtable.instance(instanceId1); + const table = instance.table(tableId1); + try { + // This call will fail because we are trying to hit a bogus endpoint. + // The idea here is that we just want to record at least one metric + // so that the exporter gets executed. + await table.getRows(); + } catch (e: unknown) { + // Try blocks just need a catch/finally block. + } + } catch (e) { + done(new Error('An error occurred while running the script')); + done(e); + } + })().catch(err => { + throw err; + }); + }); + it('should send the metrics to Google Cloud Monitoring for a ReadRows call with a second project', done => { + (async () => { + try { + // This is the second project the test is configured to work with: + const projectId = SECOND_PROJECT_ID; + const bigtable = await mockBigtable(projectId, done); + for (const instanceId of [instanceId1, instanceId2]) { + await setupBigtable(bigtable, columnFamilyId, instanceId, [ + tableId1, + tableId2, + ]); + const instance = bigtable.instance(instanceId); + const table = instance.table(tableId1); + await table.getRows(); + const table2 = instance.table(tableId2); + await table2.getRows(); + } + } catch (e) { + done(new Error('An error occurred while running the script')); + done(e); + } + })().catch(err => { + throw err; + }); + }); + }); + describe('Bigtable/ClientSideMetricsToGCMTimeout', () => { + // This test suite simulates a situation where the user creates multiple + // clients and ensures that the exporter doesn't produce any errors even + // when multiple clients are attempting an export. + async function mockBigtable( + projectId: string, + done: mocha.Done, + onExportSuccess?: () => void, + ) { + class TestExporter extends CloudMonitoringExporter { + constructor(options: ClientOptions) { + super(options); + } + + async export( + metrics: ResourceMetrics, + resultCallback: (result: ExportResult) => void, + ): Promise { + try { + await super.export(metrics, async (result: ExportResult) => { + try { + // The code is expected to be 0 because the + // result from calling export was successful. + assert.strictEqual(result.code, 0); + resultCallback({code: 0}); + if (onExportSuccess) { + onExportSuccess(); + } + } catch (error) { + // The code here isn't 0 so we report the original error to the + // mocha test runner. + // The test fails here because it means that an export was + // unsuccessful. + done(result); + done(error); + resultCallback({code: 0}); + } + }); + } catch (error) { + done(error); + resultCallback({code: 0}); + } + } + } + + /* + Below we mock out the table so that it sends the metrics to a test exporter + that will still send the metrics to Google Cloud Monitoring, but then also + ensure the export was successful and pass the test with code 0 if it is + successful. + */ + return getFakeBigtable(projectId, getHandlerFromExporter(TestExporter)); + } + + it('should send the metrics to Google Cloud Monitoring for a ReadRows call', done => { + let testFinished = false; + /* + We need to create a timeout here because if we don't then mocha shuts down + the test as it is sleeping before the GCPMetricsHandler has a chance to + export the data. When the timeout is finished, if there were no export + errors then the test passes. + */ + setTimeout(() => { + testFinished = true; + done(); + }, 120000); + (async () => { + try { + const bigtable1 = await mockBigtable(defaultProjectId, done); + const bigtable2 = await mockBigtable(defaultProjectId, done); + for (const bigtable of [bigtable1, bigtable2]) { + for (const instanceId of [instanceId1, instanceId2]) { + await setupBigtable(bigtable, columnFamilyId, instanceId, [ + tableId1, + tableId2, + ]); + const instance = bigtable.instance(instanceId); + const table = instance.table(tableId1); + await table.getRows(); + const table2 = instance.table(tableId2); + await table2.getRows(); + } + } + } catch (e) { + done(new Error('An error occurred while running the script')); + done(e); + } + })().catch(err => { + throw err; + }); + }); + it('should send the metrics to Google Cloud Monitoring for a ReadRows call with thirty clients', done => { + /* + We need to create a timeout here because if we don't then mocha shuts down + the test as it is sleeping before the GCPMetricsHandler has a chance to + export the data. When the timeout is finished, if there were no export + errors then the test passes. + */ + const testTimeout = setTimeout(() => { + done(new Error('The test timed out')); + }, 480000); + let testComplete = false; + const numClients = 30; + (async () => { + try { + const bigtableList = []; + const completedSet = new Set(); + for ( + let bigtableCount = 0; + bigtableCount < numClients; + bigtableCount++ + ) { + const currentCount = bigtableCount; + const onExportSuccess = () => { + completedSet.add(currentCount); + if (completedSet.size === numClients) { + // If every client has completed the export then pass the test. + clearTimeout(testTimeout); + if (!testComplete) { + testComplete = true; + done(); + } + } + }; + bigtableList.push( + await mockBigtable(defaultProjectId, done, onExportSuccess), + ); + } + for (const bigtable of bigtableList) { + for (const instanceId of [instanceId1, instanceId2]) { + await setupBigtable(bigtable, columnFamilyId, instanceId, [ + tableId1, + tableId2, + ]); + const instance = bigtable.instance(instanceId); + const table = instance.table(tableId1); + await table.getRows(); + const table2 = instance.table(tableId2); + await table2.getRows(); + } + } + } catch (e) { + done(e); + done(new Error('An error occurred while running the script')); + } + })().catch(err => { + throw err; + }); + }); + }); + describe('Bigtable/ClientSideMetricsToMetricsHandler', () => { + async function mockBigtable( + projectId: string, + done: mocha.Done, + checkFn: ( + projectId: string, + requestsHandled: (OnOperationCompleteData | OnAttemptCompleteData)[], + ) => void, + ) { + let handlerRequestCount = 0; + class TestGCPMetricsHandler extends TestMetricsHandler { + projectId = projectId; + onOperationComplete(data: OnOperationCompleteData) { + handlerRequestCount++; + try { + super.onOperationComplete(data); + if (handlerRequestCount > 1) { + checkFn(projectId, this.requestsHandled); + done(); + } + } catch (e) { + done(e); + } + } + } + + const bigtable = getFakeBigtable(projectId, TestGCPMetricsHandler); + await setupBigtable(bigtable, columnFamilyId, instanceId1, [ + tableId1, + tableId2, + ]); + return bigtable; + } + + it('should send the metrics to the metrics handler for a ReadRows call', done => { + (async () => { + const bigtable = await mockBigtable( + defaultProjectId, + done, + checkMultiRowCall, + ); + const instance = bigtable.instance(instanceId1); + const table = instance.table(tableId1); + await table.getRows(); + const table2 = instance.table(tableId2); + await table2.getRows(); + })().catch(err => { + throw err; + }); + }); + it('should pass the projectId to the metrics handler properly', done => { + (async () => { + const bigtable = await mockBigtable( + defaultProjectId, + done, + checkMultiRowCall, + ); + const instance = bigtable.instance(instanceId1); + const table = instance.table(tableId1); + await table.getRows(); + const table2 = instance.table(tableId2); + await table2.getRows(); + })().catch(err => { + throw err; + }); + }); + it('should send the metrics to the metrics handler for a single row read', done => { + (async () => { + try { + const projectId = SECOND_PROJECT_ID; + const bigtable = await mockBigtable( + projectId, + done, + checkSingleRowCall, + ); + const instance = bigtable.instance(instanceId1); + const table = instance.table(tableId1); + const row = new Row(table, 'rowId'); + await row.get(); + const table2 = instance.table(tableId2); + const row2 = new Row(table2, 'rowId'); + await row2.get(); + } catch (e) { + done(e); + } + })().catch(err => { + throw err; + }); + }); + }); +}); From 7b45ceb3dab36215424dae2fc82903067b702d69 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 9 Jul 2025 12:01:12 -0400 Subject: [PATCH 03/31] Group ReadRows under separate describe block --- .../client-side-metrics-all-methods.ts | 402 +++++++++--------- 1 file changed, 204 insertions(+), 198 deletions(-) diff --git a/system-test/client-side-metrics-all-methods.ts b/system-test/client-side-metrics-all-methods.ts index 587f56bd6..5619e9fd7 100644 --- a/system-test/client-side-metrics-all-methods.ts +++ b/system-test/client-side-metrics-all-methods.ts @@ -354,78 +354,80 @@ describe('Bigtable/ClientSideMetrics', () => { ); } - it('should send the metrics to Google Cloud Monitoring for a ReadRows call', done => { - (async () => { - try { - const bigtable = await mockBigtable(defaultProjectId, done); - for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtable(bigtable, columnFamilyId, instanceId, [ - tableId1, - tableId2, - ]); - const instance = bigtable.instance(instanceId); - const table = instance.table(tableId1); - await table.getRows(); - const table2 = instance.table(tableId2); - await table2.getRows(); + describe('ReadRows', () => { + it('should send the metrics to Google Cloud Monitoring for a ReadRows call', done => { + (async () => { + try { + const bigtable = await mockBigtable(defaultProjectId, done); + for (const instanceId of [instanceId1, instanceId2]) { + await setupBigtable(bigtable, columnFamilyId, instanceId, [ + tableId1, + tableId2, + ]); + const instance = bigtable.instance(instanceId); + const table = instance.table(tableId1); + await table.getRows(); + const table2 = instance.table(tableId2); + await table2.getRows(); + } + } catch (e) { + done(new Error('An error occurred while running the script')); + done(e); } - } catch (e) { - done(new Error('An error occurred while running the script')); - done(e); - } - })().catch(err => { - throw err; + })().catch(err => { + throw err; + }); }); - }); - it('should send the metrics to Google Cloud Monitoring for a custom endpoint', done => { - (async () => { - try { - const bigtable = await mockBigtable( - defaultProjectId, - done, - 'bogus-endpoint', - ); - const instance = bigtable.instance(instanceId1); - const table = instance.table(tableId1); + it('should send the metrics to Google Cloud Monitoring for a custom endpoint', done => { + (async () => { try { - // This call will fail because we are trying to hit a bogus endpoint. - // The idea here is that we just want to record at least one metric - // so that the exporter gets executed. - await table.getRows(); - } catch (e: unknown) { - // Try blocks just need a catch/finally block. + const bigtable = await mockBigtable( + defaultProjectId, + done, + 'bogus-endpoint', + ); + const instance = bigtable.instance(instanceId1); + const table = instance.table(tableId1); + try { + // This call will fail because we are trying to hit a bogus endpoint. + // The idea here is that we just want to record at least one metric + // so that the exporter gets executed. + await table.getRows(); + } catch (e: unknown) { + // Try blocks just need a catch/finally block. + } + } catch (e) { + done(new Error('An error occurred while running the script')); + done(e); } - } catch (e) { - done(new Error('An error occurred while running the script')); - done(e); - } - })().catch(err => { - throw err; + })().catch(err => { + throw err; + }); }); - }); - it('should send the metrics to Google Cloud Monitoring for a ReadRows call with a second project', done => { - (async () => { - try { - // This is the second project the test is configured to work with: - const projectId = SECOND_PROJECT_ID; - const bigtable = await mockBigtable(projectId, done); - for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtable(bigtable, columnFamilyId, instanceId, [ - tableId1, - tableId2, - ]); - const instance = bigtable.instance(instanceId); - const table = instance.table(tableId1); - await table.getRows(); - const table2 = instance.table(tableId2); - await table2.getRows(); + it('should send the metrics to Google Cloud Monitoring for a ReadRows call with a second project', done => { + (async () => { + try { + // This is the second project the test is configured to work with: + const projectId = SECOND_PROJECT_ID; + const bigtable = await mockBigtable(projectId, done); + for (const instanceId of [instanceId1, instanceId2]) { + await setupBigtable(bigtable, columnFamilyId, instanceId, [ + tableId1, + tableId2, + ]); + const instance = bigtable.instance(instanceId); + const table = instance.table(tableId1); + await table.getRows(); + const table2 = instance.table(tableId2); + await table2.getRows(); + } + } catch (e) { + done(new Error('An error occurred while running the script')); + done(e); } - } catch (e) { - done(new Error('An error occurred while running the script')); - done(e); - } - })().catch(err => { - throw err; + })().catch(err => { + throw err; + }); }); }); }); @@ -483,99 +485,101 @@ describe('Bigtable/ClientSideMetrics', () => { return getFakeBigtable(projectId, getHandlerFromExporter(TestExporter)); } - it('should send the metrics to Google Cloud Monitoring for a ReadRows call', done => { - let testFinished = false; - /* - We need to create a timeout here because if we don't then mocha shuts down - the test as it is sleeping before the GCPMetricsHandler has a chance to - export the data. When the timeout is finished, if there were no export - errors then the test passes. - */ - setTimeout(() => { - testFinished = true; - done(); - }, 120000); - (async () => { - try { - const bigtable1 = await mockBigtable(defaultProjectId, done); - const bigtable2 = await mockBigtable(defaultProjectId, done); - for (const bigtable of [bigtable1, bigtable2]) { - for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtable(bigtable, columnFamilyId, instanceId, [ - tableId1, - tableId2, - ]); - const instance = bigtable.instance(instanceId); - const table = instance.table(tableId1); - await table.getRows(); - const table2 = instance.table(tableId2); - await table2.getRows(); + describe('ReadRows', () => { + it('should send the metrics to Google Cloud Monitoring for a ReadRows call', done => { + let testFinished = false; + /* + We need to create a timeout here because if we don't then mocha shuts down + the test as it is sleeping before the GCPMetricsHandler has a chance to + export the data. When the timeout is finished, if there were no export + errors then the test passes. + */ + setTimeout(() => { + testFinished = true; + done(); + }, 120000); + (async () => { + try { + const bigtable1 = await mockBigtable(defaultProjectId, done); + const bigtable2 = await mockBigtable(defaultProjectId, done); + for (const bigtable of [bigtable1, bigtable2]) { + for (const instanceId of [instanceId1, instanceId2]) { + await setupBigtable(bigtable, columnFamilyId, instanceId, [ + tableId1, + tableId2, + ]); + const instance = bigtable.instance(instanceId); + const table = instance.table(tableId1); + await table.getRows(); + const table2 = instance.table(tableId2); + await table2.getRows(); + } } + } catch (e) { + done(new Error('An error occurred while running the script')); + done(e); } - } catch (e) { - done(new Error('An error occurred while running the script')); - done(e); - } - })().catch(err => { - throw err; + })().catch(err => { + throw err; + }); }); - }); - it('should send the metrics to Google Cloud Monitoring for a ReadRows call with thirty clients', done => { - /* - We need to create a timeout here because if we don't then mocha shuts down - the test as it is sleeping before the GCPMetricsHandler has a chance to - export the data. When the timeout is finished, if there were no export - errors then the test passes. - */ - const testTimeout = setTimeout(() => { - done(new Error('The test timed out')); - }, 480000); - let testComplete = false; - const numClients = 30; - (async () => { - try { - const bigtableList = []; - const completedSet = new Set(); - for ( - let bigtableCount = 0; - bigtableCount < numClients; - bigtableCount++ - ) { - const currentCount = bigtableCount; - const onExportSuccess = () => { - completedSet.add(currentCount); - if (completedSet.size === numClients) { - // If every client has completed the export then pass the test. - clearTimeout(testTimeout); - if (!testComplete) { - testComplete = true; - done(); + it('should send the metrics to Google Cloud Monitoring for a ReadRows call with thirty clients', done => { + /* + We need to create a timeout here because if we don't then mocha shuts down + the test as it is sleeping before the GCPMetricsHandler has a chance to + export the data. When the timeout is finished, if there were no export + errors then the test passes. + */ + const testTimeout = setTimeout(() => { + done(new Error('The test timed out')); + }, 480000); + let testComplete = false; + const numClients = 30; + (async () => { + try { + const bigtableList = []; + const completedSet = new Set(); + for ( + let bigtableCount = 0; + bigtableCount < numClients; + bigtableCount++ + ) { + const currentCount = bigtableCount; + const onExportSuccess = () => { + completedSet.add(currentCount); + if (completedSet.size === numClients) { + // If every client has completed the export then pass the test. + clearTimeout(testTimeout); + if (!testComplete) { + testComplete = true; + done(); + } } + }; + bigtableList.push( + await mockBigtable(defaultProjectId, done, onExportSuccess), + ); + } + for (const bigtable of bigtableList) { + for (const instanceId of [instanceId1, instanceId2]) { + await setupBigtable(bigtable, columnFamilyId, instanceId, [ + tableId1, + tableId2, + ]); + const instance = bigtable.instance(instanceId); + const table = instance.table(tableId1); + await table.getRows(); + const table2 = instance.table(tableId2); + await table2.getRows(); } - }; - bigtableList.push( - await mockBigtable(defaultProjectId, done, onExportSuccess), - ); - } - for (const bigtable of bigtableList) { - for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtable(bigtable, columnFamilyId, instanceId, [ - tableId1, - tableId2, - ]); - const instance = bigtable.instance(instanceId); - const table = instance.table(tableId1); - await table.getRows(); - const table2 = instance.table(tableId2); - await table2.getRows(); } + } catch (e) { + done(e); + done(new Error('An error occurred while running the script')); } - } catch (e) { - done(e); - done(new Error('An error occurred while running the script')); - } - })().catch(err => { - throw err; + })().catch(err => { + throw err; + }); }); }); }); @@ -613,59 +617,61 @@ describe('Bigtable/ClientSideMetrics', () => { return bigtable; } - it('should send the metrics to the metrics handler for a ReadRows call', done => { - (async () => { - const bigtable = await mockBigtable( - defaultProjectId, - done, - checkMultiRowCall, - ); - const instance = bigtable.instance(instanceId1); - const table = instance.table(tableId1); - await table.getRows(); - const table2 = instance.table(tableId2); - await table2.getRows(); - })().catch(err => { - throw err; - }); - }); - it('should pass the projectId to the metrics handler properly', done => { - (async () => { - const bigtable = await mockBigtable( - defaultProjectId, - done, - checkMultiRowCall, - ); - const instance = bigtable.instance(instanceId1); - const table = instance.table(tableId1); - await table.getRows(); - const table2 = instance.table(tableId2); - await table2.getRows(); - })().catch(err => { - throw err; + describe('ReadRows', () => { + it('should send the metrics to the metrics handler for a ReadRows call', done => { + (async () => { + const bigtable = await mockBigtable( + defaultProjectId, + done, + checkMultiRowCall, + ); + const instance = bigtable.instance(instanceId1); + const table = instance.table(tableId1); + await table.getRows(); + const table2 = instance.table(tableId2); + await table2.getRows(); + })().catch(err => { + throw err; + }); }); - }); - it('should send the metrics to the metrics handler for a single row read', done => { - (async () => { - try { - const projectId = SECOND_PROJECT_ID; + it('should pass the projectId to the metrics handler properly', done => { + (async () => { const bigtable = await mockBigtable( - projectId, + defaultProjectId, done, - checkSingleRowCall, + checkMultiRowCall, ); const instance = bigtable.instance(instanceId1); const table = instance.table(tableId1); - const row = new Row(table, 'rowId'); - await row.get(); + await table.getRows(); const table2 = instance.table(tableId2); - const row2 = new Row(table2, 'rowId'); - await row2.get(); - } catch (e) { - done(e); - } - })().catch(err => { - throw err; + await table2.getRows(); + })().catch(err => { + throw err; + }); + }); + it('should send the metrics to the metrics handler for a single row read', done => { + (async () => { + try { + const projectId = SECOND_PROJECT_ID; + const bigtable = await mockBigtable( + projectId, + done, + checkSingleRowCall, + ); + const instance = bigtable.instance(instanceId1); + const table = instance.table(tableId1); + const row = new Row(table, 'rowId'); + await row.get(); + const table2 = instance.table(tableId2); + const row2 = new Row(table2, 'rowId'); + await row2.get(); + } catch (e) { + done(e); + } + })().catch(err => { + throw err; + }); }); }); }); From bf9a52f22f5fa20bec8f4f9d4d0cbc3567008ffa Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 9 Jul 2025 15:40:13 -0400 Subject: [PATCH 04/31] Add mutateRows tests --- .../client-side-metrics-all-methods.ts | 232 +++++++++++++++++- 1 file changed, 231 insertions(+), 1 deletion(-) diff --git a/system-test/client-side-metrics-all-methods.ts b/system-test/client-side-metrics-all-methods.ts index 5619e9fd7..63b9931bf 100644 --- a/system-test/client-side-metrics-all-methods.ts +++ b/system-test/client-side-metrics-all-methods.ts @@ -23,6 +23,7 @@ import * as assert from 'assert'; import {GCPMetricsHandler} from '../src/client-side-metrics/gcp-metrics-handler'; import * as proxyquire from 'proxyquire'; import {Bigtable} from '../src'; +import {Mutation} from '../src/mutation'; import {Row} from '../src/row'; import {setupBigtable} from './client-side-metrics-setup-table'; import {TestMetricsHandler} from '../test-common/test-metrics-handler'; @@ -30,7 +31,7 @@ import { OnAttemptCompleteData, OnOperationCompleteData, } from '../src/client-side-metrics/metrics-handler'; -import {ClientOptions, ServiceError} from 'google-gax'; +import {ClientOptions} from 'google-gax'; import {ClientSideMetricsConfigManager} from '../src/client-side-metrics/metrics-config-manager'; import {MetricServiceClient} from '@google-cloud/monitoring'; @@ -175,6 +176,18 @@ function checkMultiRowCall( ); } +function checkMutateRowsCall( + projectId: string, + requestsHandled: (OnOperationCompleteData | OnAttemptCompleteData)[] = [], +) { + readRowsAssertionCheck( + projectId, + requestsHandled, + 'Bigtable.MutateRows', + 'true', + ); +} + function checkSingleRowCall( projectId: string, requestsHandled: (OnOperationCompleteData | OnAttemptCompleteData)[] = [], @@ -187,6 +200,16 @@ function checkSingleRowCall( ); } +const mutation = { + key: 'rowId', + data: { + cf1: { + column: 1, + }, + }, + method: Mutation.methods.INSERT, +}; + /** * Checks if metrics have been published to Google Cloud Monitoring. * @@ -430,6 +453,82 @@ describe('Bigtable/ClientSideMetrics', () => { }); }); }); + describe.only('MutateRows', () => { + it('should send the metrics to Google Cloud Monitoring for a MutateRows call', done => { + (async () => { + try { + const bigtable = await mockBigtable(defaultProjectId, done); + for (const instanceId of [instanceId1, instanceId2]) { + await setupBigtable(bigtable, columnFamilyId, instanceId, [ + tableId1, + tableId2, + ]); + const instance = bigtable.instance(instanceId); + const table = instance.table(tableId1); + await table.mutate(mutation); + const table2 = instance.table(tableId2); + await table2.mutate(mutation); + } + } catch (e) { + done(new Error('An error occurred while running the script')); + done(e); + } + })().catch(err => { + throw err; + }); + }); + it('should send the metrics to Google Cloud Monitoring for a custom endpoint', done => { + (async () => { + try { + const bigtable = await mockBigtable( + defaultProjectId, + done, + 'bogus-endpoint', + ); + const instance = bigtable.instance(instanceId1); + const table = instance.table(tableId1); + try { + // This call will fail because we are trying to hit a bogus endpoint. + // The idea here is that we just want to record at least one metric + // so that the exporter gets executed. + await table.mutate(mutation); + } catch (e: unknown) { + // Try blocks just need a catch/finally block. + } + } catch (e) { + done(new Error('An error occurred while running the script')); + done(e); + } + })().catch(err => { + throw err; + }); + }); + it('should send the metrics to Google Cloud Monitoring for a MutateRows call with a second project', done => { + (async () => { + try { + // This is the second project the test is configured to work with: + const projectId = SECOND_PROJECT_ID; + const bigtable = await mockBigtable(projectId, done); + for (const instanceId of [instanceId1, instanceId2]) { + await setupBigtable(bigtable, columnFamilyId, instanceId, [ + tableId1, + tableId2, + ]); + const instance = bigtable.instance(instanceId); + const table = instance.table(tableId1); + await table.mutate(mutation); + const table2 = instance.table(tableId2); + await table2.mutate(mutation); + } + } catch (e) { + done(new Error('An error occurred while running the script')); + done(e); + } + })().catch(err => { + throw err; + }); + }); + }); }); describe('Bigtable/ClientSideMetricsToGCMTimeout', () => { // This test suite simulates a situation where the user creates multiple @@ -582,6 +681,103 @@ describe('Bigtable/ClientSideMetrics', () => { }); }); }); + describe.only('MutateRows', () => { + it('should send the metrics to Google Cloud Monitoring for a MutateRows call', done => { + let testFinished = false; + /* + We need to create a timeout here because if we don't then mocha shuts down + the test as it is sleeping before the GCPMetricsHandler has a chance to + export the data. When the timeout is finished, if there were no export + errors then the test passes. + */ + setTimeout(() => { + testFinished = true; + done(); + }, 120000); + (async () => { + try { + const bigtable1 = await mockBigtable(defaultProjectId, done); + const bigtable2 = await mockBigtable(defaultProjectId, done); + for (const bigtable of [bigtable1, bigtable2]) { + for (const instanceId of [instanceId1, instanceId2]) { + await setupBigtable(bigtable, columnFamilyId, instanceId, [ + tableId1, + tableId2, + ]); + const instance = bigtable.instance(instanceId); + const table = instance.table(tableId1); + await table.mutate(mutation); + const table2 = instance.table(tableId2); + await table2.mutate(mutation); + } + } + } catch (e) { + done(new Error('An error occurred while running the script')); + done(e); + } + })().catch(err => { + throw err; + }); + }); + it('should send the metrics to Google Cloud Monitoring for a MutateRows call with thirty clients', done => { + /* + We need to create a timeout here because if we don't then mocha shuts down + the test as it is sleeping before the GCPMetricsHandler has a chance to + export the data. When the timeout is finished, if there were no export + errors then the test passes. + */ + const testTimeout = setTimeout(() => { + done(new Error('The test timed out')); + }, 480000); + let testComplete = false; + const numClients = 30; + (async () => { + try { + const bigtableList = []; + const completedSet = new Set(); + for ( + let bigtableCount = 0; + bigtableCount < numClients; + bigtableCount++ + ) { + const currentCount = bigtableCount; + const onExportSuccess = () => { + completedSet.add(currentCount); + if (completedSet.size === numClients) { + // If every client has completed the export then pass the test. + clearTimeout(testTimeout); + if (!testComplete) { + testComplete = true; + done(); + } + } + }; + bigtableList.push( + await mockBigtable(defaultProjectId, done, onExportSuccess), + ); + } + for (const bigtable of bigtableList) { + for (const instanceId of [instanceId1, instanceId2]) { + await setupBigtable(bigtable, columnFamilyId, instanceId, [ + tableId1, + tableId2, + ]); + const instance = bigtable.instance(instanceId); + const table = instance.table(tableId1); + await table.mutate(mutation); + const table2 = instance.table(tableId2); + await table2.mutate(mutation); + } + } + } catch (e) { + done(e); + done(new Error('An error occurred while running the script')); + } + })().catch(err => { + throw err; + }); + }); + }); }); describe('Bigtable/ClientSideMetricsToMetricsHandler', () => { async function mockBigtable( @@ -674,5 +870,39 @@ describe('Bigtable/ClientSideMetrics', () => { }); }); }); + describe.only('MutateRows', () => { + it('should send the metrics to the metrics handler for a MutateRows call', done => { + (async () => { + const bigtable = await mockBigtable( + defaultProjectId, + done, + checkMutateRowsCall, + ); + const instance = bigtable.instance(instanceId1); + const table = instance.table(tableId1); + await table.mutate(mutation); + const table2 = instance.table(tableId2); + await table2.mutate(mutation); + })().catch(err => { + throw err; + }); + }); + it('should pass the projectId to the metrics handler properly', done => { + (async () => { + const bigtable = await mockBigtable( + defaultProjectId, + done, + checkMutateRowsCall, + ); + const instance = bigtable.instance(instanceId1); + const table = instance.table(tableId1); + await table.mutate(mutation); + const table2 = instance.table(tableId2); + await table2.mutate(mutation); + })().catch(err => { + throw err; + }); + }); + }); }); }); From 20167af8e5f89b805887f4a4093f212f6e8f36db Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 9 Jul 2025 16:04:55 -0400 Subject: [PATCH 05/31] Add onResponse to mutateRows collection --- src/tabular-api-surface.ts | 1 + system-test/client-side-metrics-all-methods.ts | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/tabular-api-surface.ts b/src/tabular-api-surface.ts index 10eedf31d..bda339bac 100644 --- a/src/tabular-api-surface.ts +++ b/src/tabular-api-surface.ts @@ -513,6 +513,7 @@ Please use the format 'prezzy' or '${instance.name}/tables/prezzy'.`); mutationErrorsByEntryIndex.set(originalEntriesIndex, errorDetails); metricsCollector.onResponse(); }); + metricsCollector.onResponse(); }) .on('end', onBatchResponse); numRequestsMade++; diff --git a/system-test/client-side-metrics-all-methods.ts b/system-test/client-side-metrics-all-methods.ts index 63b9931bf..f9dfa687b 100644 --- a/system-test/client-side-metrics-all-methods.ts +++ b/system-test/client-side-metrics-all-methods.ts @@ -453,7 +453,7 @@ describe('Bigtable/ClientSideMetrics', () => { }); }); }); - describe.only('MutateRows', () => { + describe('MutateRows', () => { it('should send the metrics to Google Cloud Monitoring for a MutateRows call', done => { (async () => { try { @@ -681,7 +681,7 @@ describe('Bigtable/ClientSideMetrics', () => { }); }); }); - describe.only('MutateRows', () => { + describe('MutateRows', () => { it('should send the metrics to Google Cloud Monitoring for a MutateRows call', done => { let testFinished = false; /* From 21f96e9b73208dd95d2bf1e0886821fd6002f394 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 10 Jul 2025 10:04:37 -0400 Subject: [PATCH 06/31] Eliminate the extra mutateRows calls --- src/tabular-api-surface.ts | 3 +++ system-test/client-side-metrics-all-methods.ts | 8 ++++++-- system-test/client-side-metrics-setup-table.ts | 3 +++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/tabular-api-surface.ts b/src/tabular-api-surface.ts index bda339bac..42f0420cf 100644 --- a/src/tabular-api-surface.ts +++ b/src/tabular-api-surface.ts @@ -333,6 +333,7 @@ Please use the format 'prezzy' or '${instance.name}/tables/prezzy'.`); optionsOrCallback?: MutateOptions | MutateCallback, cb?: MutateCallback, ): void | Promise { + console.trace('calling mutate'); const callback = typeof optionsOrCallback === 'function' ? optionsOrCallback : cb!; const options = @@ -391,6 +392,7 @@ Please use the format 'prezzy' or '${instance.name}/tables/prezzy'.`); }; const onBatchResponse = (err: ServiceError | null) => { + console.trace('onBatchResponse'); // Return if the error happened before a request was made if (numRequestsMade === 0) { callback(err); @@ -491,6 +493,7 @@ Please use the format 'prezzy' or '${instance.name}/tables/prezzy'.`); metricsCollector.handleStatusAndMetadata(requestStream); requestStream .on('error', (err: ServiceError) => { + console.log('error'); onBatchResponse(err); }) .on('data', (obj: google.bigtable.v2.IMutateRowsResponse) => { diff --git a/system-test/client-side-metrics-all-methods.ts b/system-test/client-side-metrics-all-methods.ts index f9dfa687b..bd0f74e23 100644 --- a/system-test/client-side-metrics-all-methods.ts +++ b/system-test/client-side-metrics-all-methods.ts @@ -267,6 +267,10 @@ describe('Bigtable/ClientSideMetrics', () => { before(async () => { const bigtable = new Bigtable(); + // For easier debugging, don't include metrics handlers in the config + // manager. This helps us step through the metrics handler for an individual + // test more easily. + bigtable._metricsConfigManager = new ClientSideMetricsConfigManager([]); for (const instanceId of [instanceId1, instanceId2]) { await setupBigtable(bigtable, columnFamilyId, instanceId, [ tableId1, @@ -870,8 +874,8 @@ describe('Bigtable/ClientSideMetrics', () => { }); }); }); - describe.only('MutateRows', () => { - it('should send the metrics to the metrics handler for a MutateRows call', done => { + describe('MutateRows', () => { + it.only('should send the metrics to the metrics handler for a MutateRows call', done => { (async () => { const bigtable = await mockBigtable( defaultProjectId, diff --git a/system-test/client-side-metrics-setup-table.ts b/system-test/client-side-metrics-setup-table.ts index 003d9b3bf..dd0e99654 100644 --- a/system-test/client-side-metrics-setup-table.ts +++ b/system-test/client-side-metrics-setup-table.ts @@ -56,6 +56,8 @@ export async function setupBigtable( } } // Add some data so that a firstResponseLatency is recorded. + /* + TODO: Add this back in later await currentTable.insert([ { key: 'rowId', @@ -67,5 +69,6 @@ export async function setupBigtable( }, }, ]); + */ } } From 689efadd31a01301ae15bfd0daff5cf285a1f6a3 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 10 Jul 2025 10:39:35 -0400 Subject: [PATCH 07/31] Change the test frame to work without inserting --- .../client-side-metrics-all-methods.ts | 64 ++++++++++++++++--- .../client-side-metrics-setup-table.ts | 23 +++++-- 2 files changed, 73 insertions(+), 14 deletions(-) diff --git a/system-test/client-side-metrics-all-methods.ts b/system-test/client-side-metrics-all-methods.ts index bd0f74e23..e6e47bdcc 100644 --- a/system-test/client-side-metrics-all-methods.ts +++ b/system-test/client-side-metrics-all-methods.ts @@ -25,7 +25,7 @@ import * as proxyquire from 'proxyquire'; import {Bigtable} from '../src'; import {Mutation} from '../src/mutation'; import {Row} from '../src/row'; -import {setupBigtable} from './client-side-metrics-setup-table'; +import {setupBigtable, setupBigtableWithoutInsert} from './client-side-metrics-setup-table'; import {TestMetricsHandler} from '../test-common/test-metrics-handler'; import { OnAttemptCompleteData, @@ -257,7 +257,7 @@ async function checkForPublishedMetrics(projectId: string) { } } -describe('Bigtable/ClientSideMetrics', () => { +describe.only('Bigtable/ClientSideMetrics', () => { const instanceId1 = 'emulator-test-instance'; const instanceId2 = 'emulator-test-instance2'; const tableId1 = 'my-table'; @@ -784,7 +784,7 @@ describe('Bigtable/ClientSideMetrics', () => { }); }); describe('Bigtable/ClientSideMetricsToMetricsHandler', () => { - async function mockBigtable( + async function getFakeBigtableWithHandler( projectId: string, done: mocha.Done, checkFn: ( @@ -808,8 +808,22 @@ describe('Bigtable/ClientSideMetrics', () => { } } } + return getFakeBigtable(projectId, TestGCPMetricsHandler); + } - const bigtable = getFakeBigtable(projectId, TestGCPMetricsHandler); + async function mockBigtableWithInserts( + projectId: string, + done: mocha.Done, + checkFn: ( + projectId: string, + requestsHandled: (OnOperationCompleteData | OnAttemptCompleteData)[], + ) => void, + ) { + const bigtable = await getFakeBigtableWithHandler( + projectId, + done, + checkFn, + ); await setupBigtable(bigtable, columnFamilyId, instanceId1, [ tableId1, tableId2, @@ -817,10 +831,40 @@ describe('Bigtable/ClientSideMetrics', () => { return bigtable; } + /** + * Returns a bigtable client with a test metrics handler that will check + * the metrics it receives and pass/fail the test if we get the right + * metrics. This method doesn't insert data so that extra mutateRows calls + * don't get made because those extra calls will produce different metrics. + * + * @param projectId + * @param done + * @param checkFn + */ + async function mockBigtableWithNoInserts( + projectId: string, + done: mocha.Done, + checkFn: ( + projectId: string, + requestsHandled: (OnOperationCompleteData | OnAttemptCompleteData)[], + ) => void, + ) { + const bigtable = await getFakeBigtableWithHandler( + projectId, + done, + checkFn, + ); + await setupBigtableWithoutInsert(bigtable, columnFamilyId, instanceId1, [ + tableId1, + tableId2, + ]); + return bigtable; + } + describe('ReadRows', () => { it('should send the metrics to the metrics handler for a ReadRows call', done => { (async () => { - const bigtable = await mockBigtable( + const bigtable = await mockBigtableWithInserts( defaultProjectId, done, checkMultiRowCall, @@ -836,7 +880,7 @@ describe('Bigtable/ClientSideMetrics', () => { }); it('should pass the projectId to the metrics handler properly', done => { (async () => { - const bigtable = await mockBigtable( + const bigtable = await mockBigtableWithInserts( defaultProjectId, done, checkMultiRowCall, @@ -854,7 +898,7 @@ describe('Bigtable/ClientSideMetrics', () => { (async () => { try { const projectId = SECOND_PROJECT_ID; - const bigtable = await mockBigtable( + const bigtable = await mockBigtableWithInserts( projectId, done, checkSingleRowCall, @@ -875,9 +919,9 @@ describe('Bigtable/ClientSideMetrics', () => { }); }); describe('MutateRows', () => { - it.only('should send the metrics to the metrics handler for a MutateRows call', done => { + it('should send the metrics to the metrics handler for a MutateRows call', done => { (async () => { - const bigtable = await mockBigtable( + const bigtable = await mockBigtableWithNoInserts( defaultProjectId, done, checkMutateRowsCall, @@ -893,7 +937,7 @@ describe('Bigtable/ClientSideMetrics', () => { }); it('should pass the projectId to the metrics handler properly', done => { (async () => { - const bigtable = await mockBigtable( + const bigtable = await mockBigtableWithNoInserts( defaultProjectId, done, checkMutateRowsCall, diff --git a/system-test/client-side-metrics-setup-table.ts b/system-test/client-side-metrics-setup-table.ts index dd0e99654..519f91f96 100644 --- a/system-test/client-side-metrics-setup-table.ts +++ b/system-test/client-side-metrics-setup-table.ts @@ -13,7 +13,7 @@ // limitations under the License. import {Bigtable} from '../src'; -export async function setupBigtable( +export async function setupBigtableWithoutInsert( bigtable: Bigtable, columnFamilyId: string, instanceId: string, @@ -56,8 +56,24 @@ export async function setupBigtable( } } // Add some data so that a firstResponseLatency is recorded. - /* - TODO: Add this back in later + } +} + +export async function setupBigtable( + bigtable: Bigtable, + columnFamilyId: string, + instanceId: string, + tableIds: string[], +) { + await setupBigtableWithoutInsert( + bigtable, + columnFamilyId, + instanceId, + tableIds, + ); + const instance = bigtable.instance(instanceId); + const tables = tableIds.map(tableId => instance.table(tableId)); + for (const currentTable of tables) { await currentTable.insert([ { key: 'rowId', @@ -69,6 +85,5 @@ export async function setupBigtable( }, }, ]); - */ } } From 9d40bf7def4da5ba44c815d10c624833872dc694 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 10 Jul 2025 10:40:41 -0400 Subject: [PATCH 08/31] Remove console traces --- src/tabular-api-surface.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/tabular-api-surface.ts b/src/tabular-api-surface.ts index 42f0420cf..146548f9a 100644 --- a/src/tabular-api-surface.ts +++ b/src/tabular-api-surface.ts @@ -333,7 +333,6 @@ Please use the format 'prezzy' or '${instance.name}/tables/prezzy'.`); optionsOrCallback?: MutateOptions | MutateCallback, cb?: MutateCallback, ): void | Promise { - console.trace('calling mutate'); const callback = typeof optionsOrCallback === 'function' ? optionsOrCallback : cb!; const options = @@ -392,7 +391,6 @@ Please use the format 'prezzy' or '${instance.name}/tables/prezzy'.`); }; const onBatchResponse = (err: ServiceError | null) => { - console.trace('onBatchResponse'); // Return if the error happened before a request was made if (numRequestsMade === 0) { callback(err); From 2bb1a59de118d871ce0fd406b69e6e954b855497 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 10 Jul 2025 11:06:55 -0400 Subject: [PATCH 09/31] Remove the error console log --- src/tabular-api-surface.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tabular-api-surface.ts b/src/tabular-api-surface.ts index 146548f9a..bda339bac 100644 --- a/src/tabular-api-surface.ts +++ b/src/tabular-api-surface.ts @@ -491,7 +491,6 @@ Please use the format 'prezzy' or '${instance.name}/tables/prezzy'.`); metricsCollector.handleStatusAndMetadata(requestStream); requestStream .on('error', (err: ServiceError) => { - console.log('error'); onBatchResponse(err); }) .on('data', (obj: google.bigtable.v2.IMutateRowsResponse) => { From 079a607b532a4192e36722dce665f45d81034fdf Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 10 Jul 2025 11:20:42 -0400 Subject: [PATCH 10/31] Inserts will conflate results for readRows too --- system-test/client-side-metrics-all-methods.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/system-test/client-side-metrics-all-methods.ts b/system-test/client-side-metrics-all-methods.ts index e6e47bdcc..0405faa6b 100644 --- a/system-test/client-side-metrics-all-methods.ts +++ b/system-test/client-side-metrics-all-methods.ts @@ -25,7 +25,10 @@ import * as proxyquire from 'proxyquire'; import {Bigtable} from '../src'; import {Mutation} from '../src/mutation'; import {Row} from '../src/row'; -import {setupBigtable, setupBigtableWithoutInsert} from './client-side-metrics-setup-table'; +import { + setupBigtable, + setupBigtableWithoutInsert, +} from './client-side-metrics-setup-table'; import {TestMetricsHandler} from '../test-common/test-metrics-handler'; import { OnAttemptCompleteData, @@ -783,7 +786,7 @@ describe.only('Bigtable/ClientSideMetrics', () => { }); }); }); - describe('Bigtable/ClientSideMetricsToMetricsHandler', () => { + describe.only('Bigtable/ClientSideMetricsToMetricsHandler', () => { async function getFakeBigtableWithHandler( projectId: string, done: mocha.Done, @@ -864,7 +867,7 @@ describe.only('Bigtable/ClientSideMetrics', () => { describe('ReadRows', () => { it('should send the metrics to the metrics handler for a ReadRows call', done => { (async () => { - const bigtable = await mockBigtableWithInserts( + const bigtable = await mockBigtableWithNoInserts( defaultProjectId, done, checkMultiRowCall, @@ -880,7 +883,7 @@ describe.only('Bigtable/ClientSideMetrics', () => { }); it('should pass the projectId to the metrics handler properly', done => { (async () => { - const bigtable = await mockBigtableWithInserts( + const bigtable = await mockBigtableWithNoInserts( defaultProjectId, done, checkMultiRowCall, @@ -898,7 +901,7 @@ describe.only('Bigtable/ClientSideMetrics', () => { (async () => { try { const projectId = SECOND_PROJECT_ID; - const bigtable = await mockBigtableWithInserts( + const bigtable = await mockBigtableWithNoInserts( projectId, done, checkSingleRowCall, From 168e132053887faea7ffad57251d99832b072197 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 10 Jul 2025 11:27:56 -0400 Subject: [PATCH 11/31] Remove only --- system-test/client-side-metrics-all-methods.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system-test/client-side-metrics-all-methods.ts b/system-test/client-side-metrics-all-methods.ts index 0405faa6b..52e4da8f2 100644 --- a/system-test/client-side-metrics-all-methods.ts +++ b/system-test/client-side-metrics-all-methods.ts @@ -786,7 +786,7 @@ describe.only('Bigtable/ClientSideMetrics', () => { }); }); }); - describe.only('Bigtable/ClientSideMetricsToMetricsHandler', () => { + describe('Bigtable/ClientSideMetricsToMetricsHandler', () => { async function getFakeBigtableWithHandler( projectId: string, done: mocha.Done, From cdbdc20d0f75c43935c570857ff8ea4daef6690f Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 10 Jul 2025 11:39:30 -0400 Subject: [PATCH 12/31] Remove only --- system-test/client-side-metrics-all-methods.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system-test/client-side-metrics-all-methods.ts b/system-test/client-side-metrics-all-methods.ts index 52e4da8f2..69fb3d0b9 100644 --- a/system-test/client-side-metrics-all-methods.ts +++ b/system-test/client-side-metrics-all-methods.ts @@ -260,7 +260,7 @@ async function checkForPublishedMetrics(projectId: string) { } } -describe.only('Bigtable/ClientSideMetrics', () => { +describe('Bigtable/ClientSideMetrics', () => { const instanceId1 = 'emulator-test-instance'; const instanceId2 = 'emulator-test-instance2'; const tableId1 = 'my-table'; From d56ddf3cf50c7242cdfcae8a4a2ff07ef5f46877 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 10 Jul 2025 14:07:06 -0400 Subject: [PATCH 13/31] Remove the extra onResponse call --- src/tabular-api-surface.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tabular-api-surface.ts b/src/tabular-api-surface.ts index bda339bac..949138c36 100644 --- a/src/tabular-api-surface.ts +++ b/src/tabular-api-surface.ts @@ -511,7 +511,6 @@ Please use the format 'prezzy' or '${instance.name}/tables/prezzy'.`); // eslint-disable-next-line @typescript-eslint/no-explicit-any (errorDetails as any).entry = originalEntry; mutationErrorsByEntryIndex.set(originalEntriesIndex, errorDetails); - metricsCollector.onResponse(); }); metricsCollector.onResponse(); }) From 9b6ed026d250379fd0d71ecd1f0cd09480586f7a Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Mon, 14 Jul 2025 11:16:54 -0400 Subject: [PATCH 14/31] Include onOperationComplete in the callback --- src/tabular-api-surface.ts | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/tabular-api-surface.ts b/src/tabular-api-surface.ts index 949138c36..82d753cbd 100644 --- a/src/tabular-api-surface.ts +++ b/src/tabular-api-surface.ts @@ -341,6 +341,14 @@ Please use the format 'prezzy' or '${instance.name}/tables/prezzy'.`); (a, b) => a.concat(b), [], ); + const collectMetricsCallback = ( + code: number, + err: ServiceError | PartialFailureError | null, + apiResponse?: google.protobuf.Empty, + ) => { + metricsCollector.onOperationComplete(code ? code : 0); + callback(err, apiResponse); + }; const metricsCollector = this.bigtable._metricsConfigManager.createOperation( @@ -393,7 +401,7 @@ Please use the format 'prezzy' or '${instance.name}/tables/prezzy'.`); const onBatchResponse = (err: ServiceError | null) => { // Return if the error happened before a request was made if (numRequestsMade === 0) { - callback(err); + collectMetricsCallback(err ? err.code : 0, err); metricsCollector.onOperationComplete(err ? err.code : 0); return; } @@ -410,7 +418,6 @@ Please use the format 'prezzy' or '${instance.name}/tables/prezzy'.`); setTimeout(makeNextBatchRequest, nextDelay); return; } - metricsCollector.onOperationComplete(err ? err.code : 0); // If there's no more pending mutations, set the error // to null @@ -420,7 +427,10 @@ Please use the format 'prezzy' or '${instance.name}/tables/prezzy'.`); const mutationErrors = Array.from(mutationErrorsByEntryIndex.values()); if (mutationErrorsByEntryIndex.size !== 0) { - callback(new PartialFailureError(mutationErrors, err)); + collectMetricsCallback( + err ? err.code : 0, + new PartialFailureError(mutationErrors, err), + ); return; } if (err) { @@ -434,10 +444,10 @@ Please use the format 'prezzy' or '${instance.name}/tables/prezzy'.`); .filter(index => !mutationErrorsByEntryIndex.has(index)) .map(() => err), ); - callback(err); + collectMetricsCallback(err ? err.code : 0, err); return; } - callback(err); + collectMetricsCallback(0, err); }; metricsCollector.onOperationStart(); From bc7097825ef8a6b505030e5df42a01c9a0bb14c0 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Mon, 14 Jul 2025 11:27:30 -0400 Subject: [PATCH 15/31] Remove the onOperationComplete call --- src/tabular-api-surface.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tabular-api-surface.ts b/src/tabular-api-surface.ts index 82d753cbd..562f50e61 100644 --- a/src/tabular-api-surface.ts +++ b/src/tabular-api-surface.ts @@ -402,7 +402,6 @@ Please use the format 'prezzy' or '${instance.name}/tables/prezzy'.`); // Return if the error happened before a request was made if (numRequestsMade === 0) { collectMetricsCallback(err ? err.code : 0, err); - metricsCollector.onOperationComplete(err ? err.code : 0); return; } From 5022a7a036ff0fc7c5cec7dbbedc357ab767118c Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Fri, 18 Jul 2025 13:53:09 -0400 Subject: [PATCH 16/31] Get rid of error code fragment --- src/tabular-api-surface.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/tabular-api-surface.ts b/src/tabular-api-surface.ts index 562f50e61..8ee651e12 100644 --- a/src/tabular-api-surface.ts +++ b/src/tabular-api-surface.ts @@ -342,10 +342,11 @@ Please use the format 'prezzy' or '${instance.name}/tables/prezzy'.`); [], ); const collectMetricsCallback = ( - code: number, + originalError: ServiceError | null, err: ServiceError | PartialFailureError | null, apiResponse?: google.protobuf.Empty, ) => { + const code = originalError ? originalError.code : 0; metricsCollector.onOperationComplete(code ? code : 0); callback(err, apiResponse); }; @@ -401,7 +402,7 @@ Please use the format 'prezzy' or '${instance.name}/tables/prezzy'.`); const onBatchResponse = (err: ServiceError | null) => { // Return if the error happened before a request was made if (numRequestsMade === 0) { - collectMetricsCallback(err ? err.code : 0, err); + collectMetricsCallback(err, err); return; } @@ -427,7 +428,7 @@ Please use the format 'prezzy' or '${instance.name}/tables/prezzy'.`); const mutationErrors = Array.from(mutationErrorsByEntryIndex.values()); if (mutationErrorsByEntryIndex.size !== 0) { collectMetricsCallback( - err ? err.code : 0, + err, new PartialFailureError(mutationErrors, err), ); return; @@ -443,10 +444,10 @@ Please use the format 'prezzy' or '${instance.name}/tables/prezzy'.`); .filter(index => !mutationErrorsByEntryIndex.has(index)) .map(() => err), ); - collectMetricsCallback(err ? err.code : 0, err); + collectMetricsCallback(err, err); return; } - collectMetricsCallback(0, err); + collectMetricsCallback(err, err); }; metricsCollector.onOperationStart(); From 9aa6ce1cd2f659ee1fc780ce6e8b104337d4fa82 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Fri, 18 Jul 2025 14:30:50 -0400 Subject: [PATCH 17/31] onResponse handler moved into metrics collector --- src/client-side-metrics/operation-metrics-collector.ts | 5 ++++- src/tabular-api-surface.ts | 1 - src/utils/createReadStreamInternal.ts | 1 - 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/client-side-metrics/operation-metrics-collector.ts b/src/client-side-metrics/operation-metrics-collector.ts index 4851eddbb..1d0e15cf7 100644 --- a/src/client-side-metrics/operation-metrics-collector.ts +++ b/src/client-side-metrics/operation-metrics-collector.ts @@ -183,7 +183,10 @@ export class OperationMetricsCollector { }) => { this.onStatusMetadataReceived(status); }, - ); + ) + .on('data', () => { + this.onResponse(); + }); } /** diff --git a/src/tabular-api-surface.ts b/src/tabular-api-surface.ts index 8ee651e12..086324a10 100644 --- a/src/tabular-api-surface.ts +++ b/src/tabular-api-surface.ts @@ -522,7 +522,6 @@ Please use the format 'prezzy' or '${instance.name}/tables/prezzy'.`); (errorDetails as any).entry = originalEntry; mutationErrorsByEntryIndex.set(originalEntriesIndex, errorDetails); }); - metricsCollector.onResponse(); }) .on('end', onBatchResponse); numRequestsMade++; diff --git a/src/utils/createReadStreamInternal.ts b/src/utils/createReadStreamInternal.ts index 9158bdc03..b3626cf3c 100644 --- a/src/utils/createReadStreamInternal.ts +++ b/src/utils/createReadStreamInternal.ts @@ -413,7 +413,6 @@ export function createReadStreamInternal( // Reset error count after a successful read so the backoff // time won't keep increasing when as stream had multiple errors numConsecutiveErrors = 0; - metricsCollector.onResponse(); }) .on('end', () => { activeRequestStream = null; From 04e4e77e3559d3ab725106e4f50d654445cf8f6b Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Mon, 21 Jul 2025 10:10:51 -0400 Subject: [PATCH 18/31] Rename handleStatusAndMetadata --- src/client-side-metrics/operation-metrics-collector.ts | 2 +- src/tabular-api-surface.ts | 2 +- src/utils/createReadStreamInternal.ts | 2 +- system-test/read-rows-acceptance-tests.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/client-side-metrics/operation-metrics-collector.ts b/src/client-side-metrics/operation-metrics-collector.ts index 1d0e15cf7..3852b81bb 100644 --- a/src/client-side-metrics/operation-metrics-collector.ts +++ b/src/client-side-metrics/operation-metrics-collector.ts @@ -168,7 +168,7 @@ export class OperationMetricsCollector { * * @param stream */ - handleStatusAndMetadata(stream: AbortableDuplex) { + wrapRequest(stream: AbortableDuplex) { stream .on( 'metadata', diff --git a/src/tabular-api-surface.ts b/src/tabular-api-surface.ts index 086324a10..8f36f8776 100644 --- a/src/tabular-api-surface.ts +++ b/src/tabular-api-surface.ts @@ -498,7 +498,7 @@ Please use the format 'prezzy' or '${instance.name}/tables/prezzy'.`); gaxOpts: options.gaxOptions, retryOpts, }); - metricsCollector.handleStatusAndMetadata(requestStream); + metricsCollector.wrapRequest(requestStream); requestStream .on('error', (err: ServiceError) => { onBatchResponse(err); diff --git a/src/utils/createReadStreamInternal.ts b/src/utils/createReadStreamInternal.ts index b3626cf3c..69283c78d 100644 --- a/src/utils/createReadStreamInternal.ts +++ b/src/utils/createReadStreamInternal.ts @@ -364,7 +364,7 @@ export function createReadStreamInternal( rowStream = pumpify.obj([requestStream, chunkTransformer, toRowStream]); - metricsCollector.handleStatusAndMetadata(requestStream); + metricsCollector.wrapRequest(requestStream); rowStream .on('error', (error: ServiceError) => { rowStreamUnpipe(rowStream, userStream); diff --git a/system-test/read-rows-acceptance-tests.ts b/system-test/read-rows-acceptance-tests.ts index 8d70db610..530ac03d7 100644 --- a/system-test/read-rows-acceptance-tests.ts +++ b/system-test/read-rows-acceptance-tests.ts @@ -41,7 +41,7 @@ class FakeOperationMetricsCollector extends OperationMetricsCollector { onAttemptStart() {} onAttemptComplete() {} onOperationStart() {} - handleStatusAndMetadata() {} + wrapRequest() {} onMetadataReceived() {} onRowReachesUser() {} onStatusMetadataReceived() {} From 524eb478042bb488d079bec71eb8871938f5bf62 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Mon, 21 Jul 2025 10:22:27 -0400 Subject: [PATCH 19/31] Add comments, shorten snippet --- src/tabular-api-surface.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/tabular-api-surface.ts b/src/tabular-api-surface.ts index 8f36f8776..898b88ef8 100644 --- a/src/tabular-api-surface.ts +++ b/src/tabular-api-surface.ts @@ -346,8 +346,19 @@ Please use the format 'prezzy' or '${instance.name}/tables/prezzy'.`); err: ServiceError | PartialFailureError | null, apiResponse?: google.protobuf.Empty, ) => { + // originalError is the error that was sent from the gapic layer. The + // compiler guarantees that it contains a code which needs to be + // provided when an operation is marked complete. + // + // err is the error we intend to send back to the user. Often it is the + // same as originalError, but in one case we construct a + // PartialFailureError and send that back to the user instead. In this + // case, we still need to pass the originalError into the method + // because the PartialFailureError doesn't have a code, but we need to + // communicate a code to the metrics collector. + // const code = originalError ? originalError.code : 0; - metricsCollector.onOperationComplete(code ? code : 0); + metricsCollector.onOperationComplete(code); callback(err, apiResponse); }; From 285651f7b88c6f836e163b442a561fa11bd5e4f4 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Mon, 21 Jul 2025 10:36:50 -0400 Subject: [PATCH 20/31] Add the wrapRequest method to the mock --- test/table.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/table.ts b/test/table.ts index 913e77c12..b0eb41891 100644 --- a/test/table.ts +++ b/test/table.ts @@ -71,7 +71,7 @@ class FakeMetricsCollector { onAttemptStart() {} onAttemptComplete() {} onMetadataReceived() {} - handleStatusAndMetadata() {} + wrapRequest() {} onStatusMetadataReceived() {} onRowReachesUser() {} } From 42366c9c59a83c24021206e3fa21dcac0bd2b3ec Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Mon, 21 Jul 2025 10:40:19 -0400 Subject: [PATCH 21/31] Pass null along instead --- src/tabular-api-surface.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tabular-api-surface.ts b/src/tabular-api-surface.ts index 898b88ef8..559450ade 100644 --- a/src/tabular-api-surface.ts +++ b/src/tabular-api-surface.ts @@ -458,7 +458,7 @@ Please use the format 'prezzy' or '${instance.name}/tables/prezzy'.`); collectMetricsCallback(err, err); return; } - collectMetricsCallback(err, err); + collectMetricsCallback(null, null); }; metricsCollector.onOperationStart(); From 1514d0cfd721797bd47f3e24d6ad4d8e695bd790 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Mon, 21 Jul 2025 15:23:03 -0400 Subject: [PATCH 22/31] Add retries comment --- src/tabular-api-surface.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/tabular-api-surface.ts b/src/tabular-api-surface.ts index 559450ade..2efacd1c9 100644 --- a/src/tabular-api-surface.ts +++ b/src/tabular-api-surface.ts @@ -421,6 +421,10 @@ Please use the format 'prezzy' or '${instance.name}/tables/prezzy'.`); timeout && timeout < new Date().getTime() - callTimeMillis ); if (isRetryable(err, timeoutExceeded)) { + // If the timeout or max retries is exceeded or if there are no + // pending indices left then the client doesn't retry. + // Otherwise, the client will retry if there is no error or if the + // error has a retryable status code. const backOffSettings = options.gaxOptions?.retry?.backoffSettings || DEFAULT_BACKOFF_SETTINGS; From 3c05e26732d0571d3c2b45a198c59192363fd23f Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Mon, 28 Jul 2025 10:25:41 -0400 Subject: [PATCH 23/31] Use the same setup table code as before --- system-test/client-side-metrics.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system-test/client-side-metrics.ts b/system-test/client-side-metrics.ts index 3d6f9d450..a75330a5c 100644 --- a/system-test/client-side-metrics.ts +++ b/system-test/client-side-metrics.ts @@ -24,7 +24,7 @@ import {GCPMetricsHandler} from '../src/client-side-metrics/gcp-metrics-handler' import * as proxyquire from 'proxyquire'; import {Bigtable} from '../src'; import {Row} from '../src/row'; -import {setupBigtable} from './client-side-metrics-setup-table'; +import {setupBigtableWithoutInsert as setupBigtable} from './client-side-metrics-setup-table'; import {TestMetricsHandler} from '../test-common/test-metrics-handler'; import { OnAttemptCompleteData, From e4d2c28835e7a4c4c2ba159f200549b9a80c80ae Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Mon, 28 Jul 2025 10:28:57 -0400 Subject: [PATCH 24/31] Rename method to setupBigtableWithInsert --- .../client-side-metrics-all-methods.ts | 22 +++++++++---------- .../client-side-metrics-setup-table.ts | 2 +- test/metric-service-client-credentials.ts | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/system-test/client-side-metrics-all-methods.ts b/system-test/client-side-metrics-all-methods.ts index 69fb3d0b9..7ddc53308 100644 --- a/system-test/client-side-metrics-all-methods.ts +++ b/system-test/client-side-metrics-all-methods.ts @@ -26,7 +26,7 @@ import {Bigtable} from '../src'; import {Mutation} from '../src/mutation'; import {Row} from '../src/row'; import { - setupBigtable, + setupBigtableWithInsert, setupBigtableWithoutInsert, } from './client-side-metrics-setup-table'; import {TestMetricsHandler} from '../test-common/test-metrics-handler'; @@ -275,7 +275,7 @@ describe('Bigtable/ClientSideMetrics', () => { // test more easily. bigtable._metricsConfigManager = new ClientSideMetricsConfigManager([]); for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtable(bigtable, columnFamilyId, instanceId, [ + await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ tableId1, tableId2, ]); @@ -390,7 +390,7 @@ describe('Bigtable/ClientSideMetrics', () => { try { const bigtable = await mockBigtable(defaultProjectId, done); for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtable(bigtable, columnFamilyId, instanceId, [ + await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ tableId1, tableId2, ]); @@ -441,7 +441,7 @@ describe('Bigtable/ClientSideMetrics', () => { const projectId = SECOND_PROJECT_ID; const bigtable = await mockBigtable(projectId, done); for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtable(bigtable, columnFamilyId, instanceId, [ + await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ tableId1, tableId2, ]); @@ -466,7 +466,7 @@ describe('Bigtable/ClientSideMetrics', () => { try { const bigtable = await mockBigtable(defaultProjectId, done); for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtable(bigtable, columnFamilyId, instanceId, [ + await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ tableId1, tableId2, ]); @@ -517,7 +517,7 @@ describe('Bigtable/ClientSideMetrics', () => { const projectId = SECOND_PROJECT_ID; const bigtable = await mockBigtable(projectId, done); for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtable(bigtable, columnFamilyId, instanceId, [ + await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ tableId1, tableId2, ]); @@ -610,7 +610,7 @@ describe('Bigtable/ClientSideMetrics', () => { const bigtable2 = await mockBigtable(defaultProjectId, done); for (const bigtable of [bigtable1, bigtable2]) { for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtable(bigtable, columnFamilyId, instanceId, [ + await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ tableId1, tableId2, ]); @@ -668,7 +668,7 @@ describe('Bigtable/ClientSideMetrics', () => { } for (const bigtable of bigtableList) { for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtable(bigtable, columnFamilyId, instanceId, [ + await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ tableId1, tableId2, ]); @@ -707,7 +707,7 @@ describe('Bigtable/ClientSideMetrics', () => { const bigtable2 = await mockBigtable(defaultProjectId, done); for (const bigtable of [bigtable1, bigtable2]) { for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtable(bigtable, columnFamilyId, instanceId, [ + await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ tableId1, tableId2, ]); @@ -765,7 +765,7 @@ describe('Bigtable/ClientSideMetrics', () => { } for (const bigtable of bigtableList) { for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtable(bigtable, columnFamilyId, instanceId, [ + await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ tableId1, tableId2, ]); @@ -827,7 +827,7 @@ describe('Bigtable/ClientSideMetrics', () => { done, checkFn, ); - await setupBigtable(bigtable, columnFamilyId, instanceId1, [ + await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId1, [ tableId1, tableId2, ]); diff --git a/system-test/client-side-metrics-setup-table.ts b/system-test/client-side-metrics-setup-table.ts index 519f91f96..e8d109e0f 100644 --- a/system-test/client-side-metrics-setup-table.ts +++ b/system-test/client-side-metrics-setup-table.ts @@ -59,7 +59,7 @@ export async function setupBigtableWithoutInsert( } } -export async function setupBigtable( +export async function setupBigtableWithInsert( bigtable: Bigtable, columnFamilyId: string, instanceId: string, diff --git a/test/metric-service-client-credentials.ts b/test/metric-service-client-credentials.ts index edd8001eb..2506d9fca 100644 --- a/test/metric-service-client-credentials.ts +++ b/test/metric-service-client-credentials.ts @@ -15,7 +15,7 @@ import * as proxyquire from 'proxyquire'; import {ClientOptions, grpc} from 'google-gax'; import * as assert from 'assert'; -import {setupBigtable} from '../system-test/client-side-metrics-setup-table'; +import {setupBigtableWithInsert} from '../system-test/client-side-metrics-setup-table'; import {MetricServiceClient} from '@google-cloud/monitoring'; describe('Bigtable/MetricServiceClientCredentials', () => { From 33f449a831fb0724253775af8e624c5b19f42114 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Mon, 28 Jul 2025 10:30:24 -0400 Subject: [PATCH 25/31] Keep setupBigtable name as setupBigtable --- system-test/client-side-metrics-all-methods.ts | 4 ++-- system-test/client-side-metrics-setup-table.ts | 4 ++-- system-test/client-side-metrics.ts | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/system-test/client-side-metrics-all-methods.ts b/system-test/client-side-metrics-all-methods.ts index 7ddc53308..e7556bf39 100644 --- a/system-test/client-side-metrics-all-methods.ts +++ b/system-test/client-side-metrics-all-methods.ts @@ -27,7 +27,7 @@ import {Mutation} from '../src/mutation'; import {Row} from '../src/row'; import { setupBigtableWithInsert, - setupBigtableWithoutInsert, + setupBigtable, } from './client-side-metrics-setup-table'; import {TestMetricsHandler} from '../test-common/test-metrics-handler'; import { @@ -857,7 +857,7 @@ describe('Bigtable/ClientSideMetrics', () => { done, checkFn, ); - await setupBigtableWithoutInsert(bigtable, columnFamilyId, instanceId1, [ + await setupBigtable(bigtable, columnFamilyId, instanceId1, [ tableId1, tableId2, ]); diff --git a/system-test/client-side-metrics-setup-table.ts b/system-test/client-side-metrics-setup-table.ts index e8d109e0f..4f56d9c5b 100644 --- a/system-test/client-side-metrics-setup-table.ts +++ b/system-test/client-side-metrics-setup-table.ts @@ -13,7 +13,7 @@ // limitations under the License. import {Bigtable} from '../src'; -export async function setupBigtableWithoutInsert( +export async function setupBigtable( bigtable: Bigtable, columnFamilyId: string, instanceId: string, @@ -65,7 +65,7 @@ export async function setupBigtableWithInsert( instanceId: string, tableIds: string[], ) { - await setupBigtableWithoutInsert( + await setupBigtable( bigtable, columnFamilyId, instanceId, diff --git a/system-test/client-side-metrics.ts b/system-test/client-side-metrics.ts index a75330a5c..3d6f9d450 100644 --- a/system-test/client-side-metrics.ts +++ b/system-test/client-side-metrics.ts @@ -24,7 +24,7 @@ import {GCPMetricsHandler} from '../src/client-side-metrics/gcp-metrics-handler' import * as proxyquire from 'proxyquire'; import {Bigtable} from '../src'; import {Row} from '../src/row'; -import {setupBigtableWithoutInsert as setupBigtable} from './client-side-metrics-setup-table'; +import {setupBigtable} from './client-side-metrics-setup-table'; import {TestMetricsHandler} from '../test-common/test-metrics-handler'; import { OnAttemptCompleteData, From 926c480aa4d07763d47ab994336f42cd03c77f80 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Mon, 28 Jul 2025 10:33:05 -0400 Subject: [PATCH 26/31] Eliminate unused import --- test/metric-service-client-credentials.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/test/metric-service-client-credentials.ts b/test/metric-service-client-credentials.ts index 2506d9fca..f1ec1a881 100644 --- a/test/metric-service-client-credentials.ts +++ b/test/metric-service-client-credentials.ts @@ -15,7 +15,6 @@ import * as proxyquire from 'proxyquire'; import {ClientOptions, grpc} from 'google-gax'; import * as assert from 'assert'; -import {setupBigtableWithInsert} from '../system-test/client-side-metrics-setup-table'; import {MetricServiceClient} from '@google-cloud/monitoring'; describe('Bigtable/MetricServiceClientCredentials', () => { From a82647ba4d45481706e25df52bec2e4f64406caa Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Mon, 28 Jul 2025 14:36:43 +0000 Subject: [PATCH 27/31] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .../client-side-metrics-all-methods.ts | 80 +++++++++++-------- .../client-side-metrics-setup-table.ts | 7 +- 2 files changed, 49 insertions(+), 38 deletions(-) diff --git a/system-test/client-side-metrics-all-methods.ts b/system-test/client-side-metrics-all-methods.ts index e7556bf39..63ed95d5c 100644 --- a/system-test/client-side-metrics-all-methods.ts +++ b/system-test/client-side-metrics-all-methods.ts @@ -390,10 +390,12 @@ describe('Bigtable/ClientSideMetrics', () => { try { const bigtable = await mockBigtable(defaultProjectId, done); for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ - tableId1, - tableId2, - ]); + await setupBigtableWithInsert( + bigtable, + columnFamilyId, + instanceId, + [tableId1, tableId2], + ); const instance = bigtable.instance(instanceId); const table = instance.table(tableId1); await table.getRows(); @@ -441,10 +443,12 @@ describe('Bigtable/ClientSideMetrics', () => { const projectId = SECOND_PROJECT_ID; const bigtable = await mockBigtable(projectId, done); for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ - tableId1, - tableId2, - ]); + await setupBigtableWithInsert( + bigtable, + columnFamilyId, + instanceId, + [tableId1, tableId2], + ); const instance = bigtable.instance(instanceId); const table = instance.table(tableId1); await table.getRows(); @@ -466,10 +470,12 @@ describe('Bigtable/ClientSideMetrics', () => { try { const bigtable = await mockBigtable(defaultProjectId, done); for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ - tableId1, - tableId2, - ]); + await setupBigtableWithInsert( + bigtable, + columnFamilyId, + instanceId, + [tableId1, tableId2], + ); const instance = bigtable.instance(instanceId); const table = instance.table(tableId1); await table.mutate(mutation); @@ -517,10 +523,12 @@ describe('Bigtable/ClientSideMetrics', () => { const projectId = SECOND_PROJECT_ID; const bigtable = await mockBigtable(projectId, done); for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ - tableId1, - tableId2, - ]); + await setupBigtableWithInsert( + bigtable, + columnFamilyId, + instanceId, + [tableId1, tableId2], + ); const instance = bigtable.instance(instanceId); const table = instance.table(tableId1); await table.mutate(mutation); @@ -610,10 +618,12 @@ describe('Bigtable/ClientSideMetrics', () => { const bigtable2 = await mockBigtable(defaultProjectId, done); for (const bigtable of [bigtable1, bigtable2]) { for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ - tableId1, - tableId2, - ]); + await setupBigtableWithInsert( + bigtable, + columnFamilyId, + instanceId, + [tableId1, tableId2], + ); const instance = bigtable.instance(instanceId); const table = instance.table(tableId1); await table.getRows(); @@ -668,10 +678,12 @@ describe('Bigtable/ClientSideMetrics', () => { } for (const bigtable of bigtableList) { for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ - tableId1, - tableId2, - ]); + await setupBigtableWithInsert( + bigtable, + columnFamilyId, + instanceId, + [tableId1, tableId2], + ); const instance = bigtable.instance(instanceId); const table = instance.table(tableId1); await table.getRows(); @@ -707,10 +719,12 @@ describe('Bigtable/ClientSideMetrics', () => { const bigtable2 = await mockBigtable(defaultProjectId, done); for (const bigtable of [bigtable1, bigtable2]) { for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ - tableId1, - tableId2, - ]); + await setupBigtableWithInsert( + bigtable, + columnFamilyId, + instanceId, + [tableId1, tableId2], + ); const instance = bigtable.instance(instanceId); const table = instance.table(tableId1); await table.mutate(mutation); @@ -765,10 +779,12 @@ describe('Bigtable/ClientSideMetrics', () => { } for (const bigtable of bigtableList) { for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ - tableId1, - tableId2, - ]); + await setupBigtableWithInsert( + bigtable, + columnFamilyId, + instanceId, + [tableId1, tableId2], + ); const instance = bigtable.instance(instanceId); const table = instance.table(tableId1); await table.mutate(mutation); diff --git a/system-test/client-side-metrics-setup-table.ts b/system-test/client-side-metrics-setup-table.ts index 4f56d9c5b..4bac03e49 100644 --- a/system-test/client-side-metrics-setup-table.ts +++ b/system-test/client-side-metrics-setup-table.ts @@ -65,12 +65,7 @@ export async function setupBigtableWithInsert( instanceId: string, tableIds: string[], ) { - await setupBigtable( - bigtable, - columnFamilyId, - instanceId, - tableIds, - ); + await setupBigtable(bigtable, columnFamilyId, instanceId, tableIds); const instance = bigtable.instance(instanceId); const tables = tableIds.map(tableId => instance.table(tableId)); for (const currentTable of tables) { From a0de09b7500711683e483bb283765fb1cfc81ab3 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Mon, 28 Jul 2025 14:39:41 +0000 Subject: [PATCH 28/31] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .../client-side-metrics-all-methods.ts | 80 +++++++++++-------- .../client-side-metrics-setup-table.ts | 7 +- 2 files changed, 49 insertions(+), 38 deletions(-) diff --git a/system-test/client-side-metrics-all-methods.ts b/system-test/client-side-metrics-all-methods.ts index e7556bf39..63ed95d5c 100644 --- a/system-test/client-side-metrics-all-methods.ts +++ b/system-test/client-side-metrics-all-methods.ts @@ -390,10 +390,12 @@ describe('Bigtable/ClientSideMetrics', () => { try { const bigtable = await mockBigtable(defaultProjectId, done); for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ - tableId1, - tableId2, - ]); + await setupBigtableWithInsert( + bigtable, + columnFamilyId, + instanceId, + [tableId1, tableId2], + ); const instance = bigtable.instance(instanceId); const table = instance.table(tableId1); await table.getRows(); @@ -441,10 +443,12 @@ describe('Bigtable/ClientSideMetrics', () => { const projectId = SECOND_PROJECT_ID; const bigtable = await mockBigtable(projectId, done); for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ - tableId1, - tableId2, - ]); + await setupBigtableWithInsert( + bigtable, + columnFamilyId, + instanceId, + [tableId1, tableId2], + ); const instance = bigtable.instance(instanceId); const table = instance.table(tableId1); await table.getRows(); @@ -466,10 +470,12 @@ describe('Bigtable/ClientSideMetrics', () => { try { const bigtable = await mockBigtable(defaultProjectId, done); for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ - tableId1, - tableId2, - ]); + await setupBigtableWithInsert( + bigtable, + columnFamilyId, + instanceId, + [tableId1, tableId2], + ); const instance = bigtable.instance(instanceId); const table = instance.table(tableId1); await table.mutate(mutation); @@ -517,10 +523,12 @@ describe('Bigtable/ClientSideMetrics', () => { const projectId = SECOND_PROJECT_ID; const bigtable = await mockBigtable(projectId, done); for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ - tableId1, - tableId2, - ]); + await setupBigtableWithInsert( + bigtable, + columnFamilyId, + instanceId, + [tableId1, tableId2], + ); const instance = bigtable.instance(instanceId); const table = instance.table(tableId1); await table.mutate(mutation); @@ -610,10 +618,12 @@ describe('Bigtable/ClientSideMetrics', () => { const bigtable2 = await mockBigtable(defaultProjectId, done); for (const bigtable of [bigtable1, bigtable2]) { for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ - tableId1, - tableId2, - ]); + await setupBigtableWithInsert( + bigtable, + columnFamilyId, + instanceId, + [tableId1, tableId2], + ); const instance = bigtable.instance(instanceId); const table = instance.table(tableId1); await table.getRows(); @@ -668,10 +678,12 @@ describe('Bigtable/ClientSideMetrics', () => { } for (const bigtable of bigtableList) { for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ - tableId1, - tableId2, - ]); + await setupBigtableWithInsert( + bigtable, + columnFamilyId, + instanceId, + [tableId1, tableId2], + ); const instance = bigtable.instance(instanceId); const table = instance.table(tableId1); await table.getRows(); @@ -707,10 +719,12 @@ describe('Bigtable/ClientSideMetrics', () => { const bigtable2 = await mockBigtable(defaultProjectId, done); for (const bigtable of [bigtable1, bigtable2]) { for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ - tableId1, - tableId2, - ]); + await setupBigtableWithInsert( + bigtable, + columnFamilyId, + instanceId, + [tableId1, tableId2], + ); const instance = bigtable.instance(instanceId); const table = instance.table(tableId1); await table.mutate(mutation); @@ -765,10 +779,12 @@ describe('Bigtable/ClientSideMetrics', () => { } for (const bigtable of bigtableList) { for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ - tableId1, - tableId2, - ]); + await setupBigtableWithInsert( + bigtable, + columnFamilyId, + instanceId, + [tableId1, tableId2], + ); const instance = bigtable.instance(instanceId); const table = instance.table(tableId1); await table.mutate(mutation); diff --git a/system-test/client-side-metrics-setup-table.ts b/system-test/client-side-metrics-setup-table.ts index 4f56d9c5b..4bac03e49 100644 --- a/system-test/client-side-metrics-setup-table.ts +++ b/system-test/client-side-metrics-setup-table.ts @@ -65,12 +65,7 @@ export async function setupBigtableWithInsert( instanceId: string, tableIds: string[], ) { - await setupBigtable( - bigtable, - columnFamilyId, - instanceId, - tableIds, - ); + await setupBigtable(bigtable, columnFamilyId, instanceId, tableIds); const instance = bigtable.instance(instanceId); const tables = tableIds.map(tableId => instance.table(tableId)); for (const currentTable of tables) { From 9b13fba79f346bc8a91d5f8bd24ecd6b2ed9064a Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Mon, 28 Jul 2025 10:45:53 -0400 Subject: [PATCH 29/31] Run the linter --- .../client-side-metrics-all-methods.ts | 80 +++++++++++-------- .../client-side-metrics-setup-table.ts | 7 +- 2 files changed, 49 insertions(+), 38 deletions(-) diff --git a/system-test/client-side-metrics-all-methods.ts b/system-test/client-side-metrics-all-methods.ts index e7556bf39..63ed95d5c 100644 --- a/system-test/client-side-metrics-all-methods.ts +++ b/system-test/client-side-metrics-all-methods.ts @@ -390,10 +390,12 @@ describe('Bigtable/ClientSideMetrics', () => { try { const bigtable = await mockBigtable(defaultProjectId, done); for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ - tableId1, - tableId2, - ]); + await setupBigtableWithInsert( + bigtable, + columnFamilyId, + instanceId, + [tableId1, tableId2], + ); const instance = bigtable.instance(instanceId); const table = instance.table(tableId1); await table.getRows(); @@ -441,10 +443,12 @@ describe('Bigtable/ClientSideMetrics', () => { const projectId = SECOND_PROJECT_ID; const bigtable = await mockBigtable(projectId, done); for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ - tableId1, - tableId2, - ]); + await setupBigtableWithInsert( + bigtable, + columnFamilyId, + instanceId, + [tableId1, tableId2], + ); const instance = bigtable.instance(instanceId); const table = instance.table(tableId1); await table.getRows(); @@ -466,10 +470,12 @@ describe('Bigtable/ClientSideMetrics', () => { try { const bigtable = await mockBigtable(defaultProjectId, done); for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ - tableId1, - tableId2, - ]); + await setupBigtableWithInsert( + bigtable, + columnFamilyId, + instanceId, + [tableId1, tableId2], + ); const instance = bigtable.instance(instanceId); const table = instance.table(tableId1); await table.mutate(mutation); @@ -517,10 +523,12 @@ describe('Bigtable/ClientSideMetrics', () => { const projectId = SECOND_PROJECT_ID; const bigtable = await mockBigtable(projectId, done); for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ - tableId1, - tableId2, - ]); + await setupBigtableWithInsert( + bigtable, + columnFamilyId, + instanceId, + [tableId1, tableId2], + ); const instance = bigtable.instance(instanceId); const table = instance.table(tableId1); await table.mutate(mutation); @@ -610,10 +618,12 @@ describe('Bigtable/ClientSideMetrics', () => { const bigtable2 = await mockBigtable(defaultProjectId, done); for (const bigtable of [bigtable1, bigtable2]) { for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ - tableId1, - tableId2, - ]); + await setupBigtableWithInsert( + bigtable, + columnFamilyId, + instanceId, + [tableId1, tableId2], + ); const instance = bigtable.instance(instanceId); const table = instance.table(tableId1); await table.getRows(); @@ -668,10 +678,12 @@ describe('Bigtable/ClientSideMetrics', () => { } for (const bigtable of bigtableList) { for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ - tableId1, - tableId2, - ]); + await setupBigtableWithInsert( + bigtable, + columnFamilyId, + instanceId, + [tableId1, tableId2], + ); const instance = bigtable.instance(instanceId); const table = instance.table(tableId1); await table.getRows(); @@ -707,10 +719,12 @@ describe('Bigtable/ClientSideMetrics', () => { const bigtable2 = await mockBigtable(defaultProjectId, done); for (const bigtable of [bigtable1, bigtable2]) { for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ - tableId1, - tableId2, - ]); + await setupBigtableWithInsert( + bigtable, + columnFamilyId, + instanceId, + [tableId1, tableId2], + ); const instance = bigtable.instance(instanceId); const table = instance.table(tableId1); await table.mutate(mutation); @@ -765,10 +779,12 @@ describe('Bigtable/ClientSideMetrics', () => { } for (const bigtable of bigtableList) { for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ - tableId1, - tableId2, - ]); + await setupBigtableWithInsert( + bigtable, + columnFamilyId, + instanceId, + [tableId1, tableId2], + ); const instance = bigtable.instance(instanceId); const table = instance.table(tableId1); await table.mutate(mutation); diff --git a/system-test/client-side-metrics-setup-table.ts b/system-test/client-side-metrics-setup-table.ts index 4f56d9c5b..4bac03e49 100644 --- a/system-test/client-side-metrics-setup-table.ts +++ b/system-test/client-side-metrics-setup-table.ts @@ -65,12 +65,7 @@ export async function setupBigtableWithInsert( instanceId: string, tableIds: string[], ) { - await setupBigtable( - bigtable, - columnFamilyId, - instanceId, - tableIds, - ); + await setupBigtable(bigtable, columnFamilyId, instanceId, tableIds); const instance = bigtable.instance(instanceId); const tables = tableIds.map(tableId => instance.table(tableId)); for (const currentTable of tables) { From 15942392cc477706e2d1bedc40d6b54cfe9a4f46 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Mon, 28 Jul 2025 11:27:22 -0400 Subject: [PATCH 30/31] =?UTF-8?q?Make=20sure=20the=20table=20is=20set=20up?= =?UTF-8?q?=20properly,=20but=20don=E2=80=99t?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit add a mutateRow call for the handlers tests --- system-test/client-side-metrics.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/system-test/client-side-metrics.ts b/system-test/client-side-metrics.ts index 3d6f9d450..1f622f758 100644 --- a/system-test/client-side-metrics.ts +++ b/system-test/client-side-metrics.ts @@ -24,7 +24,10 @@ import {GCPMetricsHandler} from '../src/client-side-metrics/gcp-metrics-handler' import * as proxyquire from 'proxyquire'; import {Bigtable} from '../src'; import {Row} from '../src/row'; -import {setupBigtable} from './client-side-metrics-setup-table'; +import { + setupBigtable, + setupBigtableWithInsert, +} from './client-side-metrics-setup-table'; import {TestMetricsHandler} from '../test-common/test-metrics-handler'; import { OnAttemptCompleteData, @@ -247,7 +250,7 @@ describe('Bigtable/ClientSideMetrics', () => { before(async () => { const bigtable = new Bigtable(); for (const instanceId of [instanceId1, instanceId2]) { - await setupBigtable(bigtable, columnFamilyId, instanceId, [ + await setupBigtableWithInsert(bigtable, columnFamilyId, instanceId, [ tableId1, tableId2, ]); @@ -586,7 +589,7 @@ describe('Bigtable/ClientSideMetrics', () => { }); }); }); - describe('Bigtable/ClientSideMetricsToMetricsHandler', () => { + describe.only('Bigtable/ClientSideMetricsToMetricsHandler', () => { async function mockBigtable( projectId: string, done: mocha.Done, From eb3fefe66c541a06c7987d0f998d39ffdc058c1f Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Mon, 28 Jul 2025 11:28:10 -0400 Subject: [PATCH 31/31] Remove only --- system-test/client-side-metrics.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system-test/client-side-metrics.ts b/system-test/client-side-metrics.ts index 1f622f758..e0a132a71 100644 --- a/system-test/client-side-metrics.ts +++ b/system-test/client-side-metrics.ts @@ -589,7 +589,7 @@ describe('Bigtable/ClientSideMetrics', () => { }); }); }); - describe.only('Bigtable/ClientSideMetricsToMetricsHandler', () => { + describe('Bigtable/ClientSideMetricsToMetricsHandler', () => { async function mockBigtable( projectId: string, done: mocha.Done,