From 0bed27f098d44da8acdbab6dc7804687c8d15d04 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Wed, 16 Sep 2026 14:02:42 +0000 Subject: [PATCH 1/6] test(bigtable): add integration tests for internal metrics and single-row reads - BuiltinMetricsIT: add testInternalMetrics to verify per_connection_error_count is exported to Cloud Monitoring; remove gRPC DirectPath metric tests that were not universally applicable - ReadIT: add three tests (readSingleRowWithReadRow, readSingleRowWithRowKeyQuery, readSingleRowWithRowRangeQuery) covering readRow API, rowKey query, and closed/closed row range query, each asserting the exact cell content returned --- .../bigtable/data/v2/it/BuiltinMetricsIT.java | 84 +++++++++++++++-- .../cloud/bigtable/data/v2/it/ReadIT.java | 93 +++++++++++++++++++ 2 files changed, 171 insertions(+), 6 deletions(-) diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java index cb944a2aec7a..452ec1879406 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java @@ -24,6 +24,7 @@ import static com.google.common.truth.TruthJUnit.assume; import com.google.api.client.util.Lists; +import com.google.api.gax.rpc.NotFoundException; import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; import com.google.cloud.bigtable.admin.v2.models.Table; @@ -41,6 +42,7 @@ import com.google.cloud.monitoring.v3.MetricServiceClient; import com.google.common.base.Stopwatch; import com.google.common.collect.BoundType; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Range; import com.google.monitoring.v3.ListTimeSeriesRequest; @@ -72,14 +74,12 @@ import org.junit.After; import org.junit.Before; import org.junit.ClassRule; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.Timeout; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -@Ignore("Temporarily disable flaky test") @RunWith(JUnit4.class) public class BuiltinMetricsIT { @ClassRule public static TestEnvRule testEnvRule = new TestEnvRule(); @@ -97,6 +97,12 @@ public class BuiltinMetricsIT { private InMemoryMetricReader metricReader; + // OTel instrument names for all Bigtable client metrics (public and internal) carry the internal + // namespace prefix; the public/internal split only happens at export time via each metric's + // external name. The custom OTEL InMemoryMetricReader therefore sees the fully qualified names. + private static final String INTERNAL_INSTRUMENT_PREFIX = + "bigtable.googleapis.com/internal/client/"; + public static String[] VIEWS = { "operation_latencies", "attempt_latencies", @@ -160,7 +166,6 @@ public void tearDown() { if (tableDefault != null) { tableAdminClient.deleteTable(tableDefault.getId()); } - if (clientCustomOtel != null) { clientCustomOtel.close(); } @@ -233,6 +238,58 @@ public void testBuiltinMetricsWithDefaultOTEL() throws Exception { } } + @Test + public void testInternalMetrics() throws Exception { + logger.info("Started testing internal metrics"); + tableDefault = + tableAdminClient.createTable( + CreateTableRequest.of(PrefixGenerator.newPrefix("BuiltinMetricsIT#testInternal")) + .addFamily("cf")); + logger.info("Create default table: " + tableDefault.getId()); + + Instant start = Instant.now().minus(Duration.ofSeconds(10)); + + // Send a MutateRow and ReadRows request and measure the latencies for these requests. + clientDefault.mutateRow( + RowMutation.create(TableId.of(tableDefault.getId()), "a-new-key") + .setCell("cf", "q", "abc")); + ArrayList ignored = + Lists.newArrayList( + clientDefault.readRows(Query.create(TableId.of(tableDefault.getId())).limit(10))); + + // This stopwatch is used for to limit fetching of metric data in verifyMetrics + Stopwatch metricsPollingStopwatch = Stopwatch.createStarted(); + + ProjectName name = ProjectName.of(testEnvRule.env().getProjectId()); + + // Interval is set in the monarch request when query metric timestamps. + // Restrict it to before we send to request and 3 minute after we send the request. If + // it turns out to be still flaky we can increase the filter range. + Instant end = Instant.now().plus(Duration.ofMinutes(3)); + TimeInterval interval = + TimeInterval.newBuilder() + .setStartTime(Timestamps.fromMillis(start.toEpochMilli())) + .setEndTime(Timestamps.fromMillis(end.toEpochMilli())) + .build(); + + List views = ImmutableList.of("per_connection_error_count"); + for (String view : views) { + // Filter on instance name + String metricFilter = + String.format( + "metric.type=\"bigtable.googleapis.com/internal/client/%s\" AND" + + " resource.labels.instance=\"%s\"", + view, testEnvRule.env().getInstanceId()); + ListTimeSeriesRequest.Builder requestBuilder = + ListTimeSeriesRequest.newBuilder() + .setName(name.toString()) + .setFilter(metricFilter) + .setInterval(interval) + .setView(ListTimeSeriesRequest.TimeSeriesView.FULL); + verifyMetricsArePublished(requestBuilder.build(), metricsPollingStopwatch, view); + } + } + @Test public void testBuiltinMetricsWithCustomOTEL() throws Exception { logger.info("Started testing builtin metrics with custom OTEL"); @@ -271,7 +328,10 @@ public void testBuiltinMetricsWithCustomOTEL() throws Exception { if (view.equals("application_blocking_latencies")) { otelMetricName = "application_latencies"; } - MetricData dataFromReader = getMetricData(metricReader, otelMetricName); + // The InMemoryMetricReader records instruments under their fully qualified OTel name, so look + // up the metric by the internal-namespace-prefixed name. + MetricData dataFromReader = + getMetricData(metricReader, INTERNAL_INSTRUMENT_PREFIX + otelMetricName); // Filter on instance and method name // Verify that metrics are correct for MutateRows request @@ -309,7 +369,7 @@ public void testBuiltinMetricsWithCustomOTEL() throws Exception { private ListTimeSeriesResponse verifyMetricsArePublished( ListTimeSeriesRequest request, Stopwatch metricsPollingStopwatch, String view) throws Exception { - ListTimeSeriesResponse response = metricClient.listTimeSeriesCallable().call(request); + ListTimeSeriesResponse response = listTimeSeriesToleratingNotFound(request); while (response.getTimeSeriesCount() == 0 && metricsPollingStopwatch.elapsed(TimeUnit.MINUTES) < 10) { logger.log( @@ -322,7 +382,7 @@ private ListTimeSeriesResponse verifyMetricsArePublished( + metricsPollingStopwatch.elapsed(TimeUnit.MINUTES)); // Call listTimeSeries every minute Thread.sleep(Duration.ofMinutes(1).toMillis()); - response = metricClient.listTimeSeriesCallable().call(request); + response = listTimeSeriesToleratingNotFound(request); } assertWithMessage("View " + view + " didn't return any data.") @@ -332,6 +392,18 @@ private ListTimeSeriesResponse verifyMetricsArePublished( return response; } + // A metric's descriptor is created lazily the first time that metric is exported, so a brand new + // internal metric can return NOT_FOUND for several minutes before it becomes queryable (Cloud + // Monitoring reports it may take up to 10 minutes). Treat NOT_FOUND as "no data yet" so the + // polling loop keeps retrying within its budget instead of failing immediately. + private ListTimeSeriesResponse listTimeSeriesToleratingNotFound(ListTimeSeriesRequest request) { + try { + return metricClient.listTimeSeriesCallable().call(request); + } catch (NotFoundException e) { + return ListTimeSeriesResponse.getDefaultInstance(); + } + } + private void verifyMetricsWithMetricsReader( ListTimeSeriesResponse response, MetricData dataFromReader) { diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java index e5af176e2172..48576bdcc247 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java @@ -823,6 +823,99 @@ public void onSuccess(Row result) { assertThat(found.get()).isTrue(); } + @Test + public void readSingleRowWithReadRow() throws Exception { + String rowKey = prefix + "-readSingleRowWithReadRow"; + TableId tableId = testEnvRule.env().getTableId(); + String familyId = testEnvRule.env().getFamilyId(); + long timestampMicros = System.currentTimeMillis() * 1_000; + + testEnvRule + .env() + .getDataClient() + .mutateRow( + RowMutation.create(tableId, rowKey).setCell(familyId, "q", timestampMicros, "value")); + + Row expectedRow = + Row.create( + ByteString.copyFromUtf8(rowKey), + ImmutableList.of( + RowCell.create( + familyId, + ByteString.copyFromUtf8("q"), + timestampMicros, + ImmutableList.of(), + ByteString.copyFromUtf8("value")))); + + Row row = testEnvRule.env().getDataClient().readRow(tableId, rowKey); + assertThat(row).isEqualTo(expectedRow); + } + + @Test + public void readSingleRowWithRowKeyQuery() throws Exception { + String rowKey = prefix + "-readSingleRowWithRowKeyQuery"; + TableId tableId = testEnvRule.env().getTableId(); + String familyId = testEnvRule.env().getFamilyId(); + long timestampMicros = System.currentTimeMillis() * 1_000; + + testEnvRule + .env() + .getDataClient() + .mutateRow( + RowMutation.create(tableId, rowKey).setCell(familyId, "q", timestampMicros, "value")); + + Row expectedRow = + Row.create( + ByteString.copyFromUtf8(rowKey), + ImmutableList.of( + RowCell.create( + familyId, + ByteString.copyFromUtf8("q"), + timestampMicros, + ImmutableList.of(), + ByteString.copyFromUtf8("value")))); + + List rows = + Lists.newArrayList( + testEnvRule.env().getDataClient().readRows(Query.create(tableId).rowKey(rowKey))); + assertThat(rows).containsExactly(expectedRow); + } + + @Test + public void readSingleRowWithRowRangeQuery() throws Exception { + String rowKey = prefix + "-readSingleRowWithRowRangeQuery"; + TableId tableId = testEnvRule.env().getTableId(); + String familyId = testEnvRule.env().getFamilyId(); + long timestampMicros = System.currentTimeMillis() * 1_000; + + testEnvRule + .env() + .getDataClient() + .mutateRow( + RowMutation.create(tableId, rowKey).setCell(familyId, "q", timestampMicros, "value")); + + Row expectedRow = + Row.create( + ByteString.copyFromUtf8(rowKey), + ImmutableList.of( + RowCell.create( + familyId, + ByteString.copyFromUtf8("q"), + timestampMicros, + ImmutableList.of(), + ByteString.copyFromUtf8("value")))); + + List rows = + Lists.newArrayList( + testEnvRule + .env() + .getDataClient() + .readRows( + Query.create(tableId) + .range(ByteStringRange.unbounded().startClosed(rowKey).endClosed(rowKey)))); + assertThat(rows).containsExactly(expectedRow); + } + static class AccumulatingObserver implements ResponseObserver { final List responses = Lists.newArrayList(); From ecef11a195f2361ee67f026df2a7219fc0cefef0 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Wed, 16 Sep 2026 14:15:28 +0000 Subject: [PATCH 2/6] address review comments - BuiltinMetricsIT: remove table creation in testInternalMetrics (use shared test table; per_connection_error_count is connection-level); extend query interval to 10 min to match polling budget; revert INTERNAL_INSTRUMENT_PREFIX usage in testBuiltinMetricsWithCustomOTEL - ReadIT: extract repeated write+expected-row setup into writeTestRowAndBuildExpected helper --- .../bigtable/data/v2/it/BuiltinMetricsIT.java | 33 ++------ .../cloud/bigtable/data/v2/it/ReadIT.java | 79 ++++++------------- 2 files changed, 30 insertions(+), 82 deletions(-) diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java index 452ec1879406..66be336b4929 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java @@ -97,12 +97,6 @@ public class BuiltinMetricsIT { private InMemoryMetricReader metricReader; - // OTel instrument names for all Bigtable client metrics (public and internal) carry the internal - // namespace prefix; the public/internal split only happens at export time via each metric's - // external name. The custom OTEL InMemoryMetricReader therefore sees the fully qualified names. - private static final String INTERNAL_INSTRUMENT_PREFIX = - "bigtable.googleapis.com/internal/client/"; - public static String[] VIEWS = { "operation_latencies", "attempt_latencies", @@ -241,31 +235,23 @@ public void testBuiltinMetricsWithDefaultOTEL() throws Exception { @Test public void testInternalMetrics() throws Exception { logger.info("Started testing internal metrics"); - tableDefault = - tableAdminClient.createTable( - CreateTableRequest.of(PrefixGenerator.newPrefix("BuiltinMetricsIT#testInternal")) - .addFamily("cf")); - logger.info("Create default table: " + tableDefault.getId()); + + TableId tableId = testEnvRule.env().getTableId(); + String familyId = testEnvRule.env().getFamilyId(); Instant start = Instant.now().minus(Duration.ofSeconds(10)); - // Send a MutateRow and ReadRows request and measure the latencies for these requests. - clientDefault.mutateRow( - RowMutation.create(TableId.of(tableDefault.getId()), "a-new-key") - .setCell("cf", "q", "abc")); + // Send a MutateRow and ReadRows request to generate connection activity. + clientDefault.mutateRow(RowMutation.create(tableId, "a-new-key").setCell(familyId, "q", "abc")); ArrayList ignored = - Lists.newArrayList( - clientDefault.readRows(Query.create(TableId.of(tableDefault.getId())).limit(10))); + Lists.newArrayList(clientDefault.readRows(Query.create(tableId).limit(10))); // This stopwatch is used for to limit fetching of metric data in verifyMetrics Stopwatch metricsPollingStopwatch = Stopwatch.createStarted(); ProjectName name = ProjectName.of(testEnvRule.env().getProjectId()); - // Interval is set in the monarch request when query metric timestamps. - // Restrict it to before we send to request and 3 minute after we send the request. If - // it turns out to be still flaky we can increase the filter range. - Instant end = Instant.now().plus(Duration.ofMinutes(3)); + Instant end = Instant.now().plus(Duration.ofMinutes(10)); TimeInterval interval = TimeInterval.newBuilder() .setStartTime(Timestamps.fromMillis(start.toEpochMilli())) @@ -328,10 +314,7 @@ public void testBuiltinMetricsWithCustomOTEL() throws Exception { if (view.equals("application_blocking_latencies")) { otelMetricName = "application_latencies"; } - // The InMemoryMetricReader records instruments under their fully qualified OTel name, so look - // up the metric by the internal-namespace-prefixed name. - MetricData dataFromReader = - getMetricData(metricReader, INTERNAL_INSTRUMENT_PREFIX + otelMetricName); + MetricData dataFromReader = getMetricData(metricReader, otelMetricName); // Filter on instance and method name // Verify that metrics are correct for MutateRows request diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java index 48576bdcc247..67b192e9785a 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java @@ -827,25 +827,7 @@ public void onSuccess(Row result) { public void readSingleRowWithReadRow() throws Exception { String rowKey = prefix + "-readSingleRowWithReadRow"; TableId tableId = testEnvRule.env().getTableId(); - String familyId = testEnvRule.env().getFamilyId(); - long timestampMicros = System.currentTimeMillis() * 1_000; - - testEnvRule - .env() - .getDataClient() - .mutateRow( - RowMutation.create(tableId, rowKey).setCell(familyId, "q", timestampMicros, "value")); - - Row expectedRow = - Row.create( - ByteString.copyFromUtf8(rowKey), - ImmutableList.of( - RowCell.create( - familyId, - ByteString.copyFromUtf8("q"), - timestampMicros, - ImmutableList.of(), - ByteString.copyFromUtf8("value")))); + Row expectedRow = writeTestRowAndBuildExpected(tableId, rowKey); Row row = testEnvRule.env().getDataClient().readRow(tableId, rowKey); assertThat(row).isEqualTo(expectedRow); @@ -855,25 +837,7 @@ public void readSingleRowWithReadRow() throws Exception { public void readSingleRowWithRowKeyQuery() throws Exception { String rowKey = prefix + "-readSingleRowWithRowKeyQuery"; TableId tableId = testEnvRule.env().getTableId(); - String familyId = testEnvRule.env().getFamilyId(); - long timestampMicros = System.currentTimeMillis() * 1_000; - - testEnvRule - .env() - .getDataClient() - .mutateRow( - RowMutation.create(tableId, rowKey).setCell(familyId, "q", timestampMicros, "value")); - - Row expectedRow = - Row.create( - ByteString.copyFromUtf8(rowKey), - ImmutableList.of( - RowCell.create( - familyId, - ByteString.copyFromUtf8("q"), - timestampMicros, - ImmutableList.of(), - ByteString.copyFromUtf8("value")))); + Row expectedRow = writeTestRowAndBuildExpected(tableId, rowKey); List rows = Lists.newArrayList( @@ -885,25 +849,7 @@ public void readSingleRowWithRowKeyQuery() throws Exception { public void readSingleRowWithRowRangeQuery() throws Exception { String rowKey = prefix + "-readSingleRowWithRowRangeQuery"; TableId tableId = testEnvRule.env().getTableId(); - String familyId = testEnvRule.env().getFamilyId(); - long timestampMicros = System.currentTimeMillis() * 1_000; - - testEnvRule - .env() - .getDataClient() - .mutateRow( - RowMutation.create(tableId, rowKey).setCell(familyId, "q", timestampMicros, "value")); - - Row expectedRow = - Row.create( - ByteString.copyFromUtf8(rowKey), - ImmutableList.of( - RowCell.create( - familyId, - ByteString.copyFromUtf8("q"), - timestampMicros, - ImmutableList.of(), - ByteString.copyFromUtf8("value")))); + Row expectedRow = writeTestRowAndBuildExpected(tableId, rowKey); List rows = Lists.newArrayList( @@ -916,6 +862,25 @@ public void readSingleRowWithRowRangeQuery() throws Exception { assertThat(rows).containsExactly(expectedRow); } + private Row writeTestRowAndBuildExpected(TableId tableId, String rowKey) { + String familyId = testEnvRule.env().getFamilyId(); + long timestampMicros = System.currentTimeMillis() * 1_000; + testEnvRule + .env() + .getDataClient() + .mutateRow( + RowMutation.create(tableId, rowKey).setCell(familyId, "q", timestampMicros, "value")); + return Row.create( + ByteString.copyFromUtf8(rowKey), + ImmutableList.of( + RowCell.create( + familyId, + ByteString.copyFromUtf8("q"), + timestampMicros, + ImmutableList.of(), + ByteString.copyFromUtf8("value")))); + } + static class AccumulatingObserver implements ResponseObserver { final List responses = Lists.newArrayList(); From e842df21b38901294f7f13eac46a6f494bbe3023 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Wed, 16 Sep 2026 14:23:16 +0000 Subject: [PATCH 3/6] address review: create unique table in testInternalMetrics and clean up Create a dedicated table with a unique prefix so metrics from this run are isolated by time window from prior runs, and delete the table in a finally block to guarantee cleanup. --- .../bigtable/data/v2/it/BuiltinMetricsIT.java | 72 ++++++++++--------- 1 file changed, 40 insertions(+), 32 deletions(-) diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java index 66be336b4929..1d3a88361010 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java @@ -235,44 +235,52 @@ public void testBuiltinMetricsWithDefaultOTEL() throws Exception { @Test public void testInternalMetrics() throws Exception { logger.info("Started testing internal metrics"); + Table table = + tableAdminClient.createTable( + CreateTableRequest.of(PrefixGenerator.newPrefix("BuiltinMetricsIT#testInternal")) + .addFamily("cf")); + logger.info("Create table: " + table.getId()); - TableId tableId = testEnvRule.env().getTableId(); - String familyId = testEnvRule.env().getFamilyId(); + try { + Instant start = Instant.now().minus(Duration.ofSeconds(10)); - Instant start = Instant.now().minus(Duration.ofSeconds(10)); + // Send a MutateRow and ReadRows request to generate connection activity. + clientDefault.mutateRow( + RowMutation.create(TableId.of(table.getId()), "a-new-key").setCell("cf", "q", "abc")); + ArrayList ignored = + Lists.newArrayList( + clientDefault.readRows(Query.create(TableId.of(table.getId())).limit(10))); - // Send a MutateRow and ReadRows request to generate connection activity. - clientDefault.mutateRow(RowMutation.create(tableId, "a-new-key").setCell(familyId, "q", "abc")); - ArrayList ignored = - Lists.newArrayList(clientDefault.readRows(Query.create(tableId).limit(10))); + // This stopwatch is used for to limit fetching of metric data in verifyMetrics + Stopwatch metricsPollingStopwatch = Stopwatch.createStarted(); - // This stopwatch is used for to limit fetching of metric data in verifyMetrics - Stopwatch metricsPollingStopwatch = Stopwatch.createStarted(); + ProjectName name = ProjectName.of(testEnvRule.env().getProjectId()); - ProjectName name = ProjectName.of(testEnvRule.env().getProjectId()); - - Instant end = Instant.now().plus(Duration.ofMinutes(10)); - TimeInterval interval = - TimeInterval.newBuilder() - .setStartTime(Timestamps.fromMillis(start.toEpochMilli())) - .setEndTime(Timestamps.fromMillis(end.toEpochMilli())) - .build(); + Instant end = Instant.now().plus(Duration.ofMinutes(10)); + TimeInterval interval = + TimeInterval.newBuilder() + .setStartTime(Timestamps.fromMillis(start.toEpochMilli())) + .setEndTime(Timestamps.fromMillis(end.toEpochMilli())) + .build(); - List views = ImmutableList.of("per_connection_error_count"); - for (String view : views) { - // Filter on instance name - String metricFilter = - String.format( - "metric.type=\"bigtable.googleapis.com/internal/client/%s\" AND" - + " resource.labels.instance=\"%s\"", - view, testEnvRule.env().getInstanceId()); - ListTimeSeriesRequest.Builder requestBuilder = - ListTimeSeriesRequest.newBuilder() - .setName(name.toString()) - .setFilter(metricFilter) - .setInterval(interval) - .setView(ListTimeSeriesRequest.TimeSeriesView.FULL); - verifyMetricsArePublished(requestBuilder.build(), metricsPollingStopwatch, view); + List views = ImmutableList.of("per_connection_error_count"); + for (String view : views) { + // Filter on instance name + String metricFilter = + String.format( + "metric.type=\"bigtable.googleapis.com/internal/client/%s\" AND" + + " resource.labels.instance=\"%s\"", + view, testEnvRule.env().getInstanceId()); + ListTimeSeriesRequest.Builder requestBuilder = + ListTimeSeriesRequest.newBuilder() + .setName(name.toString()) + .setFilter(metricFilter) + .setInterval(interval) + .setView(ListTimeSeriesRequest.TimeSeriesView.FULL); + verifyMetricsArePublished(requestBuilder.build(), metricsPollingStopwatch, view); + } + } finally { + tableAdminClient.deleteTable(table.getId()); } } From f659f5b5ec1a6bac39fee1fcd114cfa134b1bdd1 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Wed, 16 Sep 2026 14:29:01 +0000 Subject: [PATCH 4/6] address review: create a fresh Bigtable instance in testInternalMetrics Create a dedicated Bigtable instance with a unique name, run operations against it, filter the Cloud Monitoring query on the unique instance ID, and delete the instance in a finally block. This ensures metrics queried belong strictly to this test run and not prior runs on the shared instance. --- .../bigtable/data/v2/it/BuiltinMetricsIT.java | 112 +++++++++++------- 1 file changed, 71 insertions(+), 41 deletions(-) diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java index 1d3a88361010..dfa54c1aa9b6 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java @@ -25,7 +25,11 @@ import com.google.api.client.util.Lists; import com.google.api.gax.rpc.NotFoundException; +import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminClient; import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; +import com.google.cloud.bigtable.admin.v2.models.Cluster; +import com.google.cloud.bigtable.admin.v2.models.CreateInstanceRequest; import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; import com.google.cloud.bigtable.admin.v2.models.Table; import com.google.cloud.bigtable.data.v2.BigtableDataClient; @@ -235,52 +239,78 @@ public void testBuiltinMetricsWithDefaultOTEL() throws Exception { @Test public void testInternalMetrics() throws Exception { logger.info("Started testing internal metrics"); - Table table = - tableAdminClient.createTable( - CreateTableRequest.of(PrefixGenerator.newPrefix("BuiltinMetricsIT#testInternal")) - .addFamily("cf")); - logger.info("Create table: " + table.getId()); - - try { - Instant start = Instant.now().minus(Duration.ofSeconds(10)); - - // Send a MutateRow and ReadRows request to generate connection activity. - clientDefault.mutateRow( - RowMutation.create(TableId.of(table.getId()), "a-new-key").setCell("cf", "q", "abc")); - ArrayList ignored = - Lists.newArrayList( - clientDefault.readRows(Query.create(TableId.of(table.getId())).limit(10))); - // This stopwatch is used for to limit fetching of metric data in verifyMetrics - Stopwatch metricsPollingStopwatch = Stopwatch.createStarted(); + // Discover the existing instance's cluster zone so we can create a sibling instance. + BigtableInstanceAdminClient instanceAdmin = testEnvRule.env().getInstanceAdminClient(); + List clusters = instanceAdmin.listClusters(testEnvRule.env().getInstanceId()); + Cluster existingCluster = clusters.get(0); + + String testInstanceId = PrefixGenerator.newPrefix("bt-metrics-it"); + String testClusterId = testInstanceId + "-c1"; + instanceAdmin.createInstance( + CreateInstanceRequest.of(testInstanceId) + .addCluster( + testClusterId, + existingCluster.getZone(), + existingCluster.getServeNodes(), + existingCluster.getStorageType())); + logger.info("Created test instance: " + testInstanceId); - ProjectName name = ProjectName.of(testEnvRule.env().getProjectId()); - - Instant end = Instant.now().plus(Duration.ofMinutes(10)); - TimeInterval interval = - TimeInterval.newBuilder() - .setStartTime(Timestamps.fromMillis(start.toEpochMilli())) - .setEndTime(Timestamps.fromMillis(end.toEpochMilli())) + try { + // Create a table and a data client scoped to the new instance. + BigtableTableAdminClient testTableAdmin = + BigtableTableAdminClient.create( + BigtableTableAdminSettings.newBuilder() + .setProjectId(testEnvRule.env().getProjectId()) + .setInstanceId(testInstanceId) + .build()); + testTableAdmin.createTable(CreateTableRequest.of("test-table").addFamily("cf")); + + BigtableDataSettings testDataSettings = + testEnvRule.env().getDataClientSettings().toBuilder() + .setInstanceId(testInstanceId) .build(); - - List views = ImmutableList.of("per_connection_error_count"); - for (String view : views) { - // Filter on instance name - String metricFilter = - String.format( - "metric.type=\"bigtable.googleapis.com/internal/client/%s\" AND" - + " resource.labels.instance=\"%s\"", - view, testEnvRule.env().getInstanceId()); - ListTimeSeriesRequest.Builder requestBuilder = - ListTimeSeriesRequest.newBuilder() - .setName(name.toString()) - .setFilter(metricFilter) - .setInterval(interval) - .setView(ListTimeSeriesRequest.TimeSeriesView.FULL); - verifyMetricsArePublished(requestBuilder.build(), metricsPollingStopwatch, view); + try (BigtableDataClient testClient = BigtableDataClient.create(testDataSettings)) { + Instant start = Instant.now().minus(Duration.ofSeconds(10)); + + // Send a MutateRow and ReadRows request to generate connection activity. + testClient.mutateRow( + RowMutation.create(TableId.of("test-table"), "a-new-key").setCell("cf", "q", "abc")); + ArrayList ignored = + Lists.newArrayList( + testClient.readRows(Query.create(TableId.of("test-table")).limit(10))); + + // This stopwatch is used for to limit fetching of metric data in verifyMetrics + Stopwatch metricsPollingStopwatch = Stopwatch.createStarted(); + + ProjectName name = ProjectName.of(testEnvRule.env().getProjectId()); + + Instant end = Instant.now().plus(Duration.ofMinutes(10)); + TimeInterval interval = + TimeInterval.newBuilder() + .setStartTime(Timestamps.fromMillis(start.toEpochMilli())) + .setEndTime(Timestamps.fromMillis(end.toEpochMilli())) + .build(); + + List views = ImmutableList.of("per_connection_error_count"); + for (String view : views) { + String metricFilter = + String.format( + "metric.type=\"bigtable.googleapis.com/internal/client/%s\" AND" + + " resource.labels.instance=\"%s\"", + view, testInstanceId); + ListTimeSeriesRequest.Builder requestBuilder = + ListTimeSeriesRequest.newBuilder() + .setName(name.toString()) + .setFilter(metricFilter) + .setInterval(interval) + .setView(ListTimeSeriesRequest.TimeSeriesView.FULL); + verifyMetricsArePublished(requestBuilder.build(), metricsPollingStopwatch, view); + } } } finally { - tableAdminClient.deleteTable(table.getId()); + instanceAdmin.deleteInstance(testInstanceId); + logger.info("Deleted test instance: " + testInstanceId); } } From 2bce4abec6f70dc9115286bfcbf3e4a8362178e7 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Wed, 16 Sep 2026 15:32:50 +0000 Subject: [PATCH 5/6] restore INTERNAL_INSTRUMENT_PREFIX in testBuiltinMetricsWithCustomOTEL The OTel instruments are registered under the full bigtable.googleapis.com/internal/client/ name, so the InMemoryMetricReader stores them under that full name. The getMetricData lookup must use the prefix to find them. --- .../cloud/bigtable/data/v2/it/BuiltinMetricsIT.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java index dfa54c1aa9b6..40621275e88f 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java @@ -101,6 +101,12 @@ public class BuiltinMetricsIT { private InMemoryMetricReader metricReader; + // OTel instruments are always registered under the internal/client/ namespace. The client/ + // vs internal/client/ split only happens at Cloud Monitoring export time, so the + // InMemoryMetricReader sees the full internal name. + private static final String INTERNAL_INSTRUMENT_PREFIX = + "bigtable.googleapis.com/internal/client/"; + public static String[] VIEWS = { "operation_latencies", "attempt_latencies", @@ -352,7 +358,8 @@ public void testBuiltinMetricsWithCustomOTEL() throws Exception { if (view.equals("application_blocking_latencies")) { otelMetricName = "application_latencies"; } - MetricData dataFromReader = getMetricData(metricReader, otelMetricName); + MetricData dataFromReader = + getMetricData(metricReader, INTERNAL_INSTRUMENT_PREFIX + otelMetricName); // Filter on instance and method name // Verify that metrics are correct for MutateRows request From a68f7ea5b4cd78566e26fb7274bdbf163adb3520 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Wed, 16 Sep 2026 19:46:26 +0000 Subject: [PATCH 6/6] revert: remove BuiltinMetricsIT changes from PR The PR should only contain single-row read tests in ReadIT.java. Restore BuiltinMetricsIT.java to its pre-PR state (with @Ignore). --- .../bigtable/data/v2/it/BuiltinMetricsIT.java | 112 +----------------- 1 file changed, 6 insertions(+), 106 deletions(-) diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java index 40621275e88f..cb944a2aec7a 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java @@ -24,12 +24,7 @@ import static com.google.common.truth.TruthJUnit.assume; import com.google.api.client.util.Lists; -import com.google.api.gax.rpc.NotFoundException; -import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminClient; import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; -import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; -import com.google.cloud.bigtable.admin.v2.models.Cluster; -import com.google.cloud.bigtable.admin.v2.models.CreateInstanceRequest; import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; import com.google.cloud.bigtable.admin.v2.models.Table; import com.google.cloud.bigtable.data.v2.BigtableDataClient; @@ -46,7 +41,6 @@ import com.google.cloud.monitoring.v3.MetricServiceClient; import com.google.common.base.Stopwatch; import com.google.common.collect.BoundType; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Range; import com.google.monitoring.v3.ListTimeSeriesRequest; @@ -78,12 +72,14 @@ import org.junit.After; import org.junit.Before; import org.junit.ClassRule; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.Timeout; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; +@Ignore("Temporarily disable flaky test") @RunWith(JUnit4.class) public class BuiltinMetricsIT { @ClassRule public static TestEnvRule testEnvRule = new TestEnvRule(); @@ -101,12 +97,6 @@ public class BuiltinMetricsIT { private InMemoryMetricReader metricReader; - // OTel instruments are always registered under the internal/client/ namespace. The client/ - // vs internal/client/ split only happens at Cloud Monitoring export time, so the - // InMemoryMetricReader sees the full internal name. - private static final String INTERNAL_INSTRUMENT_PREFIX = - "bigtable.googleapis.com/internal/client/"; - public static String[] VIEWS = { "operation_latencies", "attempt_latencies", @@ -170,6 +160,7 @@ public void tearDown() { if (tableDefault != null) { tableAdminClient.deleteTable(tableDefault.getId()); } + if (clientCustomOtel != null) { clientCustomOtel.close(); } @@ -242,84 +233,6 @@ public void testBuiltinMetricsWithDefaultOTEL() throws Exception { } } - @Test - public void testInternalMetrics() throws Exception { - logger.info("Started testing internal metrics"); - - // Discover the existing instance's cluster zone so we can create a sibling instance. - BigtableInstanceAdminClient instanceAdmin = testEnvRule.env().getInstanceAdminClient(); - List clusters = instanceAdmin.listClusters(testEnvRule.env().getInstanceId()); - Cluster existingCluster = clusters.get(0); - - String testInstanceId = PrefixGenerator.newPrefix("bt-metrics-it"); - String testClusterId = testInstanceId + "-c1"; - instanceAdmin.createInstance( - CreateInstanceRequest.of(testInstanceId) - .addCluster( - testClusterId, - existingCluster.getZone(), - existingCluster.getServeNodes(), - existingCluster.getStorageType())); - logger.info("Created test instance: " + testInstanceId); - - try { - // Create a table and a data client scoped to the new instance. - BigtableTableAdminClient testTableAdmin = - BigtableTableAdminClient.create( - BigtableTableAdminSettings.newBuilder() - .setProjectId(testEnvRule.env().getProjectId()) - .setInstanceId(testInstanceId) - .build()); - testTableAdmin.createTable(CreateTableRequest.of("test-table").addFamily("cf")); - - BigtableDataSettings testDataSettings = - testEnvRule.env().getDataClientSettings().toBuilder() - .setInstanceId(testInstanceId) - .build(); - try (BigtableDataClient testClient = BigtableDataClient.create(testDataSettings)) { - Instant start = Instant.now().minus(Duration.ofSeconds(10)); - - // Send a MutateRow and ReadRows request to generate connection activity. - testClient.mutateRow( - RowMutation.create(TableId.of("test-table"), "a-new-key").setCell("cf", "q", "abc")); - ArrayList ignored = - Lists.newArrayList( - testClient.readRows(Query.create(TableId.of("test-table")).limit(10))); - - // This stopwatch is used for to limit fetching of metric data in verifyMetrics - Stopwatch metricsPollingStopwatch = Stopwatch.createStarted(); - - ProjectName name = ProjectName.of(testEnvRule.env().getProjectId()); - - Instant end = Instant.now().plus(Duration.ofMinutes(10)); - TimeInterval interval = - TimeInterval.newBuilder() - .setStartTime(Timestamps.fromMillis(start.toEpochMilli())) - .setEndTime(Timestamps.fromMillis(end.toEpochMilli())) - .build(); - - List views = ImmutableList.of("per_connection_error_count"); - for (String view : views) { - String metricFilter = - String.format( - "metric.type=\"bigtable.googleapis.com/internal/client/%s\" AND" - + " resource.labels.instance=\"%s\"", - view, testInstanceId); - ListTimeSeriesRequest.Builder requestBuilder = - ListTimeSeriesRequest.newBuilder() - .setName(name.toString()) - .setFilter(metricFilter) - .setInterval(interval) - .setView(ListTimeSeriesRequest.TimeSeriesView.FULL); - verifyMetricsArePublished(requestBuilder.build(), metricsPollingStopwatch, view); - } - } - } finally { - instanceAdmin.deleteInstance(testInstanceId); - logger.info("Deleted test instance: " + testInstanceId); - } - } - @Test public void testBuiltinMetricsWithCustomOTEL() throws Exception { logger.info("Started testing builtin metrics with custom OTEL"); @@ -358,8 +271,7 @@ public void testBuiltinMetricsWithCustomOTEL() throws Exception { if (view.equals("application_blocking_latencies")) { otelMetricName = "application_latencies"; } - MetricData dataFromReader = - getMetricData(metricReader, INTERNAL_INSTRUMENT_PREFIX + otelMetricName); + MetricData dataFromReader = getMetricData(metricReader, otelMetricName); // Filter on instance and method name // Verify that metrics are correct for MutateRows request @@ -397,7 +309,7 @@ public void testBuiltinMetricsWithCustomOTEL() throws Exception { private ListTimeSeriesResponse verifyMetricsArePublished( ListTimeSeriesRequest request, Stopwatch metricsPollingStopwatch, String view) throws Exception { - ListTimeSeriesResponse response = listTimeSeriesToleratingNotFound(request); + ListTimeSeriesResponse response = metricClient.listTimeSeriesCallable().call(request); while (response.getTimeSeriesCount() == 0 && metricsPollingStopwatch.elapsed(TimeUnit.MINUTES) < 10) { logger.log( @@ -410,7 +322,7 @@ private ListTimeSeriesResponse verifyMetricsArePublished( + metricsPollingStopwatch.elapsed(TimeUnit.MINUTES)); // Call listTimeSeries every minute Thread.sleep(Duration.ofMinutes(1).toMillis()); - response = listTimeSeriesToleratingNotFound(request); + response = metricClient.listTimeSeriesCallable().call(request); } assertWithMessage("View " + view + " didn't return any data.") @@ -420,18 +332,6 @@ private ListTimeSeriesResponse verifyMetricsArePublished( return response; } - // A metric's descriptor is created lazily the first time that metric is exported, so a brand new - // internal metric can return NOT_FOUND for several minutes before it becomes queryable (Cloud - // Monitoring reports it may take up to 10 minutes). Treat NOT_FOUND as "no data yet" so the - // polling loop keeps retrying within its budget instead of failing immediately. - private ListTimeSeriesResponse listTimeSeriesToleratingNotFound(ListTimeSeriesRequest request) { - try { - return metricClient.listTimeSeriesCallable().call(request); - } catch (NotFoundException e) { - return ListTimeSeriesResponse.getDefaultInstance(); - } - } - private void verifyMetricsWithMetricsReader( ListTimeSeriesResponse response, MetricData dataFromReader) {