From 519dac927dacecb505c33a13d8116cfbbb0ab70a Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Tue, 10 May 2022 10:11:16 -0400 Subject: [PATCH 01/18] feat: add built in metrics measure and views --- .../stats/BuiltinMetricsRecorder.java | 170 +++++++ .../stats/BuiltinMetricsRecorderTest.java | 433 ++++++++++++++++++ 2 files changed, 603 insertions(+) create mode 100644 google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorder.java create mode 100644 google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorderTest.java diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorder.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorder.java new file mode 100644 index 000000000000..51a0feb5cbcd --- /dev/null +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorder.java @@ -0,0 +1,170 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.stats; + +import com.google.api.core.InternalApi; +import com.google.api.gax.tracing.ApiTracerFactory.OperationType; +import com.google.api.gax.tracing.SpanName; +import io.opencensus.stats.MeasureMap; +import io.opencensus.stats.StatsRecorder; +import io.opencensus.tags.TagContext; +import io.opencensus.tags.TagContextBuilder; +import io.opencensus.tags.TagKey; +import io.opencensus.tags.TagValue; +import io.opencensus.tags.Tagger; +import io.opencensus.tags.Tags; +import java.util.Map; + +/** Add built-in metrics to the measure map * */ +@InternalApi("For internal use only") +public class BuiltinMetricsRecorder { + + private final OperationType operationType; + + private final Tagger tagger; + private final StatsRecorder statsRecorder; + private final TagContext parentContext; + private final SpanName spanName; + private final Map statsAttributes; + + private MeasureMap attemptLevelNoStreaming; + private MeasureMap attemptLevelWithStreaming; + private MeasureMap operationLevelNoStreaming; + private MeasureMap operationLevelWithStreaming; + + public BuiltinMetricsRecorder( + OperationType operationType, + SpanName spanName, + Map statsAttributes, + StatsWrapper builtinMetricsWrapper) { + this.operationType = operationType; + this.tagger = Tags.getTagger(); + this.statsRecorder = builtinMetricsWrapper.getStatsRecorder(); + this.spanName = spanName; + this.parentContext = tagger.getCurrentTagContext(); + this.statsAttributes = statsAttributes; + + this.attemptLevelNoStreaming = statsRecorder.newMeasureMap(); + this.attemptLevelWithStreaming = statsRecorder.newMeasureMap(); + this.operationLevelNoStreaming = statsRecorder.newMeasureMap(); + this.operationLevelWithStreaming = statsRecorder.newMeasureMap(); + } + + public void recordAttemptLevelWithoutStreaming( + String status, String tableId, String zone, String cluster) { + TagContextBuilder tagCtx = + newTagContextBuilder(tableId, zone, cluster) + .putLocal(BuiltinMeasureConstants.STATUS, TagValue.create(status)); + + attemptLevelNoStreaming.record(tagCtx.build()); + } + + public void recordAttemptLevelWithStreaming( + String status, String tableId, String zone, String cluster) { + TagContextBuilder tagCtx = + newTagContextBuilder(tableId, zone, cluster) + .putLocal(BuiltinMeasureConstants.STATUS, TagValue.create(status)); + + if (operationType == OperationType.ServerStreaming + && spanName.getMethodName().equals("ReadRows")) { + tagCtx.putLocal(BuiltinMeasureConstants.STREAMING, TagValue.create("true")); + } else { + tagCtx.putLocal(BuiltinMeasureConstants.STREAMING, TagValue.create("false")); + } + + attemptLevelWithStreaming.record(tagCtx.build()); + } + + public void recordOperationLevelWithoutStreaming( + String status, String tableId, String zone, String cluster) { + TagContextBuilder tagCtx = + newTagContextBuilder(tableId, zone, cluster) + .putLocal(BuiltinMeasureConstants.STATUS, TagValue.create(status)); + + operationLevelNoStreaming.record(tagCtx.build()); + } + + public void recordOperationLevelWithStreaming( + String status, String tableId, String zone, String cluster) { + TagContextBuilder tagCtx = + newTagContextBuilder(tableId, zone, cluster) + .putLocal(BuiltinMeasureConstants.STATUS, TagValue.create(status)); + + if (operationType == OperationType.ServerStreaming + && spanName.getMethodName().equals("ReadRows")) { + tagCtx.putLocal(BuiltinMeasureConstants.STREAMING, TagValue.create("true")); + } else { + tagCtx.putLocal(BuiltinMeasureConstants.STREAMING, TagValue.create("false")); + } + + operationLevelWithStreaming.record(tagCtx.build()); + } + + public void recordOperationLatencies(long operationLatency) { + operationLevelWithStreaming.put(BuiltinMeasureConstants.OPERATION_LATENCIES, operationLatency); + } + + public void recordAttemptLatency(long attemptLatency) { + attemptLevelWithStreaming.put(BuiltinMeasureConstants.ATTEMPT_LATENCIES, attemptLatency); + } + + public void recordRetryCount(int attemptCount) { + operationLevelNoStreaming.put(BuiltinMeasureConstants.RETRY_COUNT, attemptCount); + } + + public void recordApplicationLatency(long applicationLatency) { + operationLevelWithStreaming.put( + BuiltinMeasureConstants.APPLICATION_LATENCIES, applicationLatency); + } + + public void recordFirstResponseLatency(long firstResponseLatency) { + operationLevelNoStreaming.put( + BuiltinMeasureConstants.FIRST_RESPONSE_LATENCIES, firstResponseLatency); + } + + public void recordGfeLatencies(long serverLatency) { + attemptLevelWithStreaming.put(BuiltinMeasureConstants.SERVER_LATENCIES, serverLatency); + } + + public void recordGfeMissingHeaders(long connectivityErrors) { + attemptLevelNoStreaming.put( + BuiltinMeasureConstants.CONNECTIVITY_ERROR_COUNT, connectivityErrors); + } + + public void recordBatchRequestThrottled( + long throttledTimeMs, String tableId, String zone, String cluster) { + MeasureMap measures = + statsRecorder + .newMeasureMap() + .put(BuiltinMeasureConstants.THROTTLING_LATENCIES, throttledTimeMs); + measures.record(newTagContextBuilder(tableId, zone, cluster).build()); + } + + private TagContextBuilder newTagContextBuilder(String tableId, String zone, String cluster) { + TagContextBuilder tagContextBuilder = + tagger + .toBuilder(parentContext) + .putLocal(BuiltinMeasureConstants.CLIENT_NAME, TagValue.create("bigtable-java")) + .putLocal(BuiltinMeasureConstants.METHOD, TagValue.create(spanName.toString())) + .putLocal(BuiltinMeasureConstants.TABLE, TagValue.create(tableId)) + .putLocal(BuiltinMeasureConstants.ZONE, TagValue.create(zone)) + .putLocal(BuiltinMeasureConstants.CLUSTER, TagValue.create(cluster)); + for (Map.Entry entry : statsAttributes.entrySet()) { + tagContextBuilder.putLocal(TagKey.create(entry.getKey()), TagValue.create(entry.getValue())); + } + return tagContextBuilder; + } +} diff --git a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorderTest.java b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorderTest.java new file mode 100644 index 000000000000..8f0c2688bf12 --- /dev/null +++ b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorderTest.java @@ -0,0 +1,433 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.stats; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.api.gax.tracing.ApiTracerFactory; +import com.google.api.gax.tracing.SpanName; +import com.google.common.collect.ImmutableMap; +import io.grpc.Status; +import org.junit.Before; +import org.junit.Test; + +public class BuiltinMetricsRecorderTest { + + private final String PROJECT_ID = "fake-project"; + private final String INSTANCE_ID = "fake-instance"; + private final String APP_PROFILE_ID = "fake-app-profile"; + + private final String TABLE_ID = "fake-table-id"; + private final String ZONE = "fake-zone"; + private final String CLUSTER = "fake-cluster"; + + private StatsWrapper wrapper; + + @Before + public void setup() { + this.wrapper = new StatsWrapper(true); + BuiltinViews views = new BuiltinViews(wrapper); + views.registerBigtableBuiltinViews(); + } + + @Test + public void testStreamingOperation() throws InterruptedException { + BuiltinMetricsRecorder tracer = + new BuiltinMetricsRecorder( + ApiTracerFactory.OperationType.ServerStreaming, + SpanName.of("Bigtable", "ReadRows"), + ImmutableMap.of( + BuiltinMeasureConstants.PROJECT_ID.getName(), PROJECT_ID, + BuiltinMeasureConstants.INSTANCE_ID.getName(), INSTANCE_ID, + BuiltinMeasureConstants.APP_PROFILE.getName(), APP_PROFILE_ID), + wrapper); + + long operationLatency = 1234; + int attemptCount = 2; + long attemptLatency = 56; + long serverLatency = 78; + long applicationLatency = 901; + long connectivityErrorCount = 15; + long throttlingLatency = 50; + long firstResponseLatency = 90; + + tracer.recordOperationLatencies(operationLatency); + tracer.recordRetryCount(attemptCount); + tracer.recordAttemptLatency(attemptLatency); + tracer.recordApplicationLatency(applicationLatency); + tracer.recordGfeLatencies(serverLatency); + tracer.recordGfeMissingHeaders(connectivityErrorCount); + tracer.recordFirstResponseLatency(firstResponseLatency); + tracer.recordBatchRequestThrottled(throttlingLatency, TABLE_ID, ZONE, CLUSTER); + + tracer.recordAttemptLevelWithoutStreaming( + Status.UNAVAILABLE.toString(), TABLE_ID, ZONE, CLUSTER); + tracer.recordAttemptLevelWithStreaming(Status.ABORTED.toString(), TABLE_ID, ZONE, CLUSTER); + tracer.recordOperationLevelWithoutStreaming("OK", TABLE_ID, ZONE, CLUSTER); + tracer.recordOperationLevelWithStreaming("OK", TABLE_ID, ZONE, CLUSTER); + + Thread.sleep(100); + + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.OPERATION_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, "Bigtable.ReadRows", + BuiltinMeasureConstants.STATUS, "OK", + BuiltinMeasureConstants.TABLE, TABLE_ID, + BuiltinMeasureConstants.ZONE, ZONE, + BuiltinMeasureConstants.CLUSTER, CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, "bigtable-java", + BuiltinMeasureConstants.STREAMING, "true"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(operationLatency); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.ATTEMPT_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.ReadRows", + BuiltinMeasureConstants.STATUS, + Status.ABORTED.toString(), + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java", + BuiltinMeasureConstants.STREAMING, + "true"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(attemptLatency); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.RETRY_COUNT_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.ReadRows", + BuiltinMeasureConstants.STATUS, + "OK", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(attemptCount); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.SERVER_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.ReadRows", + BuiltinMeasureConstants.STATUS, + Status.ABORTED.toString(), + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java", + BuiltinMeasureConstants.STREAMING, + "true", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(serverLatency); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.APPLICATION_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.ReadRows", + BuiltinMeasureConstants.STATUS, + "OK", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java", + BuiltinMeasureConstants.STREAMING, + "true"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(applicationLatency); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.CONNECTIVITY_ERROR_COUNT_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.ReadRows", + BuiltinMeasureConstants.STATUS, + Status.UNAVAILABLE.toString(), + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(connectivityErrorCount); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.THROTTLING_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, "Bigtable.ReadRows", + BuiltinMeasureConstants.TABLE, TABLE_ID, + BuiltinMeasureConstants.ZONE, ZONE, + BuiltinMeasureConstants.CLUSTER, CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, "bigtable-java"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(throttlingLatency); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.FIRST_RESPONSE_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.ReadRows", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER, + BuiltinMeasureConstants.STATUS, + "OK", + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(firstResponseLatency); + } + + @Test + public void testUnaryOperations() throws InterruptedException { + BuiltinMetricsRecorder tracer = + new BuiltinMetricsRecorder( + ApiTracerFactory.OperationType.ServerStreaming, + SpanName.of("Bigtable", "MutateRow"), + ImmutableMap.of( + BuiltinMeasureConstants.PROJECT_ID.getName(), PROJECT_ID, + BuiltinMeasureConstants.INSTANCE_ID.getName(), INSTANCE_ID, + BuiltinMeasureConstants.APP_PROFILE.getName(), APP_PROFILE_ID), + wrapper); + + long operationLatency = 1234; + int attemptCount = 2; + long attemptLatency = 56; + long serverLatency = 78; + long applicationLatency = 901; + long connectivityErrorCount = 15; + long throttlingLatency = 50; + long firstResponseLatency = 90; + + tracer.recordOperationLatencies(operationLatency); + tracer.recordRetryCount(attemptCount); + tracer.recordAttemptLatency(attemptLatency); + tracer.recordApplicationLatency(applicationLatency); + tracer.recordGfeLatencies(serverLatency); + tracer.recordGfeMissingHeaders(connectivityErrorCount); + tracer.recordFirstResponseLatency(firstResponseLatency); + tracer.recordBatchRequestThrottled(throttlingLatency, TABLE_ID, ZONE, CLUSTER); + + tracer.recordOperationLevelWithStreaming("OK", TABLE_ID, ZONE, CLUSTER); + tracer.recordOperationLevelWithoutStreaming("OK", TABLE_ID, ZONE, CLUSTER); + tracer.recordAttemptLevelWithoutStreaming( + Status.UNAVAILABLE.toString(), TABLE_ID, ZONE, CLUSTER); + tracer.recordAttemptLevelWithStreaming(Status.ABORTED.toString(), TABLE_ID, ZONE, CLUSTER); + + Thread.sleep(100); + + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.OPERATION_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, "Bigtable.MutateRow", + BuiltinMeasureConstants.STATUS, "OK", + BuiltinMeasureConstants.TABLE, TABLE_ID, + BuiltinMeasureConstants.ZONE, ZONE, + BuiltinMeasureConstants.CLUSTER, CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, "bigtable-java", + BuiltinMeasureConstants.STREAMING, "false"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(operationLatency); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.ATTEMPT_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.MutateRow", + BuiltinMeasureConstants.STATUS, + Status.ABORTED.toString(), + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java", + BuiltinMeasureConstants.STREAMING, + "false"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(attemptLatency); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.RETRY_COUNT_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.MutateRow", + BuiltinMeasureConstants.STATUS, + "OK", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(attemptCount); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.SERVER_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.MutateRow", + BuiltinMeasureConstants.STATUS, + Status.ABORTED.toString(), + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java", + BuiltinMeasureConstants.STREAMING, + "false", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(serverLatency); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.APPLICATION_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.MutateRow", + BuiltinMeasureConstants.STATUS, + "OK", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java", + BuiltinMeasureConstants.STREAMING, + "false"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(applicationLatency); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.CONNECTIVITY_ERROR_COUNT_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.MutateRow", + BuiltinMeasureConstants.STATUS, + Status.UNAVAILABLE.toString(), + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(connectivityErrorCount); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.THROTTLING_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, "Bigtable.MutateRow", + BuiltinMeasureConstants.TABLE, TABLE_ID, + BuiltinMeasureConstants.ZONE, ZONE, + BuiltinMeasureConstants.CLUSTER, CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, "bigtable-java"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(throttlingLatency); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.FIRST_RESPONSE_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.MutateRow", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER, + BuiltinMeasureConstants.STATUS, + "OK", + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(firstResponseLatency); + } +} From 8dc5cce4f727359ee01720b8bff397f790f7bcf8 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Wed, 11 May 2022 14:20:34 -0400 Subject: [PATCH 02/18] remove status from application latency --- .../stats/BuiltinMetricsRecorder.java | 19 ++++++++++++++++--- .../stats/BuiltinMetricsRecorderTest.java | 4 ++-- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorder.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorder.java index 51a0feb5cbcd..a66da11d0b78 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorder.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorder.java @@ -125,9 +125,22 @@ public void recordRetryCount(int attemptCount) { operationLevelNoStreaming.put(BuiltinMeasureConstants.RETRY_COUNT, attemptCount); } - public void recordApplicationLatency(long applicationLatency) { - operationLevelWithStreaming.put( - BuiltinMeasureConstants.APPLICATION_LATENCIES, applicationLatency); + public void recordApplicationLatency( + long applicationLatency, String tableId, String zone, String cluster) { + MeasureMap measures = + statsRecorder + .newMeasureMap() + .put(BuiltinMeasureConstants.APPLICATION_LATENCIES, applicationLatency); + + TagContextBuilder tagCtx = newTagContextBuilder(tableId, zone, cluster); + if (operationType == OperationType.ServerStreaming + && spanName.getMethodName().equals("ReadRows")) { + tagCtx.putLocal(BuiltinMeasureConstants.STREAMING, TagValue.create("true")); + } else { + tagCtx.putLocal(BuiltinMeasureConstants.STREAMING, TagValue.create("false")); + } + + measures.record(tagCtx.build()); } public void recordFirstResponseLatency(long firstResponseLatency) { diff --git a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorderTest.java b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorderTest.java index 8f0c2688bf12..ce0cb0e01195 100644 --- a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorderTest.java +++ b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorderTest.java @@ -67,7 +67,7 @@ public void testStreamingOperation() throws InterruptedException { tracer.recordOperationLatencies(operationLatency); tracer.recordRetryCount(attemptCount); tracer.recordAttemptLatency(attemptLatency); - tracer.recordApplicationLatency(applicationLatency); + tracer.recordApplicationLatency(applicationLatency, TABLE_ID, ZONE, CLUSTER); tracer.recordGfeLatencies(serverLatency); tracer.recordGfeMissingHeaders(connectivityErrorCount); tracer.recordFirstResponseLatency(firstResponseLatency); @@ -261,7 +261,7 @@ public void testUnaryOperations() throws InterruptedException { tracer.recordOperationLatencies(operationLatency); tracer.recordRetryCount(attemptCount); tracer.recordAttemptLatency(attemptLatency); - tracer.recordApplicationLatency(applicationLatency); + tracer.recordApplicationLatency(applicationLatency, TABLE_ID, ZONE, CLUSTER); tracer.recordGfeLatencies(serverLatency); tracer.recordGfeMissingHeaders(connectivityErrorCount); tracer.recordFirstResponseLatency(firstResponseLatency); From 872c4c22af8643f2db757de8e00a70b68e2c818d Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Tue, 10 May 2022 11:49:36 -0400 Subject: [PATCH 03/18] feat: update tracers to use built in metrics --- google-cloud-bigtable/pom.xml | 5 +++++ .../bigtable/data/v2/stub/EnhancedBigtableStub.java | 5 +++++ .../stub/metrics/BigtableTracerStreamingCallable.java | 4 ++-- .../v2/stub/metrics/BigtableTracerUnaryCallable.java | 11 ++++++----- .../data/v2/stub/metrics/BuiltinMetricsTracer.java | 4 ++++ .../v2/stub/metrics/BuiltinMetricsTracerFactory.java | 4 ++++ .../bigtable/data/v2/stub/metrics/MetricsTracer.java | 10 ++++++++++ .../v2/stub/metrics/BuiltinMetricsTracerTest.java | 4 ++++ pom.xml | 9 +++++++++ 9 files changed, 49 insertions(+), 7 deletions(-) diff --git a/google-cloud-bigtable/pom.xml b/google-cloud-bigtable/pom.xml index 8cd922d5f44c..e220c75cdda8 100644 --- a/google-cloud-bigtable/pom.xml +++ b/google-cloud-bigtable/pom.xml @@ -54,6 +54,11 @@ pom import + + com.google.cloud + google-cloud-bigtable-stats + 2.6.3-SNAPSHOT + diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java index ec237aabf73a..b57ecddd09be 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java @@ -90,6 +90,7 @@ import com.google.cloud.bigtable.data.v2.stub.readrows.ReadRowsUserCallable; import com.google.cloud.bigtable.data.v2.stub.readrows.RowMergingCallable; import com.google.cloud.bigtable.gaxx.retrying.ApiResultRetryAlgorithm; +import com.google.cloud.bigtable.stats.StatsWrapper; import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -377,7 +378,11 @@ public UnaryCallable createReadRowCallable(RowAdapter *
  • Upon receiving the response stream, it will merge the {@link * com.google.bigtable.v2.ReadRowsResponse.CellChunk}s in logical rows. The actual row * implementation can be configured by the {@code rowAdapter} parameter. +<<<<<<< HEAD *
  • Add bigtable tracer for tracking bigtable specific metrics. +======= + *
  • Add BigtableTracer callable for tracking Bigtable specific metrics +>>>>>>> ca28be67 (feat: update tracers to use built in metrics) *
  • Retry/resume on failure. *
  • Filter out marker rows. * diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java index 6a902029eb9d..cf8b7d802612 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java @@ -36,7 +36,7 @@ * returned null, it probably means that the request has never reached GFE, and it'll increment the * gfe_header_missing_counter in this case. * - *

    -Call {@link BigtableTracer#onRequest()} to record the request events in a stream. + *

    -Call {@link BigtableTracer#onRequest(int)} to record the request events in a stream. * *

    This class is considered an internal implementation detail and not meant to be used by * applications. @@ -84,7 +84,7 @@ private class BigtableTracerResponseObserver implements ResponseObser @Override public void onStart(final StreamController controller) { - TracedStreamController tracedController = new TracedStreamController(controller, tracer); + final TracedStreamController tracedController = new TracedStreamController(controller, tracer); outerObserver.onStart(tracedController); } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java index de53a0c94e59..1214d4f42a1c 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java @@ -28,12 +28,13 @@ import javax.annotation.Nonnull; /** - * This callable will inject a {@link GrpcResponseMetadata} to access the headers and trailers - * returned by gRPC methods upon completion. The {@link BigtableTracer} will process metrics that - * were injected in the header/trailer and publish them to OpenCensus. If {@link - * GrpcResponseMetadata#getMetadata()} returned null, it probably means that the request has never - * reached GFE, and it'll increment the gfe_header_missing_counter in this case. + * This callable will: * + *

    - Inject a {@link GrpcResponseMetadata} to access the headers and trailers returned by gRPC + * methods upon completion. The {@link BigtableTracer} will process metrics that were injected in + * the header/trailer and publish them to OpenCensus. If {@link GrpcResponseMetadata#getMetadata()} + * returned null, it probably means that the request has never reached GFE, and it'll increment the + * gfe_header_missing_counter in this case. *

    This class is considered an internal implementation detail and not meant to be used by * applications. */ diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracer.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracer.java index 2148c674e3f8..9560a5ee54ba 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracer.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracer.java @@ -1,5 +1,9 @@ /* +<<<<<<< HEAD * Copyright 2022 Google LLC +======= + * Copyright 2021 Google LLC +>>>>>>> ca28be67 (feat: update tracers to use built in metrics) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerFactory.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerFactory.java index 794997071df9..d02150c89ac4 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerFactory.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerFactory.java @@ -1,5 +1,9 @@ /* +<<<<<<< HEAD * Copyright 2022 Google LLC +======= + * Copyright 2021 Google LLC +>>>>>>> ca28be67 (feat: update tracers to use built in metrics) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/MetricsTracer.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/MetricsTracer.java index 3c63b1b5f795..efb41998bcaf 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/MetricsTracer.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/MetricsTracer.java @@ -228,4 +228,14 @@ private TagContextBuilder newTagCtxBuilder() { return tagCtx; } + + @Override + public void setLocations(String zone, String cluster) { + // noop + } + + @Override + public void onRequest() { + // noop + } } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java index 133f06767a08..36607cb48e27 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java @@ -1,5 +1,9 @@ /* +<<<<<<< HEAD * Copyright 2022 Google LLC +======= + * Copyright 2021 Google LLC +>>>>>>> ca28be67 (feat: update tracers to use built in metrics) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pom.xml b/pom.xml index f898b3952b4f..6e42834013a7 100644 --- a/pom.xml +++ b/pom.xml @@ -212,6 +212,15 @@ + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.4 + + + From 35418b260a24811e60124e523f17493a91900560 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Wed, 8 Sep 2021 16:06:07 -0400 Subject: [PATCH 04/18] feat: add response protos --- google-cloud-bigtable-stats/pom.xml | 1 - .../stats/BuiltinMetricsRecorder.java | 1 + .../stats/BuiltinMetricsRecorderTest.java | 1 + .../BigtableTracerStreamingCallable.java | 22 +++++++++++ .../metrics/BigtableTracerUnaryCallable.java | 20 ++++++++++ .../metrics/BuiltinMetricsTracerTest.java | 38 +++++++++++++++++-- .../google/bigtable/v2/ResponseParams.java | 5 ++- .../bigtable/v2/ResponseParamsOrBuilder.java | 2 +- .../bigtable/v2/ResponseParamsProto.java | 2 +- 9 files changed, 85 insertions(+), 7 deletions(-) diff --git a/google-cloud-bigtable-stats/pom.xml b/google-cloud-bigtable-stats/pom.xml index fa96d246b9d1..eff1b05e1411 100644 --- a/google-cloud-bigtable-stats/pom.xml +++ b/google-cloud-bigtable-stats/pom.xml @@ -37,7 +37,6 @@ com.google.api api-common - io.opencensus opencensus-api diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorder.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorder.java index a66da11d0b78..4cc8a829c065 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorder.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorder.java @@ -133,6 +133,7 @@ public void recordApplicationLatency( .put(BuiltinMeasureConstants.APPLICATION_LATENCIES, applicationLatency); TagContextBuilder tagCtx = newTagContextBuilder(tableId, zone, cluster); + if (operationType == OperationType.ServerStreaming && spanName.getMethodName().equals("ReadRows")) { tagCtx.putLocal(BuiltinMeasureConstants.STREAMING, TagValue.create("true")); diff --git a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorderTest.java b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorderTest.java index ce0cb0e01195..dc34d18f6281 100644 --- a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorderTest.java +++ b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorderTest.java @@ -71,6 +71,7 @@ public void testStreamingOperation() throws InterruptedException { tracer.recordGfeLatencies(serverLatency); tracer.recordGfeMissingHeaders(connectivityErrorCount); tracer.recordFirstResponseLatency(firstResponseLatency); + tracer.recordBatchRequestThrottled(throttlingLatency, TABLE_ID, ZONE, CLUSTER); tracer.recordAttemptLevelWithoutStreaming( diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java index cf8b7d802612..a564ea9d1bad 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java @@ -21,8 +21,10 @@ import com.google.api.gax.rpc.ResponseObserver; import com.google.api.gax.rpc.ServerStreamingCallable; import com.google.api.gax.rpc.StreamController; +import com.google.bigtable.v2.ResponseParams; import com.google.common.base.Preconditions; import com.google.common.base.Stopwatch; +import com.google.protobuf.InvalidProtocolBufferException; import io.grpc.Metadata; import java.util.concurrent.TimeUnit; import javax.annotation.Nonnull; @@ -102,6 +104,16 @@ public void onError(Throwable t) { Metadata metadata = responseMetadata.getMetadata(); Long latency = Util.getGfeLatency(metadata); tracer.recordGfeMetadata(latency, t); + try { + byte[] trailers = + responseMetadata + .getTrailingMetadata() + .get(Metadata.Key.of(Util.TRAILER_KEY, Metadata.BINARY_BYTE_MARSHALLER)); + ResponseParams decodedTrailers = ResponseParams.parseFrom(trailers); + tracer.setLocations(decodedTrailers.getZoneId(), decodedTrailers.getClusterId()); + } catch (NullPointerException | InvalidProtocolBufferException e) { + } + outerObserver.onError(t); } @@ -110,6 +122,16 @@ public void onComplete() { Metadata metadata = responseMetadata.getMetadata(); Long latency = Util.getGfeLatency(metadata); tracer.recordGfeMetadata(latency, null); + try { + byte[] trailers = + responseMetadata + .getTrailingMetadata() + .get(Metadata.Key.of(Util.TRAILER_KEY, Metadata.BINARY_BYTE_MARSHALLER)); + ResponseParams decodedTrailers = ResponseParams.parseFrom(trailers); + tracer.setLocations(decodedTrailers.getZoneId(), decodedTrailers.getClusterId()); + } catch (NullPointerException | InvalidProtocolBufferException e) { + } + outerObserver.onComplete(); } } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java index 1214d4f42a1c..e570df9552fb 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java @@ -22,8 +22,10 @@ import com.google.api.gax.grpc.GrpcResponseMetadata; import com.google.api.gax.rpc.ApiCallContext; import com.google.api.gax.rpc.UnaryCallable; +import com.google.bigtable.v2.ResponseParams; import com.google.common.base.Preconditions; import com.google.common.util.concurrent.MoreExecutors; +import com.google.protobuf.InvalidProtocolBufferException; import io.grpc.Metadata; import javax.annotation.Nonnull; @@ -79,6 +81,15 @@ public void onFailure(Throwable throwable) { Metadata metadata = responseMetadata.getMetadata(); Long latency = Util.getGfeLatency(metadata); tracer.recordGfeMetadata(latency, throwable); + try { + byte[] trailers = + responseMetadata + .getTrailingMetadata() + .get(Metadata.Key.of(Util.TRAILER_KEY, Metadata.BINARY_BYTE_MARSHALLER)); + ResponseParams decodedTrailers = ResponseParams.parseFrom(trailers); + tracer.setLocations(decodedTrailers.getZoneId(), decodedTrailers.getClusterId()); + } catch (NullPointerException | InvalidProtocolBufferException e) { + } } @Override @@ -86,6 +97,15 @@ public void onSuccess(ResponseT response) { Metadata metadata = responseMetadata.getMetadata(); Long latency = Util.getGfeLatency(metadata); tracer.recordGfeMetadata(latency, null); + try { + byte[] trailers = + responseMetadata + .getTrailingMetadata() + .get(Metadata.Key.of(Util.TRAILER_KEY, Metadata.BINARY_BYTE_MARSHALLER)); + ResponseParams decodedTrailers = ResponseParams.parseFrom(trailers); + tracer.setLocations(decodedTrailers.getZoneId(), decodedTrailers.getClusterId()); + } catch (NullPointerException | InvalidProtocolBufferException e) { + } } } } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java index 36607cb48e27..bbd53944b18a 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java @@ -38,6 +38,7 @@ >>>>>>> ca28be67 (feat: update tracers to use built in metrics) import com.google.bigtable.v2.MutateRowResponse; import com.google.bigtable.v2.ReadRowsRequest; import com.google.bigtable.v2.ReadRowsResponse; +import com.google.bigtable.v2.ResponseParams; import com.google.cloud.bigtable.data.v2.BigtableDataSettings; import com.google.cloud.bigtable.data.v2.FakeServiceBuilder; import com.google.cloud.bigtable.data.v2.models.Query; @@ -88,7 +89,10 @@ public class BuiltinMetricsTracerTest { private static final String INSTANCE_ID = "fake-instance"; private static final String APP_PROFILE_ID = "default"; private static final String TABLE_ID = "fake-table"; - private static final String UNDEFINED = "undefined"; + private static final String ZONE_0 = "us-west-1"; + private static final String CLUSTER_0 = "cluster-0"; + private static final String ZONE_1 = "us-east-1"; + private static final String CLUSTER_1 = "cluster-1"; private static final long FAKE_SERVER_TIMING = 50; private static final long SERVER_LATENCY = 100; private static final long APPLICATION_LATENCY = 200; @@ -112,6 +116,7 @@ public class BuiltinMetricsTracerTest { public void setUp() throws Exception { fakeService = new FakeService(); + final AtomicInteger count = new AtomicInteger(0);gi // Add an interceptor to add server-timing in headers ServerInterceptor trailersInterceptor = new ServerInterceptor() { @@ -129,6 +134,33 @@ public void sendHeaders(Metadata headers) { String.format("gfet4t7; dur=%d", FAKE_SERVER_TIMING)); super.sendHeaders(headers); } + + @Override + public void close(Status status, Metadata trailers) { + int currentCount = count.getAndIncrement(); + if (currentCount == 0) { + ResponseParams params = + ResponseParams.newBuilder() + .setZoneId(ZONE_0) + .setClusterId(CLUSTER_0) + .build(); + byte[] byteArray = params.toByteArray(); + trailers.put( + Metadata.Key.of(Util.TRAILER_KEY, Metadata.BINARY_BYTE_MARSHALLER), + byteArray); + } else { + ResponseParams params = + ResponseParams.newBuilder() + .setClusterId(CLUSTER_1) + .setZoneId(ZONE_1) + .build(); + byte[] byteArray = params.toByteArray(); + trailers.put( + Metadata.Key.of(Util.TRAILER_KEY, Metadata.BINARY_BYTE_MARSHALLER), + byteArray); + } + super.close(status, trailers); + } }, metadata); } @@ -337,8 +369,8 @@ public void testMutateRowAttempts() { // and when the record() is called in onOperationCompletion(). verify(statsRecorderWrapper, timeout(50).times(fakeService.getAttemptCounter().get() + 1)) .record(status.capture(), tableId.capture(), zone.capture(), cluster.capture()); - assertThat(zone.getAllValues()).containsExactly(UNDEFINED, UNDEFINED, UNDEFINED, UNDEFINED); - assertThat(cluster.getAllValues()).containsExactly(UNDEFINED, UNDEFINED, UNDEFINED, UNDEFINED); + assertThat(zone.getAllValues()).containsExactly(ZONE_0, ZONE_1, ZONE_1, ZONE_1); + assertThat(cluster.getAllValues()).containsExactly(CLUSTER_0, CLUSTER_1, CLUSTER_1, CLUSTER_1); assertThat(status.getAllValues()).containsExactly("UNAVAILABLE", "UNAVAILABLE", "OK", "OK"); } diff --git a/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ResponseParams.java b/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ResponseParams.java index c64592a2b422..5d1c9c7b7be3 100644 --- a/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ResponseParams.java +++ b/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ResponseParams.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -438,9 +438,12 @@ protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.Build * * 

        * Response metadata proto
+<<<<<<< HEAD
    * This is an experimental feature that will be used to get zone_id and
    * cluster_id from response trailers to tag the metrics. This should not be
    * used by customers directly
+=======
+>>>>>>> 880e0a1c (feat: add response protos)
    *
    * * Protobuf type {@code google.bigtable.v2.ResponseParams} diff --git a/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ResponseParamsOrBuilder.java b/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ResponseParamsOrBuilder.java index 118eef7410d8..7e12c30eca3a 100644 --- a/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ResponseParamsOrBuilder.java +++ b/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ResponseParamsOrBuilder.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ResponseParamsProto.java b/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ResponseParamsProto.java index 9bfba6440505..ab96c7034e6a 100644 --- a/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ResponseParamsProto.java +++ b/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ResponseParamsProto.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From c81f3232ed08cd0b6c3477efac6cbb43b5cf6021 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Wed, 29 Jun 2022 16:34:31 -0400 Subject: [PATCH 05/18] fix broken tests --- .../stats/BuiltinMetricsRecorder.java | 184 -------- .../stats/BuiltinMetricsRecorderTest.java | 434 ------------------ google-cloud-bigtable/pom.xml | 9 +- .../data/v2/stub/EnhancedBigtableStub.java | 1 - .../v2/stub/metrics/BuiltinMetricsTracer.java | 4 - .../metrics/BuiltinMetricsTracerFactory.java | 4 - .../data/v2/stub/metrics/MetricsTracer.java | 10 - .../metrics/BuiltinMetricsTracerTest.java | 6 +- 8 files changed, 5 insertions(+), 647 deletions(-) delete mode 100644 google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorder.java delete mode 100644 google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorderTest.java diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorder.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorder.java deleted file mode 100644 index 4cc8a829c065..000000000000 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorder.java +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.cloud.bigtable.stats; - -import com.google.api.core.InternalApi; -import com.google.api.gax.tracing.ApiTracerFactory.OperationType; -import com.google.api.gax.tracing.SpanName; -import io.opencensus.stats.MeasureMap; -import io.opencensus.stats.StatsRecorder; -import io.opencensus.tags.TagContext; -import io.opencensus.tags.TagContextBuilder; -import io.opencensus.tags.TagKey; -import io.opencensus.tags.TagValue; -import io.opencensus.tags.Tagger; -import io.opencensus.tags.Tags; -import java.util.Map; - -/** Add built-in metrics to the measure map * */ -@InternalApi("For internal use only") -public class BuiltinMetricsRecorder { - - private final OperationType operationType; - - private final Tagger tagger; - private final StatsRecorder statsRecorder; - private final TagContext parentContext; - private final SpanName spanName; - private final Map statsAttributes; - - private MeasureMap attemptLevelNoStreaming; - private MeasureMap attemptLevelWithStreaming; - private MeasureMap operationLevelNoStreaming; - private MeasureMap operationLevelWithStreaming; - - public BuiltinMetricsRecorder( - OperationType operationType, - SpanName spanName, - Map statsAttributes, - StatsWrapper builtinMetricsWrapper) { - this.operationType = operationType; - this.tagger = Tags.getTagger(); - this.statsRecorder = builtinMetricsWrapper.getStatsRecorder(); - this.spanName = spanName; - this.parentContext = tagger.getCurrentTagContext(); - this.statsAttributes = statsAttributes; - - this.attemptLevelNoStreaming = statsRecorder.newMeasureMap(); - this.attemptLevelWithStreaming = statsRecorder.newMeasureMap(); - this.operationLevelNoStreaming = statsRecorder.newMeasureMap(); - this.operationLevelWithStreaming = statsRecorder.newMeasureMap(); - } - - public void recordAttemptLevelWithoutStreaming( - String status, String tableId, String zone, String cluster) { - TagContextBuilder tagCtx = - newTagContextBuilder(tableId, zone, cluster) - .putLocal(BuiltinMeasureConstants.STATUS, TagValue.create(status)); - - attemptLevelNoStreaming.record(tagCtx.build()); - } - - public void recordAttemptLevelWithStreaming( - String status, String tableId, String zone, String cluster) { - TagContextBuilder tagCtx = - newTagContextBuilder(tableId, zone, cluster) - .putLocal(BuiltinMeasureConstants.STATUS, TagValue.create(status)); - - if (operationType == OperationType.ServerStreaming - && spanName.getMethodName().equals("ReadRows")) { - tagCtx.putLocal(BuiltinMeasureConstants.STREAMING, TagValue.create("true")); - } else { - tagCtx.putLocal(BuiltinMeasureConstants.STREAMING, TagValue.create("false")); - } - - attemptLevelWithStreaming.record(tagCtx.build()); - } - - public void recordOperationLevelWithoutStreaming( - String status, String tableId, String zone, String cluster) { - TagContextBuilder tagCtx = - newTagContextBuilder(tableId, zone, cluster) - .putLocal(BuiltinMeasureConstants.STATUS, TagValue.create(status)); - - operationLevelNoStreaming.record(tagCtx.build()); - } - - public void recordOperationLevelWithStreaming( - String status, String tableId, String zone, String cluster) { - TagContextBuilder tagCtx = - newTagContextBuilder(tableId, zone, cluster) - .putLocal(BuiltinMeasureConstants.STATUS, TagValue.create(status)); - - if (operationType == OperationType.ServerStreaming - && spanName.getMethodName().equals("ReadRows")) { - tagCtx.putLocal(BuiltinMeasureConstants.STREAMING, TagValue.create("true")); - } else { - tagCtx.putLocal(BuiltinMeasureConstants.STREAMING, TagValue.create("false")); - } - - operationLevelWithStreaming.record(tagCtx.build()); - } - - public void recordOperationLatencies(long operationLatency) { - operationLevelWithStreaming.put(BuiltinMeasureConstants.OPERATION_LATENCIES, operationLatency); - } - - public void recordAttemptLatency(long attemptLatency) { - attemptLevelWithStreaming.put(BuiltinMeasureConstants.ATTEMPT_LATENCIES, attemptLatency); - } - - public void recordRetryCount(int attemptCount) { - operationLevelNoStreaming.put(BuiltinMeasureConstants.RETRY_COUNT, attemptCount); - } - - public void recordApplicationLatency( - long applicationLatency, String tableId, String zone, String cluster) { - MeasureMap measures = - statsRecorder - .newMeasureMap() - .put(BuiltinMeasureConstants.APPLICATION_LATENCIES, applicationLatency); - - TagContextBuilder tagCtx = newTagContextBuilder(tableId, zone, cluster); - - if (operationType == OperationType.ServerStreaming - && spanName.getMethodName().equals("ReadRows")) { - tagCtx.putLocal(BuiltinMeasureConstants.STREAMING, TagValue.create("true")); - } else { - tagCtx.putLocal(BuiltinMeasureConstants.STREAMING, TagValue.create("false")); - } - - measures.record(tagCtx.build()); - } - - public void recordFirstResponseLatency(long firstResponseLatency) { - operationLevelNoStreaming.put( - BuiltinMeasureConstants.FIRST_RESPONSE_LATENCIES, firstResponseLatency); - } - - public void recordGfeLatencies(long serverLatency) { - attemptLevelWithStreaming.put(BuiltinMeasureConstants.SERVER_LATENCIES, serverLatency); - } - - public void recordGfeMissingHeaders(long connectivityErrors) { - attemptLevelNoStreaming.put( - BuiltinMeasureConstants.CONNECTIVITY_ERROR_COUNT, connectivityErrors); - } - - public void recordBatchRequestThrottled( - long throttledTimeMs, String tableId, String zone, String cluster) { - MeasureMap measures = - statsRecorder - .newMeasureMap() - .put(BuiltinMeasureConstants.THROTTLING_LATENCIES, throttledTimeMs); - measures.record(newTagContextBuilder(tableId, zone, cluster).build()); - } - - private TagContextBuilder newTagContextBuilder(String tableId, String zone, String cluster) { - TagContextBuilder tagContextBuilder = - tagger - .toBuilder(parentContext) - .putLocal(BuiltinMeasureConstants.CLIENT_NAME, TagValue.create("bigtable-java")) - .putLocal(BuiltinMeasureConstants.METHOD, TagValue.create(spanName.toString())) - .putLocal(BuiltinMeasureConstants.TABLE, TagValue.create(tableId)) - .putLocal(BuiltinMeasureConstants.ZONE, TagValue.create(zone)) - .putLocal(BuiltinMeasureConstants.CLUSTER, TagValue.create(cluster)); - for (Map.Entry entry : statsAttributes.entrySet()) { - tagContextBuilder.putLocal(TagKey.create(entry.getKey()), TagValue.create(entry.getValue())); - } - return tagContextBuilder; - } -} diff --git a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorderTest.java b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorderTest.java deleted file mode 100644 index dc34d18f6281..000000000000 --- a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorderTest.java +++ /dev/null @@ -1,434 +0,0 @@ -/* - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.cloud.bigtable.stats; - -import static com.google.common.truth.Truth.assertThat; - -import com.google.api.gax.tracing.ApiTracerFactory; -import com.google.api.gax.tracing.SpanName; -import com.google.common.collect.ImmutableMap; -import io.grpc.Status; -import org.junit.Before; -import org.junit.Test; - -public class BuiltinMetricsRecorderTest { - - private final String PROJECT_ID = "fake-project"; - private final String INSTANCE_ID = "fake-instance"; - private final String APP_PROFILE_ID = "fake-app-profile"; - - private final String TABLE_ID = "fake-table-id"; - private final String ZONE = "fake-zone"; - private final String CLUSTER = "fake-cluster"; - - private StatsWrapper wrapper; - - @Before - public void setup() { - this.wrapper = new StatsWrapper(true); - BuiltinViews views = new BuiltinViews(wrapper); - views.registerBigtableBuiltinViews(); - } - - @Test - public void testStreamingOperation() throws InterruptedException { - BuiltinMetricsRecorder tracer = - new BuiltinMetricsRecorder( - ApiTracerFactory.OperationType.ServerStreaming, - SpanName.of("Bigtable", "ReadRows"), - ImmutableMap.of( - BuiltinMeasureConstants.PROJECT_ID.getName(), PROJECT_ID, - BuiltinMeasureConstants.INSTANCE_ID.getName(), INSTANCE_ID, - BuiltinMeasureConstants.APP_PROFILE.getName(), APP_PROFILE_ID), - wrapper); - - long operationLatency = 1234; - int attemptCount = 2; - long attemptLatency = 56; - long serverLatency = 78; - long applicationLatency = 901; - long connectivityErrorCount = 15; - long throttlingLatency = 50; - long firstResponseLatency = 90; - - tracer.recordOperationLatencies(operationLatency); - tracer.recordRetryCount(attemptCount); - tracer.recordAttemptLatency(attemptLatency); - tracer.recordApplicationLatency(applicationLatency, TABLE_ID, ZONE, CLUSTER); - tracer.recordGfeLatencies(serverLatency); - tracer.recordGfeMissingHeaders(connectivityErrorCount); - tracer.recordFirstResponseLatency(firstResponseLatency); - - tracer.recordBatchRequestThrottled(throttlingLatency, TABLE_ID, ZONE, CLUSTER); - - tracer.recordAttemptLevelWithoutStreaming( - Status.UNAVAILABLE.toString(), TABLE_ID, ZONE, CLUSTER); - tracer.recordAttemptLevelWithStreaming(Status.ABORTED.toString(), TABLE_ID, ZONE, CLUSTER); - tracer.recordOperationLevelWithoutStreaming("OK", TABLE_ID, ZONE, CLUSTER); - tracer.recordOperationLevelWithStreaming("OK", TABLE_ID, ZONE, CLUSTER); - - Thread.sleep(100); - - assertThat( - wrapper.getAggregationValueAsLong( - BuiltinViewConstants.OPERATION_LATENCIES_VIEW, - ImmutableMap.of( - BuiltinMeasureConstants.METHOD, "Bigtable.ReadRows", - BuiltinMeasureConstants.STATUS, "OK", - BuiltinMeasureConstants.TABLE, TABLE_ID, - BuiltinMeasureConstants.ZONE, ZONE, - BuiltinMeasureConstants.CLUSTER, CLUSTER, - BuiltinMeasureConstants.CLIENT_NAME, "bigtable-java", - BuiltinMeasureConstants.STREAMING, "true"), - PROJECT_ID, - INSTANCE_ID, - APP_PROFILE_ID)) - .isEqualTo(operationLatency); - assertThat( - wrapper.getAggregationValueAsLong( - BuiltinViewConstants.ATTEMPT_LATENCIES_VIEW, - ImmutableMap.of( - BuiltinMeasureConstants.METHOD, - "Bigtable.ReadRows", - BuiltinMeasureConstants.STATUS, - Status.ABORTED.toString(), - BuiltinMeasureConstants.TABLE, - TABLE_ID, - BuiltinMeasureConstants.ZONE, - ZONE, - BuiltinMeasureConstants.CLUSTER, - CLUSTER, - BuiltinMeasureConstants.CLIENT_NAME, - "bigtable-java", - BuiltinMeasureConstants.STREAMING, - "true"), - PROJECT_ID, - INSTANCE_ID, - APP_PROFILE_ID)) - .isEqualTo(attemptLatency); - assertThat( - wrapper.getAggregationValueAsLong( - BuiltinViewConstants.RETRY_COUNT_VIEW, - ImmutableMap.of( - BuiltinMeasureConstants.METHOD, - "Bigtable.ReadRows", - BuiltinMeasureConstants.STATUS, - "OK", - BuiltinMeasureConstants.TABLE, - TABLE_ID, - BuiltinMeasureConstants.ZONE, - ZONE, - BuiltinMeasureConstants.CLUSTER, - CLUSTER, - BuiltinMeasureConstants.CLIENT_NAME, - "bigtable-java"), - PROJECT_ID, - INSTANCE_ID, - APP_PROFILE_ID)) - .isEqualTo(attemptCount); - assertThat( - wrapper.getAggregationValueAsLong( - BuiltinViewConstants.SERVER_LATENCIES_VIEW, - ImmutableMap.of( - BuiltinMeasureConstants.METHOD, - "Bigtable.ReadRows", - BuiltinMeasureConstants.STATUS, - Status.ABORTED.toString(), - BuiltinMeasureConstants.CLIENT_NAME, - "bigtable-java", - BuiltinMeasureConstants.STREAMING, - "true", - BuiltinMeasureConstants.TABLE, - TABLE_ID, - BuiltinMeasureConstants.ZONE, - ZONE, - BuiltinMeasureConstants.CLUSTER, - CLUSTER), - PROJECT_ID, - INSTANCE_ID, - APP_PROFILE_ID)) - .isEqualTo(serverLatency); - assertThat( - wrapper.getAggregationValueAsLong( - BuiltinViewConstants.APPLICATION_LATENCIES_VIEW, - ImmutableMap.of( - BuiltinMeasureConstants.METHOD, - "Bigtable.ReadRows", - BuiltinMeasureConstants.STATUS, - "OK", - BuiltinMeasureConstants.TABLE, - TABLE_ID, - BuiltinMeasureConstants.ZONE, - ZONE, - BuiltinMeasureConstants.CLUSTER, - CLUSTER, - BuiltinMeasureConstants.CLIENT_NAME, - "bigtable-java", - BuiltinMeasureConstants.STREAMING, - "true"), - PROJECT_ID, - INSTANCE_ID, - APP_PROFILE_ID)) - .isEqualTo(applicationLatency); - assertThat( - wrapper.getAggregationValueAsLong( - BuiltinViewConstants.CONNECTIVITY_ERROR_COUNT_VIEW, - ImmutableMap.of( - BuiltinMeasureConstants.METHOD, - "Bigtable.ReadRows", - BuiltinMeasureConstants.STATUS, - Status.UNAVAILABLE.toString(), - BuiltinMeasureConstants.CLIENT_NAME, - "bigtable-java", - BuiltinMeasureConstants.TABLE, - TABLE_ID, - BuiltinMeasureConstants.ZONE, - ZONE, - BuiltinMeasureConstants.CLUSTER, - CLUSTER), - PROJECT_ID, - INSTANCE_ID, - APP_PROFILE_ID)) - .isEqualTo(connectivityErrorCount); - assertThat( - wrapper.getAggregationValueAsLong( - BuiltinViewConstants.THROTTLING_LATENCIES_VIEW, - ImmutableMap.of( - BuiltinMeasureConstants.METHOD, "Bigtable.ReadRows", - BuiltinMeasureConstants.TABLE, TABLE_ID, - BuiltinMeasureConstants.ZONE, ZONE, - BuiltinMeasureConstants.CLUSTER, CLUSTER, - BuiltinMeasureConstants.CLIENT_NAME, "bigtable-java"), - PROJECT_ID, - INSTANCE_ID, - APP_PROFILE_ID)) - .isEqualTo(throttlingLatency); - assertThat( - wrapper.getAggregationValueAsLong( - BuiltinViewConstants.FIRST_RESPONSE_LATENCIES_VIEW, - ImmutableMap.of( - BuiltinMeasureConstants.METHOD, - "Bigtable.ReadRows", - BuiltinMeasureConstants.TABLE, - TABLE_ID, - BuiltinMeasureConstants.ZONE, - ZONE, - BuiltinMeasureConstants.CLUSTER, - CLUSTER, - BuiltinMeasureConstants.STATUS, - "OK", - BuiltinMeasureConstants.CLIENT_NAME, - "bigtable-java"), - PROJECT_ID, - INSTANCE_ID, - APP_PROFILE_ID)) - .isEqualTo(firstResponseLatency); - } - - @Test - public void testUnaryOperations() throws InterruptedException { - BuiltinMetricsRecorder tracer = - new BuiltinMetricsRecorder( - ApiTracerFactory.OperationType.ServerStreaming, - SpanName.of("Bigtable", "MutateRow"), - ImmutableMap.of( - BuiltinMeasureConstants.PROJECT_ID.getName(), PROJECT_ID, - BuiltinMeasureConstants.INSTANCE_ID.getName(), INSTANCE_ID, - BuiltinMeasureConstants.APP_PROFILE.getName(), APP_PROFILE_ID), - wrapper); - - long operationLatency = 1234; - int attemptCount = 2; - long attemptLatency = 56; - long serverLatency = 78; - long applicationLatency = 901; - long connectivityErrorCount = 15; - long throttlingLatency = 50; - long firstResponseLatency = 90; - - tracer.recordOperationLatencies(operationLatency); - tracer.recordRetryCount(attemptCount); - tracer.recordAttemptLatency(attemptLatency); - tracer.recordApplicationLatency(applicationLatency, TABLE_ID, ZONE, CLUSTER); - tracer.recordGfeLatencies(serverLatency); - tracer.recordGfeMissingHeaders(connectivityErrorCount); - tracer.recordFirstResponseLatency(firstResponseLatency); - tracer.recordBatchRequestThrottled(throttlingLatency, TABLE_ID, ZONE, CLUSTER); - - tracer.recordOperationLevelWithStreaming("OK", TABLE_ID, ZONE, CLUSTER); - tracer.recordOperationLevelWithoutStreaming("OK", TABLE_ID, ZONE, CLUSTER); - tracer.recordAttemptLevelWithoutStreaming( - Status.UNAVAILABLE.toString(), TABLE_ID, ZONE, CLUSTER); - tracer.recordAttemptLevelWithStreaming(Status.ABORTED.toString(), TABLE_ID, ZONE, CLUSTER); - - Thread.sleep(100); - - assertThat( - wrapper.getAggregationValueAsLong( - BuiltinViewConstants.OPERATION_LATENCIES_VIEW, - ImmutableMap.of( - BuiltinMeasureConstants.METHOD, "Bigtable.MutateRow", - BuiltinMeasureConstants.STATUS, "OK", - BuiltinMeasureConstants.TABLE, TABLE_ID, - BuiltinMeasureConstants.ZONE, ZONE, - BuiltinMeasureConstants.CLUSTER, CLUSTER, - BuiltinMeasureConstants.CLIENT_NAME, "bigtable-java", - BuiltinMeasureConstants.STREAMING, "false"), - PROJECT_ID, - INSTANCE_ID, - APP_PROFILE_ID)) - .isEqualTo(operationLatency); - assertThat( - wrapper.getAggregationValueAsLong( - BuiltinViewConstants.ATTEMPT_LATENCIES_VIEW, - ImmutableMap.of( - BuiltinMeasureConstants.METHOD, - "Bigtable.MutateRow", - BuiltinMeasureConstants.STATUS, - Status.ABORTED.toString(), - BuiltinMeasureConstants.TABLE, - TABLE_ID, - BuiltinMeasureConstants.ZONE, - ZONE, - BuiltinMeasureConstants.CLUSTER, - CLUSTER, - BuiltinMeasureConstants.CLIENT_NAME, - "bigtable-java", - BuiltinMeasureConstants.STREAMING, - "false"), - PROJECT_ID, - INSTANCE_ID, - APP_PROFILE_ID)) - .isEqualTo(attemptLatency); - assertThat( - wrapper.getAggregationValueAsLong( - BuiltinViewConstants.RETRY_COUNT_VIEW, - ImmutableMap.of( - BuiltinMeasureConstants.METHOD, - "Bigtable.MutateRow", - BuiltinMeasureConstants.STATUS, - "OK", - BuiltinMeasureConstants.TABLE, - TABLE_ID, - BuiltinMeasureConstants.ZONE, - ZONE, - BuiltinMeasureConstants.CLUSTER, - CLUSTER, - BuiltinMeasureConstants.CLIENT_NAME, - "bigtable-java"), - PROJECT_ID, - INSTANCE_ID, - APP_PROFILE_ID)) - .isEqualTo(attemptCount); - assertThat( - wrapper.getAggregationValueAsLong( - BuiltinViewConstants.SERVER_LATENCIES_VIEW, - ImmutableMap.of( - BuiltinMeasureConstants.METHOD, - "Bigtable.MutateRow", - BuiltinMeasureConstants.STATUS, - Status.ABORTED.toString(), - BuiltinMeasureConstants.CLIENT_NAME, - "bigtable-java", - BuiltinMeasureConstants.STREAMING, - "false", - BuiltinMeasureConstants.TABLE, - TABLE_ID, - BuiltinMeasureConstants.ZONE, - ZONE, - BuiltinMeasureConstants.CLUSTER, - CLUSTER), - PROJECT_ID, - INSTANCE_ID, - APP_PROFILE_ID)) - .isEqualTo(serverLatency); - assertThat( - wrapper.getAggregationValueAsLong( - BuiltinViewConstants.APPLICATION_LATENCIES_VIEW, - ImmutableMap.of( - BuiltinMeasureConstants.METHOD, - "Bigtable.MutateRow", - BuiltinMeasureConstants.STATUS, - "OK", - BuiltinMeasureConstants.TABLE, - TABLE_ID, - BuiltinMeasureConstants.ZONE, - ZONE, - BuiltinMeasureConstants.CLUSTER, - CLUSTER, - BuiltinMeasureConstants.CLIENT_NAME, - "bigtable-java", - BuiltinMeasureConstants.STREAMING, - "false"), - PROJECT_ID, - INSTANCE_ID, - APP_PROFILE_ID)) - .isEqualTo(applicationLatency); - assertThat( - wrapper.getAggregationValueAsLong( - BuiltinViewConstants.CONNECTIVITY_ERROR_COUNT_VIEW, - ImmutableMap.of( - BuiltinMeasureConstants.METHOD, - "Bigtable.MutateRow", - BuiltinMeasureConstants.STATUS, - Status.UNAVAILABLE.toString(), - BuiltinMeasureConstants.CLIENT_NAME, - "bigtable-java", - BuiltinMeasureConstants.TABLE, - TABLE_ID, - BuiltinMeasureConstants.ZONE, - ZONE, - BuiltinMeasureConstants.CLUSTER, - CLUSTER), - PROJECT_ID, - INSTANCE_ID, - APP_PROFILE_ID)) - .isEqualTo(connectivityErrorCount); - assertThat( - wrapper.getAggregationValueAsLong( - BuiltinViewConstants.THROTTLING_LATENCIES_VIEW, - ImmutableMap.of( - BuiltinMeasureConstants.METHOD, "Bigtable.MutateRow", - BuiltinMeasureConstants.TABLE, TABLE_ID, - BuiltinMeasureConstants.ZONE, ZONE, - BuiltinMeasureConstants.CLUSTER, CLUSTER, - BuiltinMeasureConstants.CLIENT_NAME, "bigtable-java"), - PROJECT_ID, - INSTANCE_ID, - APP_PROFILE_ID)) - .isEqualTo(throttlingLatency); - assertThat( - wrapper.getAggregationValueAsLong( - BuiltinViewConstants.FIRST_RESPONSE_LATENCIES_VIEW, - ImmutableMap.of( - BuiltinMeasureConstants.METHOD, - "Bigtable.MutateRow", - BuiltinMeasureConstants.TABLE, - TABLE_ID, - BuiltinMeasureConstants.ZONE, - ZONE, - BuiltinMeasureConstants.CLUSTER, - CLUSTER, - BuiltinMeasureConstants.STATUS, - "OK", - BuiltinMeasureConstants.CLIENT_NAME, - "bigtable-java"), - PROJECT_ID, - INSTANCE_ID, - APP_PROFILE_ID)) - .isEqualTo(firstResponseLatency); - } -} diff --git a/google-cloud-bigtable/pom.xml b/google-cloud-bigtable/pom.xml index e220c75cdda8..f0d992fdbeb0 100644 --- a/google-cloud-bigtable/pom.xml +++ b/google-cloud-bigtable/pom.xml @@ -54,11 +54,6 @@ pom import - - com.google.cloud - google-cloud-bigtable-stats - 2.6.3-SNAPSHOT - @@ -285,6 +280,10 @@ mockito-core test + + com.google.cloud + google-cloud-bigtable-stats + diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java index b57ecddd09be..88f2f554038b 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java @@ -90,7 +90,6 @@ import com.google.cloud.bigtable.data.v2.stub.readrows.ReadRowsUserCallable; import com.google.cloud.bigtable.data.v2.stub.readrows.RowMergingCallable; import com.google.cloud.bigtable.gaxx.retrying.ApiResultRetryAlgorithm; -import com.google.cloud.bigtable.stats.StatsWrapper; import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracer.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracer.java index 9560a5ee54ba..2148c674e3f8 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracer.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracer.java @@ -1,9 +1,5 @@ /* -<<<<<<< HEAD * Copyright 2022 Google LLC -======= - * Copyright 2021 Google LLC ->>>>>>> ca28be67 (feat: update tracers to use built in metrics) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerFactory.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerFactory.java index d02150c89ac4..794997071df9 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerFactory.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerFactory.java @@ -1,9 +1,5 @@ /* -<<<<<<< HEAD * Copyright 2022 Google LLC -======= - * Copyright 2021 Google LLC ->>>>>>> ca28be67 (feat: update tracers to use built in metrics) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/MetricsTracer.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/MetricsTracer.java index efb41998bcaf..3c63b1b5f795 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/MetricsTracer.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/MetricsTracer.java @@ -228,14 +228,4 @@ private TagContextBuilder newTagCtxBuilder() { return tagCtx; } - - @Override - public void setLocations(String zone, String cluster) { - // noop - } - - @Override - public void onRequest() { - // noop - } } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java index bbd53944b18a..3a7e0b4a3251 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java @@ -1,9 +1,5 @@ /* -<<<<<<< HEAD * Copyright 2022 Google LLC -======= - * Copyright 2021 Google LLC ->>>>>>> ca28be67 (feat: update tracers to use built in metrics) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -116,7 +112,7 @@ public class BuiltinMetricsTracerTest { public void setUp() throws Exception { fakeService = new FakeService(); - final AtomicInteger count = new AtomicInteger(0);gi + final AtomicInteger count = new AtomicInteger(0); // Add an interceptor to add server-timing in headers ServerInterceptor trailersInterceptor = new ServerInterceptor() { From 023cebd10cf2847e041c146e8fd06ec5c2c901c5 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Wed, 29 Jun 2022 17:03:05 -0400 Subject: [PATCH 06/18] remove unused code --- google-cloud-bigtable-stats/pom.xml | 1 + google-cloud-bigtable/pom.xml | 4 ---- .../cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java | 4 ---- pom.xml | 5 ----- 4 files changed, 1 insertion(+), 13 deletions(-) diff --git a/google-cloud-bigtable-stats/pom.xml b/google-cloud-bigtable-stats/pom.xml index eff1b05e1411..fa96d246b9d1 100644 --- a/google-cloud-bigtable-stats/pom.xml +++ b/google-cloud-bigtable-stats/pom.xml @@ -37,6 +37,7 @@ com.google.api api-common + io.opencensus opencensus-api diff --git a/google-cloud-bigtable/pom.xml b/google-cloud-bigtable/pom.xml index f0d992fdbeb0..8cd922d5f44c 100644 --- a/google-cloud-bigtable/pom.xml +++ b/google-cloud-bigtable/pom.xml @@ -280,10 +280,6 @@ mockito-core test - - com.google.cloud - google-cloud-bigtable-stats - diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java index 88f2f554038b..5749c87e04bf 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java @@ -377,11 +377,7 @@ public UnaryCallable createReadRowCallable(RowAdapter *
  • Upon receiving the response stream, it will merge the {@link * com.google.bigtable.v2.ReadRowsResponse.CellChunk}s in logical rows. The actual row * implementation can be configured by the {@code rowAdapter} parameter. -<<<<<<< HEAD - *
  • Add bigtable tracer for tracking bigtable specific metrics. -======= *
  • Add BigtableTracer callable for tracking Bigtable specific metrics ->>>>>>> ca28be67 (feat: update tracers to use built in metrics) *
  • Retry/resume on failure. *
  • Filter out marker rows. * diff --git a/pom.xml b/pom.xml index 6e42834013a7..5e6d006eb519 100644 --- a/pom.xml +++ b/pom.xml @@ -301,11 +301,6 @@ - - org.apache.maven.plugins - maven-shade-plugin - 3.3.0 - From 1fa3008f6ed4e11896579e42fe133da1f8013379 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Thu, 30 Jun 2022 14:28:23 -0400 Subject: [PATCH 07/18] clean up --- .../data/v2/stub/EnhancedBigtableStub.java | 2 +- .../metrics/BigtableTracerStreamingCallable.java | 6 ++++-- .../stub/metrics/BigtableTracerUnaryCallable.java | 5 ++++- pom.xml | 14 +++++--------- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java index 5749c87e04bf..ec237aabf73a 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java @@ -377,7 +377,7 @@ public UnaryCallable createReadRowCallable(RowAdapter *
  • Upon receiving the response stream, it will merge the {@link * com.google.bigtable.v2.ReadRowsResponse.CellChunk}s in logical rows. The actual row * implementation can be configured by the {@code rowAdapter} parameter. - *
  • Add BigtableTracer callable for tracking Bigtable specific metrics + *
  • Add bigtable tracer for tracking bigtable specific metrics. *
  • Retry/resume on failure. *
  • Filter out marker rows. * diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java index a564ea9d1bad..4a8929283202 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java @@ -32,12 +32,14 @@ /** * This callable will * - *

    -inject a {@link GrpcResponseMetadata} to access the headers and trailers returned by gRPC + *

    -Inject a {@link GrpcResponseMetadata} to access the headers returned by gRPC * methods upon completion. The {@link BigtableTracer} will process metrics that were injected in * the header/trailer and publish them to OpenCensus. If {@link GrpcResponseMetadata#getMetadata()} * returned null, it probably means that the request has never reached GFE, and it'll increment the * gfe_header_missing_counter in this case. * + *

    -This class will also access trailers from {@link GrpcResponseMetadata} to record zone and cluster ids. + * *

    -Call {@link BigtableTracer#onRequest(int)} to record the request events in a stream. * *

    This class is considered an internal implementation detail and not meant to be used by @@ -86,7 +88,7 @@ private class BigtableTracerResponseObserver implements ResponseObser @Override public void onStart(final StreamController controller) { - final TracedStreamController tracedController = new TracedStreamController(controller, tracer); + TracedStreamController tracedController = new TracedStreamController(controller, tracer); outerObserver.onStart(tracedController); } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java index e570df9552fb..097cd75acd5e 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java @@ -32,11 +32,14 @@ /** * This callable will: * - *

    - Inject a {@link GrpcResponseMetadata} to access the headers and trailers returned by gRPC + *

    - Inject a {@link GrpcResponseMetadata} to access the headers returned by gRPC * methods upon completion. The {@link BigtableTracer} will process metrics that were injected in * the header/trailer and publish them to OpenCensus. If {@link GrpcResponseMetadata#getMetadata()} * returned null, it probably means that the request has never reached GFE, and it'll increment the * gfe_header_missing_counter in this case. + * + *

    -This class will also access trailers from {@link GrpcResponseMetadata} to record zone and cluster ids. + * *

    This class is considered an internal implementation detail and not meant to be used by * applications. */ diff --git a/pom.xml b/pom.xml index 5e6d006eb519..f898b3952b4f 100644 --- a/pom.xml +++ b/pom.xml @@ -212,15 +212,6 @@ - - - - org.apache.maven.plugins - maven-shade-plugin - 3.2.4 - - - @@ -301,6 +292,11 @@ + + org.apache.maven.plugins + maven-shade-plugin + 3.3.0 + From 2a86c378534ac463f21c7353ee6993a7e6b7c5dc Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Thu, 30 Jun 2022 17:35:35 -0400 Subject: [PATCH 08/18] add integration test --- .../cloud/bigtable/stats/BuiltinViews.java | 2 +- .../BigtableTracerStreamingCallable.java | 8 +- .../metrics/BigtableTracerUnaryCallable.java | 8 +- .../data/v2/it/StreamingMetadataIT.java | 53 ++++++++++++ .../bigtable/data/v2/it/UnaryTrailerIT.java | 80 +++++++++++++++++++ 5 files changed, 142 insertions(+), 9 deletions(-) create mode 100644 google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetadataIT.java create mode 100644 google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/UnaryTrailerIT.java diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViews.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViews.java index b5cf180b5bf0..46f826dc5c67 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViews.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViews.java @@ -44,7 +44,7 @@ void registerPrivateViews(ViewManager viewManager) { } } - public void registerBigtableBuiltinViews() { + public static void registerBigtableBuiltinViews() { ViewManager viewManager = Stats.getViewManager(); for (View view : BIGTABLE_BUILTIN_VIEWS) { viewManager.registerView(view); diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java index 4a8929283202..8b7f76e1b81b 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java @@ -32,17 +32,17 @@ /** * This callable will * - *

    -Inject a {@link GrpcResponseMetadata} to access the headers returned by gRPC + *

  • -Inject a {@link GrpcResponseMetadata} to access the headers returned by gRPC * methods upon completion. The {@link BigtableTracer} will process metrics that were injected in * the header/trailer and publish them to OpenCensus. If {@link GrpcResponseMetadata#getMetadata()} * returned null, it probably means that the request has never reached GFE, and it'll increment the * gfe_header_missing_counter in this case. * - *

    -This class will also access trailers from {@link GrpcResponseMetadata} to record zone and cluster ids. + *

  • -This class will also access trailers from {@link GrpcResponseMetadata} to record zone and cluster ids. * - *

    -Call {@link BigtableTracer#onRequest(int)} to record the request events in a stream. + *

  • -Call {@link BigtableTracer#onRequest(int)} to record the request events in a stream. * - *

    This class is considered an internal implementation detail and not meant to be used by + *

  • This class is considered an internal implementation detail and not meant to be used by * applications. */ @InternalApi diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java index 097cd75acd5e..a5284d17e5f7 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java @@ -32,15 +32,15 @@ /** * This callable will: * - *

    - Inject a {@link GrpcResponseMetadata} to access the headers returned by gRPC + *

  • - Inject a {@link GrpcResponseMetadata} to access the headers returned by gRPC * methods upon completion. The {@link BigtableTracer} will process metrics that were injected in * the header/trailer and publish them to OpenCensus. If {@link GrpcResponseMetadata#getMetadata()} * returned null, it probably means that the request has never reached GFE, and it'll increment the * gfe_header_missing_counter in this case. * - *

    -This class will also access trailers from {@link GrpcResponseMetadata} to record zone and cluster ids. + *

  • -This class will also access trailers from {@link GrpcResponseMetadata} to record zone and cluster ids. * - *

    This class is considered an internal implementation detail and not meant to be used by + *

  • This class is considered an internal implementation detail and not meant to be used by * applications. */ @InternalApi @@ -103,7 +103,7 @@ public void onSuccess(ResponseT response) { try { byte[] trailers = responseMetadata - .getTrailingMetadata() + .getMetadata() .get(Metadata.Key.of(Util.TRAILER_KEY, Metadata.BINARY_BYTE_MARSHALLER)); ResponseParams decodedTrailers = ResponseParams.parseFrom(trailers); tracer.setLocations(decodedTrailers.getZoneId(), decodedTrailers.getClusterId()); diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetadataIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetadataIT.java new file mode 100644 index 000000000000..90748bba1626 --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetadataIT.java @@ -0,0 +1,53 @@ +package com.google.cloud.bigtable.data.v2.it; + +import com.google.api.core.ApiFuture; +import com.google.cloud.bigtable.data.v2.models.Query; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowCell; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import com.google.cloud.bigtable.test_helpers.env.TestEnvRule; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.protobuf.ByteString; +import org.junit.ClassRule; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +public class StreamingMetadataIT { + @ClassRule + public static TestEnvRule testEnvRule = new TestEnvRule(); + + public void test() { + String prefix = UUID.randomUUID().toString(); + int numRows = 5; + List expectedRows = Lists.newArrayList(); + String uniqueKey = prefix + "-read"; + + long timestampMicros = System.currentTimeMillis() * 1_000; + + for (int i = 0; i < numRows; i++) { + testEnvRule + .env() + .getDataClient() + .mutateRowCallable() + .call( + RowMutation.create(testEnvRule.env().getTableId(), uniqueKey + "-" + i) + .setCell(testEnvRule.env().getFamilyId(), "q", timestampMicros, "my-value")); + + expectedRows.add( + Row.create( + ByteString.copyFromUtf8(uniqueKey + "-" + i), + ImmutableList.of( + RowCell.create( + testEnvRule.env().getFamilyId(), + ByteString.copyFromUtf8("q"), + timestampMicros, + ImmutableList.of(), + ByteString.copyFromUtf8("my-value"))))); + } + + String tableId = testEnvRule.env().getTableId(); + } +} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/UnaryTrailerIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/UnaryTrailerIT.java new file mode 100644 index 000000000000..71810a06a5ce --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/UnaryTrailerIT.java @@ -0,0 +1,80 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.it; + +import com.google.api.core.ApiFuture; +import com.google.api.gax.grpc.GrpcResponseMetadata; +import com.google.cloud.bigtable.data.v2.models.Query; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import com.google.cloud.bigtable.stats.BuiltinViews; +import com.google.cloud.bigtable.test_helpers.env.TestEnvRule; +import com.google.common.collect.ImmutableList; +import com.google.protobuf.ByteString; +import io.grpc.Metadata; +import io.opencensus.stats.AggregationData; +import io.opencensus.stats.Stats; +import io.opencensus.stats.View; +import io.opencensus.stats.ViewData; +import io.opencensus.stats.ViewManager; +import io.opencensus.tags.TagKey; +import io.opencensus.tags.TagValue; +import org.junit.ClassRule; +import org.junit.Test; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import static com.google.common.truth.Truth.assertThat; + +public class UnaryTrailerIT { + @ClassRule + public static TestEnvRule testEnvRule = new TestEnvRule(); + + @Test + public void test() throws Exception { + BuiltinViews.registerBigtableBuiltinViews(); + + String rowKey = UUID.randomUUID().toString(); + String familyId = testEnvRule.env().getFamilyId(); + + + ApiFuture future = + testEnvRule + .env() + .getDataClient() + .mutateRowCallable() + .futureCall(RowMutation.create(testEnvRule.env().getTableId(), rowKey) + .setCell(familyId, "q", "myVal")); + + future.get(1, TimeUnit.MINUTES); + + Thread.sleep(1000); + ViewManager viewManager = Stats.getViewManager(); + ViewData viewData = viewManager.getView(View.Name.create("bigtable.googleapis.com/internal/client/operation_latencies")); + + int clusterIndex = viewData.getView().getColumns().indexOf(TagKey.create("cluster")); + int zoneIndex = viewData.getView().getColumns().indexOf(TagKey.create("zone")); + + List tagValues = viewData.getAggregationMap().entrySet().stream().findFirst().get().getKey(); + assertThat(tagValues.get(zoneIndex).asString()).isEqualTo(testEnvRule.env().getPrimaryZone()); + + assertThat(tagValues.get(clusterIndex).asString()).isEqualTo(testEnvRule.env().getPrimaryClusterId()); + } +} From 509ffdc568b2bf5ff33cc26807a94e2ae15d1141 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Fri, 1 Jul 2022 11:20:43 -0400 Subject: [PATCH 09/18] add integration tests and fix unit tests --- .../BigtableTracerStreamingCallable.java | 27 ++-- .../metrics/BigtableTracerUnaryCallable.java | 26 ++-- .../bigtable/data/v2/stub/metrics/Util.java | 2 +- .../data/v2/it/StreamingMetadataIT.java | 53 ------- .../v2/it/StreamingMetricsMetadataIT.java | 103 ++++++++++++++ .../data/v2/it/UnaryMetricsMetadataIT.java | 130 ++++++++++++++++++ .../bigtable/data/v2/it/UnaryTrailerIT.java | 80 ----------- .../metrics/BuiltinMetricsTracerTest.java | 50 ++----- 8 files changed, 267 insertions(+), 204 deletions(-) delete mode 100644 google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetadataIT.java create mode 100644 google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetricsMetadataIT.java create mode 100644 google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/UnaryMetricsMetadataIT.java delete mode 100644 google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/UnaryTrailerIT.java diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java index 8b7f76e1b81b..6f636bf55d5f 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java @@ -31,19 +31,16 @@ /** * This callable will - * - *
  • -Inject a {@link GrpcResponseMetadata} to access the headers returned by gRPC - * methods upon completion. The {@link BigtableTracer} will process metrics that were injected in - * the header/trailer and publish them to OpenCensus. If {@link GrpcResponseMetadata#getMetadata()} - * returned null, it probably means that the request has never reached GFE, and it'll increment the - * gfe_header_missing_counter in this case. - * - *
  • -This class will also access trailers from {@link GrpcResponseMetadata} to record zone and cluster ids. - * + *
  • -Inject a {@link GrpcResponseMetadata} to access the headers returned by gRPC methods upon + * completion. The {@link BigtableTracer} will process metrics that were injected in the + * header/trailer and publish them to OpenCensus. If {@link GrpcResponseMetadata#getMetadata()} + * returned null, it probably means that the request has never reached GFE, and it'll increment + * the gfe_header_missing_counter in this case. + *
  • -This class will also access trailers from {@link GrpcResponseMetadata} to record zone and + * cluster ids. *
  • -Call {@link BigtableTracer#onRequest(int)} to record the request events in a stream. - * *
  • This class is considered an internal implementation detail and not meant to be used by - * applications. + * applications. */ @InternalApi public class BigtableTracerStreamingCallable @@ -108,9 +105,7 @@ public void onError(Throwable t) { tracer.recordGfeMetadata(latency, t); try { byte[] trailers = - responseMetadata - .getTrailingMetadata() - .get(Metadata.Key.of(Util.TRAILER_KEY, Metadata.BINARY_BYTE_MARSHALLER)); + metadata.get(Metadata.Key.of(Util.RESPONSE_PRAMS_KEY, Metadata.BINARY_BYTE_MARSHALLER)); ResponseParams decodedTrailers = ResponseParams.parseFrom(trailers); tracer.setLocations(decodedTrailers.getZoneId(), decodedTrailers.getClusterId()); } catch (NullPointerException | InvalidProtocolBufferException e) { @@ -126,9 +121,7 @@ public void onComplete() { tracer.recordGfeMetadata(latency, null); try { byte[] trailers = - responseMetadata - .getTrailingMetadata() - .get(Metadata.Key.of(Util.TRAILER_KEY, Metadata.BINARY_BYTE_MARSHALLER)); + metadata.get(Metadata.Key.of(Util.RESPONSE_PRAMS_KEY, Metadata.BINARY_BYTE_MARSHALLER)); ResponseParams decodedTrailers = ResponseParams.parseFrom(trailers); tracer.setLocations(decodedTrailers.getZoneId(), decodedTrailers.getClusterId()); } catch (NullPointerException | InvalidProtocolBufferException e) { diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java index a5284d17e5f7..0efc99fe4dd2 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java @@ -31,17 +31,15 @@ /** * This callable will: - * - *
  • - Inject a {@link GrpcResponseMetadata} to access the headers returned by gRPC - * methods upon completion. The {@link BigtableTracer} will process metrics that were injected in - * the header/trailer and publish them to OpenCensus. If {@link GrpcResponseMetadata#getMetadata()} - * returned null, it probably means that the request has never reached GFE, and it'll increment the - * gfe_header_missing_counter in this case. - * - *
  • -This class will also access trailers from {@link GrpcResponseMetadata} to record zone and cluster ids. - * + *
  • - Inject a {@link GrpcResponseMetadata} to access the headers returned by gRPC methods upon + * completion. The {@link BigtableTracer} will process metrics that were injected in the + * header/trailer and publish them to OpenCensus. If {@link GrpcResponseMetadata#getMetadata()} + * returned null, it probably means that the request has never reached GFE, and it'll increment + * the gfe_header_missing_counter in this case. + *
  • -This class will also access trailers from {@link GrpcResponseMetadata} to record zone and + * cluster ids. *
  • This class is considered an internal implementation detail and not meant to be used by - * applications. + * applications. */ @InternalApi public class BigtableTracerUnaryCallable @@ -86,9 +84,7 @@ public void onFailure(Throwable throwable) { tracer.recordGfeMetadata(latency, throwable); try { byte[] trailers = - responseMetadata - .getTrailingMetadata() - .get(Metadata.Key.of(Util.TRAILER_KEY, Metadata.BINARY_BYTE_MARSHALLER)); + metadata.get(Metadata.Key.of(Util.RESPONSE_PRAMS_KEY, Metadata.BINARY_BYTE_MARSHALLER)); ResponseParams decodedTrailers = ResponseParams.parseFrom(trailers); tracer.setLocations(decodedTrailers.getZoneId(), decodedTrailers.getClusterId()); } catch (NullPointerException | InvalidProtocolBufferException e) { @@ -102,9 +98,7 @@ public void onSuccess(ResponseT response) { tracer.recordGfeMetadata(latency, null); try { byte[] trailers = - responseMetadata - .getMetadata() - .get(Metadata.Key.of(Util.TRAILER_KEY, Metadata.BINARY_BYTE_MARSHALLER)); + metadata.get(Metadata.Key.of(Util.RESPONSE_PRAMS_KEY, Metadata.BINARY_BYTE_MARSHALLER)); ResponseParams decodedTrailers = ResponseParams.parseFrom(trailers); tracer.setLocations(decodedTrailers.getZoneId(), decodedTrailers.getClusterId()); } catch (NullPointerException | InvalidProtocolBufferException e) { diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/Util.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/Util.java index 044002902719..7487703fc0cd 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/Util.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/Util.java @@ -58,7 +58,7 @@ public class Util { Metadata.Key.of("server-timing", Metadata.ASCII_STRING_MARSHALLER); private static final Pattern SERVER_TIMING_HEADER_PATTERN = Pattern.compile(".*dur=(?\\d+)"); - static final String TRAILER_KEY = "x-goog-ext-425905942-bin"; + static final String RESPONSE_PRAMS_KEY = "x-goog-ext-425905942-bin"; /** Convert an exception into a value that can be used to create an OpenCensus tag value. */ static String extractStatus(@Nullable Throwable error) { diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetadataIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetadataIT.java deleted file mode 100644 index 90748bba1626..000000000000 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetadataIT.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.google.cloud.bigtable.data.v2.it; - -import com.google.api.core.ApiFuture; -import com.google.cloud.bigtable.data.v2.models.Query; -import com.google.cloud.bigtable.data.v2.models.Row; -import com.google.cloud.bigtable.data.v2.models.RowCell; -import com.google.cloud.bigtable.data.v2.models.RowMutation; -import com.google.cloud.bigtable.test_helpers.env.TestEnvRule; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; -import com.google.protobuf.ByteString; -import org.junit.ClassRule; - -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; - -public class StreamingMetadataIT { - @ClassRule - public static TestEnvRule testEnvRule = new TestEnvRule(); - - public void test() { - String prefix = UUID.randomUUID().toString(); - int numRows = 5; - List expectedRows = Lists.newArrayList(); - String uniqueKey = prefix + "-read"; - - long timestampMicros = System.currentTimeMillis() * 1_000; - - for (int i = 0; i < numRows; i++) { - testEnvRule - .env() - .getDataClient() - .mutateRowCallable() - .call( - RowMutation.create(testEnvRule.env().getTableId(), uniqueKey + "-" + i) - .setCell(testEnvRule.env().getFamilyId(), "q", timestampMicros, "my-value")); - - expectedRows.add( - Row.create( - ByteString.copyFromUtf8(uniqueKey + "-" + i), - ImmutableList.of( - RowCell.create( - testEnvRule.env().getFamilyId(), - ByteString.copyFromUtf8("q"), - timestampMicros, - ImmutableList.of(), - ByteString.copyFromUtf8("my-value"))))); - } - - String tableId = testEnvRule.env().getTableId(); - } -} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetricsMetadataIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetricsMetadataIT.java new file mode 100644 index 000000000000..a71773dbbac3 --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetricsMetadataIT.java @@ -0,0 +1,103 @@ +package com.google.cloud.bigtable.data.v2.it; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.TruthJUnit.assume; + +import com.google.api.core.ApiFuture; +import com.google.api.gax.rpc.NotFoundException; +import com.google.cloud.bigtable.admin.v2.models.Cluster; +import com.google.cloud.bigtable.data.v2.models.Query; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.stats.BuiltinViews; +import com.google.cloud.bigtable.test_helpers.env.EmulatorEnv; +import com.google.cloud.bigtable.test_helpers.env.TestEnvRule; +import com.google.common.collect.Lists; +import io.opencensus.stats.Stats; +import io.opencensus.stats.View; +import io.opencensus.stats.ViewData; +import io.opencensus.stats.ViewManager; +import io.opencensus.tags.TagValue; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; + +public class StreamingMetricsMetadataIT { + @ClassRule public static TestEnvRule testEnvRule = new TestEnvRule(); + + @BeforeClass + public static void setUpClass() { + assume() + .withMessage("StreamingMetricsMetadataIT is not supported on Emulator") + .that(testEnvRule.env()) + .isNotInstanceOf(EmulatorEnv.class); + BuiltinViews.registerBigtableBuiltinViews(); + } + + @Test + public void testSuccess() throws Exception { + String prefix = UUID.randomUUID().toString(); + String uniqueKey = prefix + "-read"; + + Query query = Query.create(testEnvRule.env().getTableId()).rowKey(uniqueKey); + ArrayList rows = Lists.newArrayList(testEnvRule.env().getDataClient().readRows(query)); + + // give opencensus some time to populate view data + Thread.sleep(100); + + ViewManager viewManager = Stats.getViewManager(); + ViewData viewData = + viewManager.getView( + View.Name.create("bigtable.googleapis.com/internal/client/operation_latencies")); + + List tagValues = + viewData.getAggregationMap().entrySet().stream() + .map(Map.Entry::getKey) + .flatMap(x -> x.stream()) + .collect(Collectors.toCollection(ArrayList::new)); + + ApiFuture> clustersFuture = + testEnvRule + .env() + .getInstanceAdminClient() + .listClustersAsync(testEnvRule.env().getInstanceId()); + List clusters = clustersFuture.get(1, TimeUnit.MINUTES); + + assertThat(tagValues).contains(TagValue.create(clusters.get(0).getZone())); + assertThat(tagValues).contains(TagValue.create(clusters.get(0).getId())); + } + + @Test + public void testFailure() throws InterruptedException { + String prefix = UUID.randomUUID().toString(); + String uniqueKey = prefix + "-read"; + + Query query = Query.create("non-exist-table"); + try { + Lists.newArrayList(testEnvRule.env().getDataClient().readRows(query)); + } catch (NotFoundException e) { + } + + // give opencensus some time to populate view data + Thread.sleep(100); + + ViewManager viewManager = Stats.getViewManager(); + ViewData viewData = + viewManager.getView( + View.Name.create("bigtable.googleapis.com/internal/client/operation_latencies")); + + List tagValues = + viewData.getAggregationMap().entrySet().stream() + .map(Map.Entry::getKey) + .flatMap(x -> x.stream()) + .collect(Collectors.toCollection(ArrayList::new)); + + assertThat(tagValues).contains(TagValue.create("undefined")); + assertThat(tagValues).contains(TagValue.create("undefined")); + } +} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/UnaryMetricsMetadataIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/UnaryMetricsMetadataIT.java new file mode 100644 index 000000000000..1e2c90649b84 --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/UnaryMetricsMetadataIT.java @@ -0,0 +1,130 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.it; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.TruthJUnit.assume; + +import com.google.api.core.ApiFuture; +import com.google.api.gax.rpc.NotFoundException; +import com.google.cloud.bigtable.admin.v2.models.Cluster; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import com.google.cloud.bigtable.stats.BuiltinViews; +import com.google.cloud.bigtable.test_helpers.env.EmulatorEnv; +import com.google.cloud.bigtable.test_helpers.env.TestEnvRule; +import io.opencensus.stats.Stats; +import io.opencensus.stats.View; +import io.opencensus.stats.ViewData; +import io.opencensus.stats.ViewManager; +import io.opencensus.tags.TagValue; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Ignore; +import org.junit.Test; + +@Ignore +public class UnaryMetricsMetadataIT { + @ClassRule public static TestEnvRule testEnvRule = new TestEnvRule(); + + @BeforeClass + public static void setUpClass() { + assume() + .withMessage("UnaryMetricsMetadataIT is not supported on Emulator") + .that(testEnvRule.env()) + .isNotInstanceOf(EmulatorEnv.class); + + BuiltinViews.registerBigtableBuiltinViews(); + } + + @Test + public void testSuccess() throws Exception { + String rowKey = UUID.randomUUID().toString(); + String familyId = testEnvRule.env().getFamilyId(); + + ApiFuture future = + testEnvRule + .env() + .getDataClient() + .mutateRowCallable() + .futureCall( + RowMutation.create(testEnvRule.env().getTableId(), rowKey) + .setCell(familyId, "q", "myVal")); + + future.get(1, TimeUnit.MINUTES); + + // give opencensus some time to populate view data + Thread.sleep(100); + + ViewManager viewManager = Stats.getViewManager(); + ViewData viewData = + viewManager.getView( + View.Name.create("bigtable.googleapis.com/internal/client/operation_latencies")); + + List tagValues = + viewData.getAggregationMap().entrySet().stream() + .map(Map.Entry::getKey) + .flatMap(x -> x.stream()) + .collect(Collectors.toCollection(ArrayList::new)); + + ApiFuture> clustersFuture = + testEnvRule + .env() + .getInstanceAdminClient() + .listClustersAsync(testEnvRule.env().getInstanceId()); + List clusters = clustersFuture.get(1, TimeUnit.MINUTES); + + assertThat(tagValues).contains(TagValue.create(clusters.get(0).getZone())); + assertThat(tagValues).contains(TagValue.create(clusters.get(0).getId())); + } + + @Test + public void testFailure() throws InterruptedException { + String rowKey = UUID.randomUUID().toString(); + String familyId = testEnvRule.env().getFamilyId(); + + try { + testEnvRule + .env() + .getDataClient() + .mutateRowCallable() + .call(RowMutation.create("non-exist-table", rowKey).setCell(familyId, "q", "myVal")); + } catch (NotFoundException e) { + } + + // give opencensus some time to populate view data + Thread.sleep(100); + + ViewManager viewManager = Stats.getViewManager(); + ViewData viewData = + viewManager.getView( + View.Name.create("bigtable.googleapis.com/internal/client/operation_latencies")); + + List tagValues = + viewData.getAggregationMap().entrySet().stream() + .map(Map.Entry::getKey) + .flatMap(x -> x.stream()) + .collect(Collectors.toCollection(ArrayList::new)); + + assertThat(tagValues).contains(TagValue.create("undefined")); + assertThat(tagValues).contains(TagValue.create("undefined")); + } +} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/UnaryTrailerIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/UnaryTrailerIT.java deleted file mode 100644 index 71810a06a5ce..000000000000 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/UnaryTrailerIT.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.cloud.bigtable.data.v2.it; - -import com.google.api.core.ApiFuture; -import com.google.api.gax.grpc.GrpcResponseMetadata; -import com.google.cloud.bigtable.data.v2.models.Query; -import com.google.cloud.bigtable.data.v2.models.Row; -import com.google.cloud.bigtable.data.v2.models.RowMutation; -import com.google.cloud.bigtable.stats.BuiltinViews; -import com.google.cloud.bigtable.test_helpers.env.TestEnvRule; -import com.google.common.collect.ImmutableList; -import com.google.protobuf.ByteString; -import io.grpc.Metadata; -import io.opencensus.stats.AggregationData; -import io.opencensus.stats.Stats; -import io.opencensus.stats.View; -import io.opencensus.stats.ViewData; -import io.opencensus.stats.ViewManager; -import io.opencensus.tags.TagKey; -import io.opencensus.tags.TagValue; -import org.junit.ClassRule; -import org.junit.Test; - -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.TimeUnit; - -import static com.google.common.truth.Truth.assertThat; - -public class UnaryTrailerIT { - @ClassRule - public static TestEnvRule testEnvRule = new TestEnvRule(); - - @Test - public void test() throws Exception { - BuiltinViews.registerBigtableBuiltinViews(); - - String rowKey = UUID.randomUUID().toString(); - String familyId = testEnvRule.env().getFamilyId(); - - - ApiFuture future = - testEnvRule - .env() - .getDataClient() - .mutateRowCallable() - .futureCall(RowMutation.create(testEnvRule.env().getTableId(), rowKey) - .setCell(familyId, "q", "myVal")); - - future.get(1, TimeUnit.MINUTES); - - Thread.sleep(1000); - ViewManager viewManager = Stats.getViewManager(); - ViewData viewData = viewManager.getView(View.Name.create("bigtable.googleapis.com/internal/client/operation_latencies")); - - int clusterIndex = viewData.getView().getColumns().indexOf(TagKey.create("cluster")); - int zoneIndex = viewData.getView().getColumns().indexOf(TagKey.create("zone")); - - List tagValues = viewData.getAggregationMap().entrySet().stream().findFirst().get().getKey(); - assertThat(tagValues.get(zoneIndex).asString()).isEqualTo(testEnvRule.env().getPrimaryZone()); - - assertThat(tagValues.get(clusterIndex).asString()).isEqualTo(testEnvRule.env().getPrimaryClusterId()); - } -} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java index 3a7e0b4a3251..b9bd2a926c0e 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java @@ -85,17 +85,15 @@ public class BuiltinMetricsTracerTest { private static final String INSTANCE_ID = "fake-instance"; private static final String APP_PROFILE_ID = "default"; private static final String TABLE_ID = "fake-table"; - private static final String ZONE_0 = "us-west-1"; - private static final String CLUSTER_0 = "cluster-0"; - private static final String ZONE_1 = "us-east-1"; - private static final String CLUSTER_1 = "cluster-1"; + private static final String ZONE = "us-west-1"; + private static final String CLUSTER = "cluster-0"; private static final long FAKE_SERVER_TIMING = 50; private static final long SERVER_LATENCY = 100; private static final long APPLICATION_LATENCY = 200; @Rule public final MockitoRule mockitoRule = MockitoJUnit.rule(); - private FakeService fakeService; + private final FakeService fakeService = new FakeService(); private Server server; private EnhancedBigtableStub stub; @@ -110,9 +108,6 @@ public class BuiltinMetricsTracerTest { @Before public void setUp() throws Exception { - fakeService = new FakeService(); - - final AtomicInteger count = new AtomicInteger(0); // Add an interceptor to add server-timing in headers ServerInterceptor trailersInterceptor = new ServerInterceptor() { @@ -128,34 +123,15 @@ public void sendHeaders(Metadata headers) { headers.put( Metadata.Key.of("server-timing", Metadata.ASCII_STRING_MARSHALLER), String.format("gfet4t7; dur=%d", FAKE_SERVER_TIMING)); - super.sendHeaders(headers); - } - @Override - public void close(Status status, Metadata trailers) { - int currentCount = count.getAndIncrement(); - if (currentCount == 0) { - ResponseParams params = - ResponseParams.newBuilder() - .setZoneId(ZONE_0) - .setClusterId(CLUSTER_0) - .build(); - byte[] byteArray = params.toByteArray(); - trailers.put( - Metadata.Key.of(Util.TRAILER_KEY, Metadata.BINARY_BYTE_MARSHALLER), - byteArray); - } else { - ResponseParams params = - ResponseParams.newBuilder() - .setClusterId(CLUSTER_1) - .setZoneId(ZONE_1) - .build(); - byte[] byteArray = params.toByteArray(); - trailers.put( - Metadata.Key.of(Util.TRAILER_KEY, Metadata.BINARY_BYTE_MARSHALLER), - byteArray); - } - super.close(status, trailers); + ResponseParams params = + ResponseParams.newBuilder().setZoneId(ZONE).setClusterId(CLUSTER).build(); + byte[] byteArray = params.toByteArray(); + headers.put( + Metadata.Key.of(Util.RESPONSE_PRAMS_KEY, Metadata.BINARY_BYTE_MARSHALLER), + byteArray); + + super.sendHeaders(headers); } }, metadata); @@ -365,8 +341,8 @@ public void testMutateRowAttempts() { // and when the record() is called in onOperationCompletion(). verify(statsRecorderWrapper, timeout(50).times(fakeService.getAttemptCounter().get() + 1)) .record(status.capture(), tableId.capture(), zone.capture(), cluster.capture()); - assertThat(zone.getAllValues()).containsExactly(ZONE_0, ZONE_1, ZONE_1, ZONE_1); - assertThat(cluster.getAllValues()).containsExactly(CLUSTER_0, CLUSTER_1, CLUSTER_1, CLUSTER_1); + assertThat(zone.getAllValues()).containsExactly("undefined", "undefined", ZONE, ZONE); + assertThat(cluster.getAllValues()).containsExactly("undefined", "undefined", CLUSTER, CLUSTER); assertThat(status.getAllValues()).containsExactly("UNAVAILABLE", "UNAVAILABLE", "OK", "OK"); } From 544b35d46960961acc7bd16bc5e5321e1faf8473 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Thu, 7 Jul 2022 17:33:13 -0400 Subject: [PATCH 10/18] revert proto changes --- .../src/main/java/com/google/bigtable/v2/ResponseParams.java | 5 +---- .../java/com/google/bigtable/v2/ResponseParamsOrBuilder.java | 2 +- .../java/com/google/bigtable/v2/ResponseParamsProto.java | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ResponseParams.java b/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ResponseParams.java index 5d1c9c7b7be3..c64592a2b422 100644 --- a/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ResponseParams.java +++ b/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ResponseParams.java @@ -1,5 +1,5 @@ /* - * Copyright 2022 Google LLC + * Copyright 2020 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -438,12 +438,9 @@ protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.Build * * 
        * Response metadata proto
-<<<<<<< HEAD
    * This is an experimental feature that will be used to get zone_id and
    * cluster_id from response trailers to tag the metrics. This should not be
    * used by customers directly
-=======
->>>>>>> 880e0a1c (feat: add response protos)
    *
    * * Protobuf type {@code google.bigtable.v2.ResponseParams} diff --git a/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ResponseParamsOrBuilder.java b/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ResponseParamsOrBuilder.java index 7e12c30eca3a..118eef7410d8 100644 --- a/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ResponseParamsOrBuilder.java +++ b/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ResponseParamsOrBuilder.java @@ -1,5 +1,5 @@ /* - * Copyright 2022 Google LLC + * Copyright 2020 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ResponseParamsProto.java b/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ResponseParamsProto.java index ab96c7034e6a..9bfba6440505 100644 --- a/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ResponseParamsProto.java +++ b/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ResponseParamsProto.java @@ -1,5 +1,5 @@ /* - * Copyright 2022 Google LLC + * Copyright 2020 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From 8c78b210ba0c0f73950d71cc58c7e6b6579e6559 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Thu, 7 Jul 2022 17:50:55 -0400 Subject: [PATCH 11/18] fix license --- .../data/v2/it/StreamingMetricsMetadataIT.java | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetricsMetadataIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetricsMetadataIT.java index a71773dbbac3..99d9ff1b52cd 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetricsMetadataIT.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetricsMetadataIT.java @@ -1,3 +1,18 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.google.cloud.bigtable.data.v2.it; import static com.google.common.truth.Truth.assertThat; @@ -74,9 +89,6 @@ public void testSuccess() throws Exception { @Test public void testFailure() throws InterruptedException { - String prefix = UUID.randomUUID().toString(); - String uniqueKey = prefix + "-read"; - Query query = Query.create("non-exist-table"); try { Lists.newArrayList(testEnvRule.env().getDataClient().readRows(query)); From 7ec4c8c28858c4f515ca9860220efb6b861695c1 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Fri, 8 Jul 2022 13:32:45 -0400 Subject: [PATCH 12/18] add debug log --- .../bigtable/data/v2/it/StreamingMetricsMetadataIT.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetricsMetadataIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetricsMetadataIT.java index 99d9ff1b52cd..08048ab79c1a 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetricsMetadataIT.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetricsMetadataIT.java @@ -66,6 +66,11 @@ public void testSuccess() throws Exception { Thread.sleep(100); ViewManager viewManager = Stats.getViewManager(); + + System.out.println("exported views: " + viewManager.getAllExportedViews()); + + System.out.println("view data: " + viewManager.getView(View.Name.create("bigtable.googleapis.com/internal/client/operation_latencies"))); + ViewData viewData = viewManager.getView( View.Name.create("bigtable.googleapis.com/internal/client/operation_latencies")); From 8363b0a0fd9587ab5ddbc1dfa2950f5fd9d273b4 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Fri, 8 Jul 2022 14:28:49 -0400 Subject: [PATCH 13/18] test --- .../cloud/bigtable/stats/BuiltinViews.java | 22 +++++- .../v2/it/StreamingMetricsMetadataIT.java | 61 ++++++++------- .../data/v2/it/UnaryMetricsMetadataIT.java | 75 ++++++++++--------- 3 files changed, 91 insertions(+), 67 deletions(-) diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViews.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViews.java index 46f826dc5c67..0b7f0409914e 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViews.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViews.java @@ -20,11 +20,19 @@ import com.google.common.collect.ImmutableSet; import io.opencensus.stats.Stats; import io.opencensus.stats.View; +import io.opencensus.stats.ViewData; import io.opencensus.stats.ViewManager; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + /** For registering built-in metric views */ @InternalApi("For internal use only") public class BuiltinViews { + ViewManager viewManager = Stats.getViewManager(); + @VisibleForTesting static final ImmutableSet BIGTABLE_BUILTIN_VIEWS = ImmutableSet.of( @@ -44,10 +52,20 @@ void registerPrivateViews(ViewManager viewManager) { } } - public static void registerBigtableBuiltinViews() { - ViewManager viewManager = Stats.getViewManager(); + public void registerBigtableBuiltinViews() { for (View view : BIGTABLE_BUILTIN_VIEWS) { viewManager.registerView(view); } } + + @VisibleForTesting + public List getTagValueString() { + return viewManager.getView(BuiltinViewConstants.OPERATION_LATENCIES_VIEW.getName()) + .getAggregationMap() + .entrySet().stream() + .map(Map.Entry::getKey) + .flatMap(x -> x.stream()) + .map(x -> x.toString()) + .collect(Collectors.toCollection(ArrayList::new)); + } } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetricsMetadataIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetricsMetadataIT.java index 08048ab79c1a..cb0c3dda28f8 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetricsMetadataIT.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetricsMetadataIT.java @@ -45,13 +45,16 @@ public class StreamingMetricsMetadataIT { @ClassRule public static TestEnvRule testEnvRule = new TestEnvRule(); + static BuiltinViews builtinViews = new BuiltinViews(); + + @BeforeClass public static void setUpClass() { assume() .withMessage("StreamingMetricsMetadataIT is not supported on Emulator") .that(testEnvRule.env()) .isNotInstanceOf(EmulatorEnv.class); - BuiltinViews.registerBigtableBuiltinViews(); + builtinViews.registerBigtableBuiltinViews(); } @Test @@ -65,21 +68,21 @@ public void testSuccess() throws Exception { // give opencensus some time to populate view data Thread.sleep(100); - ViewManager viewManager = Stats.getViewManager(); +// ViewManager viewManager = Stats.getViewManager(); + System.out.println("exported views: " + builtinViews.getTagValueString()); - System.out.println("exported views: " + viewManager.getAllExportedViews()); - System.out.println("view data: " + viewManager.getView(View.Name.create("bigtable.googleapis.com/internal/client/operation_latencies"))); +// ViewData viewData = +// viewManager.getView( +// View.Name.create("bigtable.googleapis.com/internal/client/operation_latencies")); - ViewData viewData = - viewManager.getView( - View.Name.create("bigtable.googleapis.com/internal/client/operation_latencies")); +// List tagValues = +// viewData.getAggregationMap().entrySet().stream() +// .map(Map.Entry::getKey) +// .flatMap(x -> x.stream()) +// .collect(Collectors.toCollection(ArrayList::new)); - List tagValues = - viewData.getAggregationMap().entrySet().stream() - .map(Map.Entry::getKey) - .flatMap(x -> x.stream()) - .collect(Collectors.toCollection(ArrayList::new)); + builtinViews.getTagValueString(); ApiFuture> clustersFuture = testEnvRule @@ -88,8 +91,10 @@ public void testSuccess() throws Exception { .listClustersAsync(testEnvRule.env().getInstanceId()); List clusters = clustersFuture.get(1, TimeUnit.MINUTES); - assertThat(tagValues).contains(TagValue.create(clusters.get(0).getZone())); - assertThat(tagValues).contains(TagValue.create(clusters.get(0).getId())); +// assertThat(builtinViews.getTagValueString()).contains(TagValue.create(clusters.get(0).getZone())); +// assertThat(builtinViews.getTagValueString()).contains(TagValue.create(clusters.get(0).getId())); + assertThat(builtinViews.getTagValueString()).contains(clusters.get(0).getZone()); + assertThat(builtinViews.getTagValueString()).contains(clusters.get(0).getId()); } @Test @@ -102,19 +107,19 @@ public void testFailure() throws InterruptedException { // give opencensus some time to populate view data Thread.sleep(100); - - ViewManager viewManager = Stats.getViewManager(); - ViewData viewData = - viewManager.getView( - View.Name.create("bigtable.googleapis.com/internal/client/operation_latencies")); - - List tagValues = - viewData.getAggregationMap().entrySet().stream() - .map(Map.Entry::getKey) - .flatMap(x -> x.stream()) - .collect(Collectors.toCollection(ArrayList::new)); - - assertThat(tagValues).contains(TagValue.create("undefined")); - assertThat(tagValues).contains(TagValue.create("undefined")); +// +// ViewManager viewManager = Stats.getViewManager(); +// ViewData viewData = +// viewManager.getView( +// View.Name.create("bigtable.googleapis.com/internal/client/operation_latencies")); +// +// List tagValues = +// viewData.getAggregationMap().entrySet().stream() +// .map(Map.Entry::getKey) +// .flatMap(x -> x.stream()) +// .collect(Collectors.toCollection(ArrayList::new)); +// +// assertThat(tagValues).contains(TagValue.create("undefined")); +// assertThat(tagValues).contains(TagValue.create("undefined")); } } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/UnaryMetricsMetadataIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/UnaryMetricsMetadataIT.java index 1e2c90649b84..b2239e1a21ad 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/UnaryMetricsMetadataIT.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/UnaryMetricsMetadataIT.java @@ -52,7 +52,8 @@ public static void setUpClass() { .that(testEnvRule.env()) .isNotInstanceOf(EmulatorEnv.class); - BuiltinViews.registerBigtableBuiltinViews(); + BuiltinViews builtinViews = new BuiltinViews(); + builtinViews.registerBigtableBuiltinViews(); } @Test @@ -74,26 +75,26 @@ public void testSuccess() throws Exception { // give opencensus some time to populate view data Thread.sleep(100); - ViewManager viewManager = Stats.getViewManager(); - ViewData viewData = - viewManager.getView( - View.Name.create("bigtable.googleapis.com/internal/client/operation_latencies")); - - List tagValues = - viewData.getAggregationMap().entrySet().stream() - .map(Map.Entry::getKey) - .flatMap(x -> x.stream()) - .collect(Collectors.toCollection(ArrayList::new)); - - ApiFuture> clustersFuture = - testEnvRule - .env() - .getInstanceAdminClient() - .listClustersAsync(testEnvRule.env().getInstanceId()); - List clusters = clustersFuture.get(1, TimeUnit.MINUTES); - - assertThat(tagValues).contains(TagValue.create(clusters.get(0).getZone())); - assertThat(tagValues).contains(TagValue.create(clusters.get(0).getId())); +// ViewManager viewManager = Stats.getViewManager(); +// ViewData viewData = +// viewManager.getView( +// View.Name.create("bigtable.googleapis.com/internal/client/operation_latencies")); +// +// List tagValues = +// viewData.getAggregationMap().entrySet().stream() +// .map(Map.Entry::getKey) +// .flatMap(x -> x.stream()) +// .collect(Collectors.toCollection(ArrayList::new)); +// +// ApiFuture> clustersFuture = +// testEnvRule +// .env() +// .getInstanceAdminClient() +// .listClustersAsync(testEnvRule.env().getInstanceId()); +// List clusters = clustersFuture.get(1, TimeUnit.MINUTES); +// +// assertThat(tagValues).contains(TagValue.create(clusters.get(0).getZone())); +// assertThat(tagValues).contains(TagValue.create(clusters.get(0).getId())); } @Test @@ -110,21 +111,21 @@ public void testFailure() throws InterruptedException { } catch (NotFoundException e) { } - // give opencensus some time to populate view data - Thread.sleep(100); - - ViewManager viewManager = Stats.getViewManager(); - ViewData viewData = - viewManager.getView( - View.Name.create("bigtable.googleapis.com/internal/client/operation_latencies")); - - List tagValues = - viewData.getAggregationMap().entrySet().stream() - .map(Map.Entry::getKey) - .flatMap(x -> x.stream()) - .collect(Collectors.toCollection(ArrayList::new)); - - assertThat(tagValues).contains(TagValue.create("undefined")); - assertThat(tagValues).contains(TagValue.create("undefined")); +// // give opencensus some time to populate view data +// Thread.sleep(100); +// +// ViewManager viewManager = Stats.getViewManager(); +// ViewData viewData = +// viewManager.getView( +// View.Name.create("bigtable.googleapis.com/internal/client/operation_latencies")); +// +// List tagValues = +// viewData.getAggregationMap().entrySet().stream() +// .map(Map.Entry::getKey) +// .flatMap(x -> x.stream()) +// .collect(Collectors.toCollection(ArrayList::new)); +// +// assertThat(tagValues).contains(TagValue.create("undefined")); +// assertThat(tagValues).contains(TagValue.create("undefined")); } } From 919dbc5960ec5d51ea1049ae6b7481040fae46b7 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Fri, 8 Jul 2022 15:44:13 -0400 Subject: [PATCH 14/18] update --- google-cloud-bigtable-stats/pom.xml | 1 - .../cloud/bigtable/stats/BuiltinViews.java | 15 +------ .../cloud/bigtable/stats/StatsWrapper.java | 16 ++++++++ .../v2/it/StreamingMetricsMetadataIT.java | 41 +++++++++---------- 4 files changed, 37 insertions(+), 36 deletions(-) diff --git a/google-cloud-bigtable-stats/pom.xml b/google-cloud-bigtable-stats/pom.xml index fa96d246b9d1..6e9bc091cacf 100644 --- a/google-cloud-bigtable-stats/pom.xml +++ b/google-cloud-bigtable-stats/pom.xml @@ -50,7 +50,6 @@ io.opencensus opencensus-impl - test com.google.truth diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViews.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViews.java index 0b7f0409914e..f3a5475c38f5 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViews.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViews.java @@ -31,7 +31,6 @@ /** For registering built-in metric views */ @InternalApi("For internal use only") public class BuiltinViews { - ViewManager viewManager = Stats.getViewManager(); @VisibleForTesting static final ImmutableSet BIGTABLE_BUILTIN_VIEWS = @@ -52,20 +51,10 @@ void registerPrivateViews(ViewManager viewManager) { } } - public void registerBigtableBuiltinViews() { + public static void registerBigtableBuiltinViews() { + ViewManager viewManager = Stats.getViewManager(); for (View view : BIGTABLE_BUILTIN_VIEWS) { viewManager.registerView(view); } } - - @VisibleForTesting - public List getTagValueString() { - return viewManager.getView(BuiltinViewConstants.OPERATION_LATENCIES_VIEW.getName()) - .getAggregationMap() - .entrySet().stream() - .map(Map.Entry::getKey) - .flatMap(x -> x.stream()) - .map(x -> x.toString()) - .collect(Collectors.toCollection(ArrayList::new)); - } } diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/StatsWrapper.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/StatsWrapper.java index 6b7bd6df9bce..1c566297bfb0 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/StatsWrapper.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/StatsWrapper.java @@ -19,8 +19,13 @@ import com.google.api.core.InternalApi; import com.google.api.gax.tracing.SpanName; +import com.google.common.annotations.VisibleForTesting; import io.opencensus.stats.Stats; + +import java.util.ArrayList; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * Wrapper class for accessing opencensus. We use a shaded version of opencensus to avoid polluting @@ -34,4 +39,15 @@ public static StatsRecorderWrapper createRecorder( return new StatsRecorderWrapper( operationType, spanName, statsAttributes, Stats.getStatsRecorder()); } + + @VisibleForTesting + public static List getTagValueString() { + return Stats.getViewManager().getView(BuiltinViewConstants.OPERATION_LATENCIES_VIEW.getName()) + .getAggregationMap() + .entrySet().stream() + .map(Map.Entry::getKey) + .flatMap(x -> x.stream()) + .map(x -> x.asString()) + .collect(Collectors.toCollection(ArrayList::new)); + } } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetricsMetadataIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetricsMetadataIT.java index cb0c3dda28f8..75098988487a 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetricsMetadataIT.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetricsMetadataIT.java @@ -24,6 +24,7 @@ import com.google.cloud.bigtable.data.v2.models.Query; import com.google.cloud.bigtable.data.v2.models.Row; import com.google.cloud.bigtable.stats.BuiltinViews; +import com.google.cloud.bigtable.stats.StatsWrapper; import com.google.cloud.bigtable.test_helpers.env.EmulatorEnv; import com.google.cloud.bigtable.test_helpers.env.TestEnvRule; import com.google.common.collect.Lists; @@ -45,16 +46,13 @@ public class StreamingMetricsMetadataIT { @ClassRule public static TestEnvRule testEnvRule = new TestEnvRule(); - static BuiltinViews builtinViews = new BuiltinViews(); - - @BeforeClass public static void setUpClass() { assume() .withMessage("StreamingMetricsMetadataIT is not supported on Emulator") .that(testEnvRule.env()) .isNotInstanceOf(EmulatorEnv.class); - builtinViews.registerBigtableBuiltinViews(); + BuiltinViews.registerBigtableBuiltinViews(); } @Test @@ -68,8 +66,7 @@ public void testSuccess() throws Exception { // give opencensus some time to populate view data Thread.sleep(100); -// ViewManager viewManager = Stats.getViewManager(); - System.out.println("exported views: " + builtinViews.getTagValueString()); + System.out.println("exported views: " + StatsWrapper.getTagValueString()); // ViewData viewData = @@ -82,7 +79,7 @@ public void testSuccess() throws Exception { // .flatMap(x -> x.stream()) // .collect(Collectors.toCollection(ArrayList::new)); - builtinViews.getTagValueString(); +// builtinViews.getTagValueString(); ApiFuture> clustersFuture = testEnvRule @@ -93,8 +90,8 @@ public void testSuccess() throws Exception { // assertThat(builtinViews.getTagValueString()).contains(TagValue.create(clusters.get(0).getZone())); // assertThat(builtinViews.getTagValueString()).contains(TagValue.create(clusters.get(0).getId())); - assertThat(builtinViews.getTagValueString()).contains(clusters.get(0).getZone()); - assertThat(builtinViews.getTagValueString()).contains(clusters.get(0).getId()); + assertThat(StatsWrapper.getTagValueString()).contains(clusters.get(0).getZone()); + assertThat(StatsWrapper.getTagValueString()).contains(clusters.get(0).getId()); } @Test @@ -108,18 +105,18 @@ public void testFailure() throws InterruptedException { // give opencensus some time to populate view data Thread.sleep(100); // -// ViewManager viewManager = Stats.getViewManager(); -// ViewData viewData = -// viewManager.getView( -// View.Name.create("bigtable.googleapis.com/internal/client/operation_latencies")); -// -// List tagValues = -// viewData.getAggregationMap().entrySet().stream() -// .map(Map.Entry::getKey) -// .flatMap(x -> x.stream()) -// .collect(Collectors.toCollection(ArrayList::new)); -// -// assertThat(tagValues).contains(TagValue.create("undefined")); -// assertThat(tagValues).contains(TagValue.create("undefined")); + ViewManager viewManager = Stats.getViewManager(); + ViewData viewData = + viewManager.getView( + View.Name.create("bigtable.googleapis.com/internal/client/operation_latencies")); + + List tagValues = + viewData.getAggregationMap().entrySet().stream() + .map(Map.Entry::getKey) + .flatMap(x -> x.stream()) + .collect(Collectors.toCollection(ArrayList::new)); + + assertThat(tagValues).contains(TagValue.create("undefined")); + assertThat(tagValues).contains(TagValue.create("undefined")); } } From e6b0d8d9a79a0d68c0ff26172f7b546052cda3ea Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Fri, 8 Jul 2022 16:31:21 -0400 Subject: [PATCH 15/18] fix integration tests --- .../cloud/bigtable/stats/BuiltinViews.java | 6 -- .../cloud/bigtable/stats/StatsWrapper.java | 17 ++--- .../v2/it/StreamingMetricsMetadataIT.java | 52 +++----------- .../data/v2/it/UnaryMetricsMetadataIT.java | 67 +++++-------------- 4 files changed, 37 insertions(+), 105 deletions(-) diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViews.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViews.java index f3a5475c38f5..e62117cb7e07 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViews.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViews.java @@ -20,14 +20,8 @@ import com.google.common.collect.ImmutableSet; import io.opencensus.stats.Stats; import io.opencensus.stats.View; -import io.opencensus.stats.ViewData; import io.opencensus.stats.ViewManager; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - /** For registering built-in metric views */ @InternalApi("For internal use only") public class BuiltinViews { diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/StatsWrapper.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/StatsWrapper.java index 1c566297bfb0..3e06f5e18510 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/StatsWrapper.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/StatsWrapper.java @@ -21,7 +21,6 @@ import com.google.api.gax.tracing.SpanName; import com.google.common.annotations.VisibleForTesting; import io.opencensus.stats.Stats; - import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -40,14 +39,16 @@ public static StatsRecorderWrapper createRecorder( operationType, spanName, statsAttributes, Stats.getStatsRecorder()); } + // This is used in integration tests to get the tag value strings from view manager because Stats + // is relocated to + // com.google.bigtable.veneer.repackaged.io.opencensus. @VisibleForTesting - public static List getTagValueString() { + public static List getOperationLatencyViewTagValueStrings() { return Stats.getViewManager().getView(BuiltinViewConstants.OPERATION_LATENCIES_VIEW.getName()) - .getAggregationMap() - .entrySet().stream() - .map(Map.Entry::getKey) - .flatMap(x -> x.stream()) - .map(x -> x.asString()) - .collect(Collectors.toCollection(ArrayList::new)); + .getAggregationMap().entrySet().stream() + .map(Map.Entry::getKey) + .flatMap(x -> x.stream()) + .map(x -> x.asString()) + .collect(Collectors.toCollection(ArrayList::new)); } } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetricsMetadataIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetricsMetadataIT.java index 75098988487a..2f9c8ff639f3 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetricsMetadataIT.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetricsMetadataIT.java @@ -28,17 +28,10 @@ import com.google.cloud.bigtable.test_helpers.env.EmulatorEnv; import com.google.cloud.bigtable.test_helpers.env.TestEnvRule; import com.google.common.collect.Lists; -import io.opencensus.stats.Stats; -import io.opencensus.stats.View; -import io.opencensus.stats.ViewData; -import io.opencensus.stats.ViewManager; -import io.opencensus.tags.TagValue; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.UUID; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; @@ -63,35 +56,20 @@ public void testSuccess() throws Exception { Query query = Query.create(testEnvRule.env().getTableId()).rowKey(uniqueKey); ArrayList rows = Lists.newArrayList(testEnvRule.env().getDataClient().readRows(query)); - // give opencensus some time to populate view data - Thread.sleep(100); - - System.out.println("exported views: " + StatsWrapper.getTagValueString()); - - -// ViewData viewData = -// viewManager.getView( -// View.Name.create("bigtable.googleapis.com/internal/client/operation_latencies")); - -// List tagValues = -// viewData.getAggregationMap().entrySet().stream() -// .map(Map.Entry::getKey) -// .flatMap(x -> x.stream()) -// .collect(Collectors.toCollection(ArrayList::new)); - -// builtinViews.getTagValueString(); - ApiFuture> clustersFuture = testEnvRule .env() .getInstanceAdminClient() .listClustersAsync(testEnvRule.env().getInstanceId()); + List clusters = clustersFuture.get(1, TimeUnit.MINUTES); -// assertThat(builtinViews.getTagValueString()).contains(TagValue.create(clusters.get(0).getZone())); -// assertThat(builtinViews.getTagValueString()).contains(TagValue.create(clusters.get(0).getId())); - assertThat(StatsWrapper.getTagValueString()).contains(clusters.get(0).getZone()); - assertThat(StatsWrapper.getTagValueString()).contains(clusters.get(0).getId()); + // give opencensus some time to populate view data + Thread.sleep(100); + + List tagValueStrings = StatsWrapper.getOperationLatencyViewTagValueStrings(); + assertThat(tagValueStrings).contains(clusters.get(0).getZone()); + assertThat(tagValueStrings).contains(clusters.get(0).getId()); } @Test @@ -104,19 +82,9 @@ public void testFailure() throws InterruptedException { // give opencensus some time to populate view data Thread.sleep(100); -// - ViewManager viewManager = Stats.getViewManager(); - ViewData viewData = - viewManager.getView( - View.Name.create("bigtable.googleapis.com/internal/client/operation_latencies")); - - List tagValues = - viewData.getAggregationMap().entrySet().stream() - .map(Map.Entry::getKey) - .flatMap(x -> x.stream()) - .collect(Collectors.toCollection(ArrayList::new)); - assertThat(tagValues).contains(TagValue.create("undefined")); - assertThat(tagValues).contains(TagValue.create("undefined")); + List tagValueStrings = StatsWrapper.getOperationLatencyViewTagValueStrings(); + assertThat(tagValueStrings).contains("undefined"); + assertThat(tagValueStrings).contains("undefined"); } } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/UnaryMetricsMetadataIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/UnaryMetricsMetadataIT.java index b2239e1a21ad..9fd132ed899a 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/UnaryMetricsMetadataIT.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/UnaryMetricsMetadataIT.java @@ -23,25 +23,16 @@ import com.google.cloud.bigtable.admin.v2.models.Cluster; import com.google.cloud.bigtable.data.v2.models.RowMutation; import com.google.cloud.bigtable.stats.BuiltinViews; +import com.google.cloud.bigtable.stats.StatsWrapper; import com.google.cloud.bigtable.test_helpers.env.EmulatorEnv; import com.google.cloud.bigtable.test_helpers.env.TestEnvRule; -import io.opencensus.stats.Stats; -import io.opencensus.stats.View; -import io.opencensus.stats.ViewData; -import io.opencensus.stats.ViewManager; -import io.opencensus.tags.TagValue; -import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.UUID; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; import org.junit.BeforeClass; import org.junit.ClassRule; -import org.junit.Ignore; import org.junit.Test; -@Ignore public class UnaryMetricsMetadataIT { @ClassRule public static TestEnvRule testEnvRule = new TestEnvRule(); @@ -51,9 +42,7 @@ public static void setUpClass() { .withMessage("UnaryMetricsMetadataIT is not supported on Emulator") .that(testEnvRule.env()) .isNotInstanceOf(EmulatorEnv.class); - - BuiltinViews builtinViews = new BuiltinViews(); - builtinViews.registerBigtableBuiltinViews(); + BuiltinViews.registerBigtableBuiltinViews(); } @Test @@ -72,29 +61,19 @@ public void testSuccess() throws Exception { future.get(1, TimeUnit.MINUTES); + ApiFuture> clustersFuture = + testEnvRule + .env() + .getInstanceAdminClient() + .listClustersAsync(testEnvRule.env().getInstanceId()); + List clusters = clustersFuture.get(1, TimeUnit.MINUTES); + // give opencensus some time to populate view data Thread.sleep(100); -// ViewManager viewManager = Stats.getViewManager(); -// ViewData viewData = -// viewManager.getView( -// View.Name.create("bigtable.googleapis.com/internal/client/operation_latencies")); -// -// List tagValues = -// viewData.getAggregationMap().entrySet().stream() -// .map(Map.Entry::getKey) -// .flatMap(x -> x.stream()) -// .collect(Collectors.toCollection(ArrayList::new)); -// -// ApiFuture> clustersFuture = -// testEnvRule -// .env() -// .getInstanceAdminClient() -// .listClustersAsync(testEnvRule.env().getInstanceId()); -// List clusters = clustersFuture.get(1, TimeUnit.MINUTES); -// -// assertThat(tagValues).contains(TagValue.create(clusters.get(0).getZone())); -// assertThat(tagValues).contains(TagValue.create(clusters.get(0).getId())); + List tagValueStrings = StatsWrapper.getOperationLatencyViewTagValueStrings(); + assertThat(tagValueStrings).contains(clusters.get(0).getZone()); + assertThat(tagValueStrings).contains(clusters.get(0).getId()); } @Test @@ -111,21 +90,11 @@ public void testFailure() throws InterruptedException { } catch (NotFoundException e) { } -// // give opencensus some time to populate view data -// Thread.sleep(100); -// -// ViewManager viewManager = Stats.getViewManager(); -// ViewData viewData = -// viewManager.getView( -// View.Name.create("bigtable.googleapis.com/internal/client/operation_latencies")); -// -// List tagValues = -// viewData.getAggregationMap().entrySet().stream() -// .map(Map.Entry::getKey) -// .flatMap(x -> x.stream()) -// .collect(Collectors.toCollection(ArrayList::new)); -// -// assertThat(tagValues).contains(TagValue.create("undefined")); -// assertThat(tagValues).contains(TagValue.create("undefined")); + // give opencensus some time to populate view data + Thread.sleep(100); + + List tagValueStrings = StatsWrapper.getOperationLatencyViewTagValueStrings(); + assertThat(tagValueStrings).contains("undefined"); + assertThat(tagValueStrings).contains("undefined"); } } From 35c5a16c1851e27a3cc502626bff82c03e32f2bc Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Fri, 8 Jul 2022 16:56:52 -0400 Subject: [PATCH 16/18] fix dependency plugin --- google-cloud-bigtable-stats/pom.xml | 8 ++++++++ .../com/google/cloud/bigtable/stats/StatsWrapper.java | 3 +-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/google-cloud-bigtable-stats/pom.xml b/google-cloud-bigtable-stats/pom.xml index 6e9bc091cacf..d2f345b5efbf 100644 --- a/google-cloud-bigtable-stats/pom.xml +++ b/google-cloud-bigtable-stats/pom.xml @@ -50,6 +50,7 @@ io.opencensus opencensus-impl + runtime com.google.truth @@ -109,6 +110,13 @@ + + org.apache.maven.plugins + maven-dependency-plugin + + true + + diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/StatsWrapper.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/StatsWrapper.java index 3e06f5e18510..abda87edd9bf 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/StatsWrapper.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/StatsWrapper.java @@ -40,8 +40,7 @@ public static StatsRecorderWrapper createRecorder( } // This is used in integration tests to get the tag value strings from view manager because Stats - // is relocated to - // com.google.bigtable.veneer.repackaged.io.opencensus. + // is relocated to com.google.bigtable.veneer.repackaged.io.opencensus. @VisibleForTesting public static List getOperationLatencyViewTagValueStrings() { return Stats.getViewManager().getView(BuiltinViewConstants.OPERATION_LATENCIES_VIEW.getName()) From 267c91cd6e9c04b4377c8df34aaa198bdcd3a00d Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Mon, 11 Jul 2022 10:59:07 -0400 Subject: [PATCH 17/18] fix pom and update annotation --- google-cloud-bigtable-stats/pom.xml | 7 ------- .../java/com/google/cloud/bigtable/stats/StatsWrapper.java | 3 +-- pom.xml | 7 +++++++ 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/google-cloud-bigtable-stats/pom.xml b/google-cloud-bigtable-stats/pom.xml index d2f345b5efbf..44d3a6949d52 100644 --- a/google-cloud-bigtable-stats/pom.xml +++ b/google-cloud-bigtable-stats/pom.xml @@ -110,13 +110,6 @@ - - org.apache.maven.plugins - maven-dependency-plugin - - true - - diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/StatsWrapper.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/StatsWrapper.java index abda87edd9bf..c30dba6e6c63 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/StatsWrapper.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/StatsWrapper.java @@ -19,7 +19,6 @@ import com.google.api.core.InternalApi; import com.google.api.gax.tracing.SpanName; -import com.google.common.annotations.VisibleForTesting; import io.opencensus.stats.Stats; import java.util.ArrayList; import java.util.List; @@ -41,7 +40,7 @@ public static StatsRecorderWrapper createRecorder( // This is used in integration tests to get the tag value strings from view manager because Stats // is relocated to com.google.bigtable.veneer.repackaged.io.opencensus. - @VisibleForTesting + @InternalApi("Visible for testing") public static List getOperationLatencyViewTagValueStrings() { return Stats.getViewManager().getView(BuiltinViewConstants.OPERATION_LATENCIES_VIEW.getName()) .getAggregationMap().entrySet().stream() diff --git a/pom.xml b/pom.xml index f898b3952b4f..7ae4cba9c637 100644 --- a/pom.xml +++ b/pom.xml @@ -297,6 +297,13 @@ maven-shade-plugin 3.3.0 + + org.apache.maven.plugins + maven-dependency-plugin + + true + + From edbb528592917af77c99c795e190cde4ec96cf5f Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Mon, 11 Jul 2022 15:02:25 +0000 Subject: [PATCH 18/18] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 495bbf69fca4..2e403afad069 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ If you are using Maven without BOM, add this to your dependencies: If you are using Gradle 5.x or later, add this to your dependencies ```Groovy -implementation platform('com.google.cloud:libraries-bom:25.4.0') +implementation platform('com.google.cloud:libraries-bom:26.0.0') implementation 'com.google.cloud:google-cloud-bigtable' ```