diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java index ae2dacb2d1aa..1ef72a91937a 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java @@ -26,7 +26,6 @@ import com.google.common.base.Ticker; import com.google.common.collect.HashMultiset; import com.google.common.collect.Multiset; -import com.google.common.util.concurrent.MoreExecutors; import io.grpc.CallOptions; import io.grpc.ClientCall; import io.grpc.ManagedChannel; @@ -296,11 +295,15 @@ public synchronized SessionStream newStream( fw.numOutstanding++; totalStreams++; - // DirectExecutor: gRPC/Netty delivers SessionStream.Listener callbacks directly on the - // I/O thread. All work must be fast and non-blocking; blocking work goes to sessionSyncContext. + // Do NOT override the call executor with directExecutor(): SessionStream.Listener callbacks + // acquire the SessionPool lock (onReady/onGoAway/onClose/onVRpcComplete). directExecutor() + // delivers those callbacks inline on the Netty event-loop (I/O) thread, so a contended pool + // lock blocks the event loop; the blocked event loop then can't deliver the very completions + // that would release sessions and drain the lock -> self-sustaining pod-wide wedge. Leaving + // the executor unset delivers callbacks on gRPC's off-loop channel executor, keeping + // pool-lock acquisition off the transport threads. ClientCall innerCall = - channelWrapper.channel.newCall( - desc, callOptions.withExecutor(MoreExecutors.directExecutor())); + channelWrapper.channel.newCall(desc, callOptions); return new SessionStreamImpl(innerCall) { // null until onBeforeSessionStart fires diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/SingleChannelPool.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/SingleChannelPool.java index bfd099f60d51..d69371adcd75 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/SingleChannelPool.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/SingleChannelPool.java @@ -19,7 +19,6 @@ import com.google.bigtable.v2.SessionClientConfiguration.ChannelPoolConfiguration; import com.google.bigtable.v2.SessionRequest; import com.google.bigtable.v2.SessionResponse; -import com.google.common.util.concurrent.MoreExecutors; import io.grpc.CallOptions; import io.grpc.ManagedChannel; import io.grpc.MethodDescriptor; @@ -46,10 +45,14 @@ public void close() { @Override public SessionStream newStream( MethodDescriptor desc, CallOptions callOptions) { - // DirectExecutor: gRPC/Netty delivers SessionStream.Listener callbacks directly on the - // I/O thread. All work must be fast and non-blocking; blocking work goes to sessionSyncContext. - return new SessionStreamImpl( - channel.newCall(desc, callOptions.withExecutor(MoreExecutors.directExecutor()))); + // Do NOT override the call executor with directExecutor(): SessionStream.Listener callbacks + // acquire the SessionPool lock (onReady/onGoAway/onClose/onVRpcComplete). directExecutor() + // delivers those callbacks inline on the Netty event-loop (I/O) thread, so a contended pool + // lock blocks the event loop; the blocked event loop then can't deliver the very completions + // that would release sessions and drain the lock -> self-sustaining pod-wide wedge. Leaving + // the executor unset delivers callbacks on gRPC's off-loop channel executor, keeping + // pool-lock acquisition off the transport threads. + return new SessionStreamImpl(channel.newCall(desc, callOptions)); } @Override