BaseWorkerClientImpl: Don't attempt to recover from a closed channel. (…

…#17052) * BaseWorkerClientImpl: Don't attempt to recover from a closed channel. This patch introduces an exception type "ChannelClosedForWritesException", which allows the BaseWorkerClientImpl to avoid retrying when the local channel has been closed. This can happen in cases of cancellation. * Add some test coverage. * wip * Add test coverage. * Style. (cherry picked from commit 4dc5942)
apache · Oct 9, 2024 · f2df70a · f2df70a
1 parent 75ac905
commit f2df70a
Show file tree

Hide file tree

Showing 6 changed files with 448 additions and 8 deletions.
diff --git a/...s-core/multi-stage-query/src/main/java/org/apache/druid/msq/rpc/BaseWorkerClientImpl.java b/...s-core/multi-stage-query/src/main/java/org/apache/druid/msq/rpc/BaseWorkerClientImpl.java
@@ -26,6 +26,7 @@
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.SettableFuture;
 import org.apache.druid.common.guava.FutureUtils;
+import org.apache.druid.frame.channel.ChannelClosedForWritesException;
 import org.apache.druid.frame.channel.ReadableByteChunksFrameChannel;
 import org.apache.druid.frame.file.FrameFileHttpResponseHandler;
 import org.apache.druid.frame.file.FrameFilePartialFetch;
@@ -221,12 +222,18 @@ public ListenableFuture<Boolean> fetchChannelData(
           public void onSuccess(FrameFilePartialFetch partialFetch)
           {
             if (partialFetch.isExceptionCaught()) {
-              // Exception while reading channel. Recoverable.
-              log.noStackTrace().info(
-                  partialFetch.getExceptionCaught(),
-                  "Encountered exception while reading channel [%s]",
-                  channel.getId()
-              );
+              if (partialFetch.getExceptionCaught() instanceof ChannelClosedForWritesException) {
+                // Channel was closed. Stop trying.
+                retVal.setException(partialFetch.getExceptionCaught());
+                return;
+              } else {
+                // Exception while reading channel. Recoverable.
+                log.noStackTrace().warn(
+                    partialFetch.getExceptionCaught(),
+                    "Attempting recovery after exception while reading channel[%s]",
+                    channel.getId()
+                );
+              }
             }
 
             // Empty fetch means this is the last fetch for the channel.