Fix AbstractQueryResourceTestClient to handle retry server errors

integration-tests/src/main/java/org/apache/druid/testing/clients/AbstractQueryResourceTestClient.java

@@ -161,12 +161,19 @@ public List<Map<String, Object>> query(String url, QueryType query)
               request,
               new BytesFullResponseHandler()
           ).get());
+
+          if (responseRef.get().getStatus().getCode() == 500) {
+            LOG.info("Server returned HTTP-500. Retrying the query request");
+            return false;
+          }
         }
         catch (Throwable t) {
           ChannelException ce = Throwables.getCauseOfType(t, ChannelException.class);
           if (ce != null) {
             LOG.info(ce, "Encountered a channel exception. Retrying the query request");
             return false;
+          } else {
+            throw new RuntimeException("non-retriable exception", t);
           }
         }
         return true;

integration-tests/src/test/java/org/apache/druid/tests/ITAbstractQueryResourceTestClientTest.java

@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.tests;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.google.common.collect.ImmutableList;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.inject.Binder;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+import com.google.inject.Provides;
+import org.apache.druid.guice.GuiceInjectors;
+import org.apache.druid.guice.IndexingServiceFirehoseModule;
+import org.apache.druid.guice.IndexingServiceInputSourceModule;
+import org.apache.druid.guice.IndexingServiceTuningConfigModule;
+import org.apache.druid.initialization.Initialization;
+import org.apache.druid.java.util.http.client.HttpClient;
+import org.apache.druid.java.util.http.client.Request;
+import org.apache.druid.java.util.http.client.response.BytesFullResponseHolder;
+import org.apache.druid.java.util.http.client.response.HttpResponseHandler;
+import org.apache.druid.testing.clients.QueryResourceTestClient;
+import org.apache.druid.testing.guice.DruidTestModule;
+import org.apache.druid.testing.guice.TestClient;
+import org.checkerframework.checker.nullness.qual.Nullable;
+import org.jboss.netty.handler.codec.http.DefaultHttpResponse;
+import org.jboss.netty.handler.codec.http.HttpHeaders;
+import org.jboss.netty.handler.codec.http.HttpResponse;
+import org.jboss.netty.handler.codec.http.HttpResponseStatus;
+import org.jboss.netty.handler.codec.http.HttpVersion;
+import org.joda.time.Duration;
+import org.testng.IModuleFactory;
+import org.testng.ITestContext;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import javax.ws.rs.core.MediaType;
+
+import java.nio.charset.Charset;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+
+import static org.testng.Assert.assertEquals;
+
+@Test(groups = TestNGGroup.HTTP_ENDPOINT)
+@Guice(moduleFactory = ITAbstractQueryResourceTestClientTest.LocalModuleFactory.class)
+public class ITAbstractQueryResourceTestClientTest
+{
+
+  static class LocalModule implements Module
+  {
+
+    DruidTestModule delegate = new DruidTestModule();
+
+    @Override
+    public void configure(Binder binder)
+    {
+      delegate.configure(binder);
+    }
+
+    @Provides
+    @TestClient
+    public HttpClient getHttpClient()
+    {
+      return new HttpClient()
+      {
+        @Override
+        public <Intermediate, Final> ListenableFuture<Final> go(Request request,
+            HttpResponseHandler<Intermediate, Final> handler)
+        {
+          @Nullable
+          Final val = null;
+
+          int counter = ITAbstractQueryResourceTestClientTest.requestCounter.getAndIncrement();
+          HttpResponse response = new DefaultHttpResponse(
+              HttpVersion.HTTP_1_1,
+              counter == 0 ? HttpResponseStatus.INTERNAL_SERVER_ERROR : HttpResponseStatus.OK);
+          response.headers().add(HttpHeaders.Names.CONTENT_TYPE, MediaType.APPLICATION_JSON);
+
+          BytesFullResponseHolder bytesFullResponseHolder = new BytesFullResponseHolder(response);
+          bytesFullResponseHolder.addChunk("[{}]".getBytes(Charset.defaultCharset()));
+          val = (@Nullable Final) bytesFullResponseHolder;
+          return Futures.<Final>immediateFuture(val);
+        }
+
+        @Override
+        public <Intermediate, Final> ListenableFuture<Final> go(Request request,
+            HttpResponseHandler<Intermediate, Final> handler, Duration readTimeout)
+        {
+          throw new RuntimeException("Unimplemented!");
+        }
+      };
+    }
+  }
+
+  static class LocalModuleFactory implements IModuleFactory
+  {
+    private static final Module MODULE = new DruidTestModule();
+    private static final Injector INJECTOR = Initialization.makeInjectorWithModules(
+        GuiceInjectors.makeStartupInjector(),
+        getModules());
+
+    public static Injector getInjector()
+    {
+      return INJECTOR;
+    }
+
+    private static List<? extends Module> getModules()
+    {
+      return ImmutableList.of(
+          new LocalModule(),
+          new IndexingServiceFirehoseModule(),
+          new IndexingServiceInputSourceModule(),
+          new IndexingServiceTuningConfigModule());
+    }
+
+    @Override
+    public Module createModule(ITestContext context, Class<?> testClass)
+    {
+      context.addInjector(Collections.singletonList(MODULE), INJECTOR);
+      return MODULE;
+    }
+  }
+
+  @Inject
+  private QueryResourceTestClient queryClient;
+
+  private static AtomicInteger requestCounter = new AtomicInteger();
+
+  @Test
+  public void testInternalServerErrorAtFirstTry() throws JsonProcessingException
+  {
+    requestCounter.set(0);
+    queryClient.query("http://192.168.99.99/asd", null);
+    assertEquals(2, requestCounter.get());
+  }
+}