From 5f88fe160e964453b02bc8b652d7e434fa835a77 Mon Sep 17 00:00:00 2001 From: Gaurav Bafna Date: Mon, 2 Sep 2024 14:53:03 +0530 Subject: [PATCH] WIP to fix retry --- .../repositories/s3/S3AsyncService.java | 3 +- .../repositories/s3/S3ClientSettings.java | 2 +- .../opensearch/repositories/s3/S3Service.java | 35 +++++++++---------- .../s3/S3ClientSettingsTests.java | 1 + 4 files changed, 21 insertions(+), 20 deletions(-) diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3AsyncService.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3AsyncService.java index d691cad9c9d03..a3a2c036791df 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3AsyncService.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3AsyncService.java @@ -18,6 +18,7 @@ import software.amazon.awssdk.core.client.config.ClientAsyncConfiguration; import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption; +import software.amazon.awssdk.core.retry.RetryMode; import software.amazon.awssdk.core.retry.RetryPolicy; import software.amazon.awssdk.core.retry.backoff.BackoffStrategy; import software.amazon.awssdk.http.async.SdkAsyncHttpClient; @@ -239,7 +240,7 @@ static ClientOverrideConfiguration buildOverrideConfiguration(final S3ClientSett RetryPolicy.builder() .numRetries(clientSettings.maxRetries) .throttlingBackoffStrategy( - clientSettings.throttleRetries ? BackoffStrategy.defaultThrottlingStrategy() : BackoffStrategy.none() + clientSettings.throttleRetries ? BackoffStrategy.defaultThrottlingStrategy(RetryMode.STANDARD) : BackoffStrategy.none() ) .build() ) diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java index e44f408e6dd12..3fbc7032be99d 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java @@ -225,7 +225,7 @@ final class S3ClientSettings { static final Setting.AffixSetting MAX_RETRIES_SETTING = Setting.affixKeySetting( PREFIX, "max_retries", - key -> Setting.intSetting(key, 3, 0, Property.NodeScope) + key -> Setting.intSetting(key, 10, 0, Property.NodeScope) ); /** Whether retries should be throttled (ie use backoff). */ diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java index fe81da31432f4..49f26e90a5fc8 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java @@ -32,6 +32,20 @@ package org.opensearch.repositories.s3; +import org.apache.http.conn.ssl.DefaultHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.protocol.HttpContext; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.cluster.metadata.RepositoryMetadata; +import org.opensearch.common.Nullable; +import org.opensearch.common.SuppressForbidden; +import org.opensearch.common.collect.MapBuilder; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.Strings; +import org.opensearch.repositories.s3.S3ClientSettings.IrsaCredentials; +import org.opensearch.repositories.s3.utils.AwsRequestSigner; +import org.opensearch.repositories.s3.utils.Protocol; import software.amazon.awssdk.auth.credentials.AwsCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider; @@ -42,6 +56,7 @@ import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption; import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.core.retry.RetryMode; import software.amazon.awssdk.core.retry.RetryPolicy; import software.amazon.awssdk.core.retry.backoff.BackoffStrategy; import software.amazon.awssdk.http.SystemPropertyTlsKeyManagersProvider; @@ -58,23 +73,7 @@ import software.amazon.awssdk.services.sts.auth.StsWebIdentityTokenFileCredentialsProvider; import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; -import org.apache.http.conn.ssl.DefaultHostnameVerifier; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.protocol.HttpContext; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.opensearch.cluster.metadata.RepositoryMetadata; -import org.opensearch.common.Nullable; -import org.opensearch.common.SuppressForbidden; -import org.opensearch.common.collect.MapBuilder; -import org.opensearch.common.settings.Settings; -import org.opensearch.core.common.Strings; -import org.opensearch.repositories.s3.S3ClientSettings.IrsaCredentials; -import org.opensearch.repositories.s3.utils.AwsRequestSigner; -import org.opensearch.repositories.s3.utils.Protocol; - import javax.net.ssl.SSLContext; - import java.io.Closeable; import java.io.IOException; import java.net.Authenticator; @@ -328,8 +327,8 @@ static ClientOverrideConfiguration buildOverrideConfiguration(final S3ClientSett RetryPolicy.Builder retryPolicy = SocketAccess.doPrivileged( () -> RetryPolicy.builder().numRetries(clientSettings.maxRetries).retryCapacityCondition(null) ); - if (!clientSettings.throttleRetries) { - retryPolicy.throttlingBackoffStrategy(BackoffStrategy.none()); + if (clientSettings.throttleRetries) { + retryPolicy.throttlingBackoffStrategy(BackoffStrategy.defaultThrottlingStrategy(RetryMode.STANDARD)); } return clientOverrideConfiguration.retryPolicy(retryPolicy.build()).build(); } diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ClientSettingsTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ClientSettingsTests.java index b47749553aeba..8dd076345b70e 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ClientSettingsTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ClientSettingsTests.java @@ -78,6 +78,7 @@ public void testThereIsADefaultClientByDefault() { assertThat(defaultSettings.connectionAcquisitionTimeoutMillis, is(15 * 60 * 1000)); assertThat(defaultSettings.maxRetries, is(3)); assertThat(defaultSettings.throttleRetries, is(true)); + } public void testDefaultClientSettingsCanBeSet() {