Skip to content

Commit

Permalink
Merge pull request #78 from aiven/affix-settings
Browse files Browse the repository at this point in the history
Affix settings
  • Loading branch information
reta authored Feb 14, 2022
2 parents 40e86cf + 7997479 commit f37b783
Show file tree
Hide file tree
Showing 23 changed files with 767 additions and 392 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@

import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;

import org.opensearch.common.settings.SecureSetting;
import org.opensearch.common.settings.SecureString;
Expand All @@ -28,46 +31,84 @@
import io.aiven.elasticsearch.repositories.CommonSettings;

import static io.aiven.elasticsearch.repositories.CommonSettings.ClientSettings.checkSettings;
import static io.aiven.elasticsearch.repositories.CommonSettings.ClientSettings.getConfigValue;
import static io.aiven.elasticsearch.repositories.CommonSettings.ClientSettings.readInputStream;
import static io.aiven.elasticsearch.repositories.CommonSettings.ClientSettings.withPrefix;

public class AzureClientSettings implements CommonSettings.ClientSettings {

static final String AZURE_PREFIX = AIVEN_PREFIX + "azure.";

static final String AZURE_CONNECTION_STRING_TEMPLATE =
"DefaultEndpointsProtocol=https;AccountName=%s;AccountKey=%s";

static final Setting<Integer> AZURE_HTTP_POOL_MIN_THREADS =
Setting.intSetting(withPrefix("azure.http.thread_pool.min"),
Runtime.getRuntime().availableProcessors() * 2 - 1, 1, Setting.Property.NodeScope);

static final Setting<Integer> AZURE_HTTP_POOL_MAX_THREADS =
Setting.intSetting(withPrefix("azure.http.thread_pool.max"),
Runtime.getRuntime().availableProcessors() * 2 - 1, 1, Setting.Property.NodeScope);

static final Setting<TimeValue> AZURE_HTTP_POOL_KEEP_ALIVE =
Setting.timeSetting(withPrefix("azure.http.thread_pool.keep_alive"), TimeValue.timeValueSeconds(30L),
Setting.Property.NodeScope);

static final Setting<Integer> AZURE_HTTP_POOL_WORKING_QUEUE_SIZE =
Setting.intSetting(withPrefix("azure.http.thread_pool.working_queue_size"), 1000, 10,
Setting.Property.NodeScope);


public static final Setting<InputStream> PUBLIC_KEY_FILE =
SecureSetting.secureFile(withPrefix("azure.public_key_file"), null);

public static final Setting<InputStream> PRIVATE_KEY_FILE =
SecureSetting.secureFile(withPrefix("azure.private_key_file"), null);

public static final Setting<SecureString> AZURE_ACCOUNT =
SecureSetting.secureString(withPrefix("azure.client.account"), null);

public static final Setting<SecureString> AZURE_ACCOUNT_KEY =
SecureSetting.secureString(withPrefix("azure.client.account.key"), null);
static final Setting.AffixSetting<Integer> AZURE_HTTP_POOL_MIN_THREADS =
Setting.affixKeySetting(
AZURE_PREFIX,
"http.thread_pool.min",
key -> Setting.intSetting(key,
Runtime.getRuntime().availableProcessors() * 2 - 1, 1,
Setting.Property.NodeScope)
);

static final Setting.AffixSetting<Integer> AZURE_HTTP_POOL_MAX_THREADS =
Setting.affixKeySetting(
AZURE_PREFIX,
"http.thread_pool.max",
key -> Setting.intSetting(key,
Runtime.getRuntime().availableProcessors() * 2 - 1, 1,
Setting.Property.NodeScope)
);

static final Setting.AffixSetting<TimeValue> AZURE_HTTP_POOL_KEEP_ALIVE =
Setting.affixKeySetting(
AZURE_PREFIX,
"http.thread_pool.keep_alive",
key -> Setting.timeSetting(key, TimeValue.timeValueSeconds(30L), Setting.Property.NodeScope)
);

static final Setting.AffixSetting<Integer> AZURE_HTTP_POOL_WORKING_QUEUE_SIZE =
Setting.affixKeySetting(
AZURE_PREFIX,
"http.thread_pool.working_queue_size",
key -> Setting.intSetting(key, 1000, 10, Setting.Property.NodeScope)
);


public static final Setting.AffixSetting<InputStream> PUBLIC_KEY_FILE =
Setting.affixKeySetting(
AZURE_PREFIX,
"public_key_file",
key -> SecureSetting.secureFile(key, null)
);

public static final Setting.AffixSetting<InputStream> PRIVATE_KEY_FILE =
Setting.affixKeySetting(
AZURE_PREFIX,
"private_key_file",
key -> SecureSetting.secureFile(key, null)
);

public static final Setting.AffixSetting<SecureString> AZURE_ACCOUNT =
Setting.affixKeySetting(
AZURE_PREFIX,
"client.account",
key -> SecureSetting.secureString(key, null)
);

public static final Setting.AffixSetting<SecureString> AZURE_ACCOUNT_KEY =
Setting.affixKeySetting(
AZURE_PREFIX,
"client.account.key",
key -> SecureSetting.secureString(key, null)
);

//default is 3 please take a look ExponentialBackoff azure class
public static final Setting<Integer> MAX_RETRIES =
Setting.intSetting(withPrefix("max_retries"), 3, Setting.Property.NodeScope);
public static final Setting.AffixSetting<Integer> MAX_RETRIES =
Setting.affixKeySetting(
AZURE_PREFIX,
"max_retries",
key -> Setting.intSetting(key, 3, Setting.Property.NodeScope)
);

private final byte[] publicKey;

Expand Down Expand Up @@ -119,22 +160,31 @@ public int maxRetries() {
return maxRetries;
}

public static AzureClientSettings create(final Settings settings) throws IOException {
checkSettings(AZURE_ACCOUNT, settings);
checkSettings(AZURE_ACCOUNT_KEY, settings);
checkSettings(PUBLIC_KEY_FILE, settings);
checkSettings(PRIVATE_KEY_FILE, settings);
public static Map<String, AzureClientSettings> create(final Settings settings) throws IOException {
final Set<String> clientNames = settings.getGroups(AZURE_PREFIX).keySet();
final var clientSettings = new HashMap<String, AzureClientSettings>();
for (final var clientName : clientNames) {
clientSettings.put(clientName, createSettings(clientName, settings));
}
return Map.copyOf(clientSettings);
}

static AzureClientSettings createSettings(final String clientName, final Settings settings) throws IOException {
checkSettings(AZURE_ACCOUNT, clientName, settings);
checkSettings(AZURE_ACCOUNT_KEY, clientName, settings);
checkSettings(PUBLIC_KEY_FILE, clientName, settings);
checkSettings(PRIVATE_KEY_FILE, clientName, settings);
return new AzureClientSettings(
readInputStream(PUBLIC_KEY_FILE, settings),
readInputStream(PRIVATE_KEY_FILE, settings),
AZURE_ACCOUNT.get(settings).toString(),
AZURE_ACCOUNT_KEY.get(settings).toString(),
MAX_RETRIES.get(settings),
readInputStream(getConfigValue(settings, clientName, PUBLIC_KEY_FILE)),
readInputStream(getConfigValue(settings, clientName, PRIVATE_KEY_FILE)),
getConfigValue(settings, clientName, AZURE_ACCOUNT).toString(),
getConfigValue(settings, clientName, AZURE_ACCOUNT_KEY).toString(),
getConfigValue(settings, clientName, MAX_RETRIES),
new HttpThreadPoolSettings(
AZURE_HTTP_POOL_MIN_THREADS.get(settings),
AZURE_HTTP_POOL_MAX_THREADS.get(settings),
AZURE_HTTP_POOL_KEEP_ALIVE.get(settings).getMillis(),
AZURE_HTTP_POOL_WORKING_QUEUE_SIZE.get(settings))
getConfigValue(settings, clientName, AZURE_HTTP_POOL_MIN_THREADS),
getConfigValue(settings, clientName, AZURE_HTTP_POOL_MAX_THREADS),
getConfigValue(settings, clientName, AZURE_HTTP_POOL_KEEP_ALIVE).getMillis(),
getConfigValue(settings, clientName, AZURE_HTTP_POOL_WORKING_QUEUE_SIZE))
);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,14 @@

import com.azure.storage.blob.BlobServiceClient;

import static io.aiven.elasticsearch.repositories.azure.AzureClientSettings.AZURE_PREFIX;

public class AzureRepositoryPlugin extends AbstractRepositoryPlugin<BlobServiceClient, AzureClientSettings> {

public static final String REPOSITORY_TYPE = "aiven-azure";

public AzureRepositoryPlugin(final Settings settings) {
super(REPOSITORY_TYPE, settings, new AzureSettingsProvider());
super(REPOSITORY_TYPE, AZURE_PREFIX, settings, new AzureSettingsProvider());
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@
import io.aiven.elasticsearch.repositories.Permissions;
import io.aiven.elasticsearch.repositories.RepositoryStorageIOProvider;
import io.aiven.elasticsearch.repositories.io.CryptoIOProvider;
import io.aiven.elasticsearch.repositories.security.EncryptionKeyProvider;

import com.azure.storage.blob.BlobContainerClient;
import com.azure.storage.blob.BlobServiceClient;
Expand All @@ -50,9 +49,8 @@ public class AzureRepositoryStorageIOProvider

static final Setting<String> CONTAINER_NAME = Setting.simpleString("container_name");

public AzureRepositoryStorageIOProvider(final AzureClientSettings clientSettings,
final EncryptionKeyProvider encryptionKeyProvider) {
super(new AzureClientProvider(), clientSettings, encryptionKeyProvider);
public AzureRepositoryStorageIOProvider(final Map<String, AzureClientSettings> clientSettings) {
super(new AzureClientProvider(), clientSettings);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
import io.aiven.elasticsearch.repositories.Permissions;
import io.aiven.elasticsearch.repositories.RepositorySettingsProvider;
import io.aiven.elasticsearch.repositories.RepositoryStorageIOProvider;
import io.aiven.elasticsearch.repositories.security.EncryptionKeyProvider;

import com.azure.storage.blob.BlobServiceClient;

Expand All @@ -34,10 +33,8 @@ protected RepositoryStorageIOProvider<BlobServiceClient, AzureClientSettings> cr
final Settings settings) throws IOException {
return Permissions.doPrivileged(() -> {
final var azureClientSettings = AzureClientSettings.create(settings);
final var encryptionKeyProvider =
EncryptionKeyProvider.of(azureClientSettings.publicKey(), azureClientSettings.privateKey());
return Permissions.doPrivileged(() ->
new AzureRepositoryStorageIOProvider(azureClientSettings, encryptionKeyProvider));
new AzureRepositoryStorageIOProvider(azureClientSettings));
});
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,10 @@
import org.mockito.junit.jupiter.MockitoExtension;

import static io.aiven.elasticsearch.repositories.CommonSettings.RepositorySettings.MAX_RETRIES;
import static io.aiven.elasticsearch.repositories.azure.AzureClientSettings.AZURE_ACCOUNT;
import static io.aiven.elasticsearch.repositories.azure.AzureClientSettings.AZURE_ACCOUNT_KEY;
import static io.aiven.elasticsearch.repositories.azure.AzureClientSettings.PRIVATE_KEY_FILE;
import static io.aiven.elasticsearch.repositories.azure.AzureClientSettings.PUBLIC_KEY_FILE;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
Expand Down Expand Up @@ -109,8 +113,9 @@ void testBuildClient() throws Exception {
.put("some_settings_2", 210)
.build();

final var azureClientSettings = AzureClientSettings.create(settings);
final var client = azureClientProvider
.buildClientIfNeeded(AzureClientSettings.create(settings), repoSettings);
.buildClientIfNeeded(azureClientSettings.get("default"), repoSettings).v2();

assertEquals("AZURE_ACCOUNT", client.getAccountName());

Expand All @@ -131,8 +136,9 @@ void testMaxRetriesOverridesClientSettings() throws Exception {
.put("some_settings_2", 210)
.build();

final var azureClientSettings = AzureClientSettings.create(settings);
final var client = azureClientProvider
.buildClientIfNeeded(AzureClientSettings.create(settings), repoSettings);
.buildClientIfNeeded(azureClientSettings.get("default"), repoSettings).v2();

assertEquals("AZURE_ACCOUNT", client.getAccountName());

Expand All @@ -145,13 +151,22 @@ void testMaxRetriesOverridesClientSettings() throws Exception {
private Settings createSettings() throws IOException {
final var secureSettings =
new DummySecureSettings()
.setString(AzureClientSettings.AZURE_ACCOUNT.getKey(), "AZURE_ACCOUNT")
.setString(AzureClientSettings.AZURE_ACCOUNT_KEY.getKey(), "AZURE_ACCOUNT_KEY")
.setFile(AzureClientSettings.PUBLIC_KEY_FILE.getKey(), Files.newInputStream(publicKeyPem))
.setFile(AzureClientSettings.PRIVATE_KEY_FILE.getKey(), Files.newInputStream(privateKeyPem));
.setString(
AZURE_ACCOUNT.getConcreteSettingForNamespace("default").getKey(),
"AZURE_ACCOUNT"
).setString(
AZURE_ACCOUNT_KEY.getConcreteSettingForNamespace("default").getKey(),
"AZURE_ACCOUNT_KEY"
).setFile(
PUBLIC_KEY_FILE.getConcreteSettingForNamespace("default").getKey(),
Files.newInputStream(publicKeyPem)
).setFile(
PRIVATE_KEY_FILE.getConcreteSettingForNamespace("default").getKey(),
Files.newInputStream(privateKeyPem)
);

return Settings.builder()
.put(AzureClientSettings.MAX_RETRIES.getKey(), 12)
.put(AzureClientSettings.MAX_RETRIES.getConcreteSettingForNamespace("default").getKey(), 12)
.setSecureSettings(secureSettings)
.build();
}
Expand Down
Loading

0 comments on commit f37b783

Please sign in to comment.