Support placeholder in e2e config.

apache · Dec 13, 2024 · 556ac6c · 556ac6c
1 parent f23d908
commit 556ac6c
Show file tree

Hide file tree

Showing 8 changed files with 136 additions and 14 deletions.
diff --git a/.../test/java/org/apache/shardingsphere/test/e2e/env/container/atomic/DockerITContainer.java b/.../test/java/org/apache/shardingsphere/test/e2e/env/container/atomic/DockerITContainer.java
@@ -24,10 +24,12 @@
 import org.testcontainers.containers.wait.strategy.DockerHealthcheckWaitStrategy;
 import org.testcontainers.images.RemoteDockerImage;
 import org.testcontainers.utility.DockerImageName;
+import org.testcontainers.utility.MountableFile;
 
 import java.time.Duration;
 import java.time.temporal.ChronoUnit;
 import java.util.Collection;
+import java.util.Map;
 import java.util.stream.Collectors;
 
 /**
@@ -77,4 +79,16 @@ private void startDependencies() {
 
     protected void postStart() {
     }
+
+    protected void mountConfigurationFiles(final Map<String, String> mountedResources) {
+        mountedResources.forEach((key, value) -> {
+            MountableFile mountableFile;
+            try {
+                mountableFile = MountableFile.forClasspathResource(key);
+            } catch (final IllegalArgumentException ignore) {
+                mountableFile = MountableFile.forHostPath(key);
+            }
+            withCopyFileToContainer(mountableFile, value);
+        });
+    }
 }
diff --git a/.../apache/shardingsphere/test/e2e/env/container/atomic/adapter/AdapterContainerFactory.java b/.../apache/shardingsphere/test/e2e/env/container/atomic/adapter/AdapterContainerFactory.java
@@ -28,8 +28,13 @@
 import org.apache.shardingsphere.test.e2e.env.container.atomic.enums.AdapterMode;
 import org.apache.shardingsphere.test.e2e.env.container.atomic.enums.AdapterType;
 import org.apache.shardingsphere.test.e2e.env.container.atomic.storage.StorageContainer;
+import org.apache.shardingsphere.test.e2e.env.container.atomic.util.ConfigPlaceholderReplacer;
 import org.apache.shardingsphere.test.e2e.env.runtime.scenario.path.ScenarioCommonPath;
 
+import java.util.Collections;
+import java.util.Map;
+import java.util.stream.Collectors;
+
 /**
  * Adapter container factory.
  */
@@ -50,6 +55,8 @@ public final class AdapterContainerFactory {
      */
     public static AdapterContainer newInstance(final AdapterMode mode, final AdapterType adapter, final DatabaseType databaseType,
                                                final String scenario, final AdaptorContainerConfiguration containerConfig, final StorageContainer storageContainer) {
+        Map<String, String> replacedResources = ConfigPlaceholderReplacer.getReplacedResources(containerConfig.getMountedResources().keySet());
+        containerConfig.setMountedResources(containerConfig.getMountedResources().entrySet().stream().collect(Collectors.toMap(each -> replacedResources.get(each.getKey()), Map.Entry::getValue)));
         switch (adapter) {
             case PROXY:
                 return AdapterMode.CLUSTER == mode
@@ -58,7 +65,8 @@ public static AdapterContainer newInstance(final AdapterMode mode, final Adapter
             case PROXY_RANDOM:
                 return new ShardingSphereMultiProxyClusterContainer(databaseType, containerConfig);
             case JDBC:
-                return new ShardingSphereJdbcContainer(storageContainer, new ScenarioCommonPath(scenario).getRuleConfigurationFile(databaseType));
+                return new ShardingSphereJdbcContainer(storageContainer,
+                        ConfigPlaceholderReplacer.getReplacedResources(Collections.singleton(new ScenarioCommonPath(scenario).getRuleConfigurationFile(databaseType))).values().iterator().next());
             default:
                 throw new RuntimeException(String.format("Unknown adapter `%s`.", adapter));
         }

diff --git a/...ingsphere/test/e2e/env/container/atomic/adapter/config/AdaptorContainerConfiguration.java b/...ingsphere/test/e2e/env/container/atomic/adapter/config/AdaptorContainerConfiguration.java
@@ -17,24 +17,26 @@
 
 package org.apache.shardingsphere.test.e2e.env.container.atomic.adapter.config;
 
+import lombok.AllArgsConstructor;
 import lombok.Getter;
-import lombok.RequiredArgsConstructor;
+import lombok.Setter;
 
 import java.util.List;
 import java.util.Map;
 
 /**
  * Adaptor container configuration.
  */
-@RequiredArgsConstructor
+@AllArgsConstructor
 @Getter
 public final class AdaptorContainerConfiguration {
 
     private final String proxyDataSourceName;
 
     private final List<String> portBindings;
 
-    private final Map<String, String> mountedResources;
+    @Setter
+    private Map<String, String> mountedResources;
 
     private final String adapterContainerImage;
 

diff --git a/...phere/test/e2e/env/container/atomic/adapter/impl/ShardingSphereProxyClusterContainer.java b/...phere/test/e2e/env/container/atomic/adapter/impl/ShardingSphereProxyClusterContainer.java
@@ -79,16 +79,12 @@ protected void configure() {
             setPortBindings(config.getPortBindings());
         }
         addEnv("TZ", "UTC");
-        mountConfigurationFiles();
+        mountConfigurationFiles(config.getMountedResources());
         setWaitStrategy(new JdbcConnectionWaitStrategy(() -> DriverManager.getConnection(DataSourceEnvironment.getURL(databaseType,
                 getHost(), getMappedPort(3307), config.getProxyDataSourceName()), ProxyContainerConstants.USERNAME, ProxyContainerConstants.PASSWORD)));
         withStartupTimeout(Duration.of(120L, ChronoUnit.SECONDS));
     }
 
-    private void mountConfigurationFiles() {
-        config.getMountedResources().forEach((key, value) -> withClasspathResourceMapping(key, value, BindMode.READ_ONLY));
-    }
-
     @Override
     public DataSource getTargetDataSource(final String serverLists) {
         DataSource dataSource = targetDataSourceProvider.get();

diff --git a/...re/test/e2e/env/container/atomic/adapter/impl/ShardingSphereProxyStandaloneContainer.java b/...re/test/e2e/env/container/atomic/adapter/impl/ShardingSphereProxyStandaloneContainer.java
@@ -65,15 +65,11 @@ public ShardingSphereProxyStandaloneContainer withAgent(final String agentHome)
     @Override
     protected void configure() {
         withExposedPorts(3307, 3308);
-        mountConfigurationFiles();
+        mountConfigurationFiles(config.getMountedResources());
         setWaitStrategy(new JdbcConnectionWaitStrategy(() -> DriverManager.getConnection(
                 DataSourceEnvironment.getURL(databaseType, getHost(), getMappedPort(3307), config.getProxyDataSourceName()), "proxy", "Proxy@123")));
     }
 
-    private void mountConfigurationFiles() {
-        config.getMountedResources().forEach((key, value) -> withClasspathResourceMapping(key, value, BindMode.READ_ONLY));
-    }
-
     @Override
     public DataSource getTargetDataSource(final String serverLists) {
         DataSource dataSource = targetDataSourceProvider.get();

diff --git a/...g/apache/shardingsphere/test/e2e/env/container/atomic/util/ConfigPlaceholderReplacer.java b/...g/apache/shardingsphere/test/e2e/env/container/atomic/util/ConfigPlaceholderReplacer.java
@@ -0,0 +1,97 @@
+package org.apache.shardingsphere.test.e2e.env.container.atomic.util;
+
+import lombok.SneakyThrows;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.SystemUtils;
+import org.apache.shardingsphere.infra.util.directory.ClasspathResourceDirectoryReader;
+import org.apache.shardingsphere.test.e2e.env.runtime.E2ETestEnvironment;
+import org.testcontainers.utility.Base58;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+public class ConfigPlaceholderReplacer {
+
+    private static final String OS_MAC_TMP_DIR = "/tmp";
+
+    private static final String E2E_PROXY_CONFIG_TMP_DIR_PREFIX = "shardingsphere-e2e-config-";
+
+    /**
+     * Get replaced resources.
+     *
+     * @param configResources config resources
+     * @return replaced resources
+     */
+    public static Map<String, String> getReplacedResources(final Collection<String> configResources) {
+        return getReplacedResources(configResources, getPlaceholderAndReplacementsMap());
+    }
+
+    @SneakyThrows(IOException.class)
+    private static Map<String, String> getReplacedResources(final Collection<String> configResources, final Map<String, String> placeholderAndReplacementsMap) {
+        Map<String, String> result = new HashMap<>();
+        Path tempDirectory = createTempDirectory().toPath();
+        for (String each : configResources) {
+            String configResource = StringUtils.removeStart(each, "/");
+            if (ClasspathResourceDirectoryReader.isDirectory(configResource)) {
+                Path subDirectory = tempDirectory.resolve(Paths.get(configResource).getFileName());
+                Files.createDirectory(subDirectory);
+                ClasspathResourceDirectoryReader.read(configResource).forEach(resource -> getReplacedTempFile(subDirectory, resource, placeholderAndReplacementsMap));
+                result.put(each, subDirectory.toFile().getAbsolutePath());
+            } else {
+                result.put(each, getReplacedTempFile(tempDirectory, each, placeholderAndReplacementsMap).getAbsolutePath());
+            }
+        }
+        return result;
+    }
+
+    @SneakyThrows(IOException.class)
+    private static File getReplacedTempFile(final Path tempDirectory, final String configResource, final Map<String, String> placeholderAndReplacementsMap) {
+        String content = getContent(configResource);
+        for (Map.Entry<String, String> entry : placeholderAndReplacementsMap.entrySet()) {
+            content = content.replace(entry.getKey(), entry.getValue());
+        }
+        File tempFile = tempDirectory.resolve(Paths.get(configResource).getFileName()).toFile();
+        tempFile.deleteOnExit();
+        try (FileWriter writer = new FileWriter(tempFile)) {
+            writer.write(content);
+        }
+        return tempFile;
+    }
+
+    private static String getContent(final String configResource) throws IOException {
+        String content;
+        InputStream resourceAsStream = ConfigPlaceholderReplacer.class.getClassLoader().getResourceAsStream(StringUtils.removeStart(configResource, "/"));
+        if (resourceAsStream != null) {
+            content = IOUtils.toString(resourceAsStream, StandardCharsets.UTF_8);
+        } else {
+            content = IOUtils.toString(Paths.get(configResource).toUri(), StandardCharsets.UTF_8);
+        }
+        return content;
+    }
+
+    private static Map<String, String> getPlaceholderAndReplacementsMap() {
+        return E2ETestEnvironment.getInstance().getPlaceholderAndReplacementsMap();
+    }
+
+    private static File createTempDirectory() {
+        try {
+            if (SystemUtils.IS_OS_MAC) {
+                return Files.createTempDirectory(Paths.get(OS_MAC_TMP_DIR), E2E_PROXY_CONFIG_TMP_DIR_PREFIX).toFile();
+            }
+            return Files.createTempDirectory(E2E_PROXY_CONFIG_TMP_DIR_PREFIX).toFile();
+        } catch (final IOException ex) {
+            return new File(E2E_PROXY_CONFIG_TMP_DIR_PREFIX + Base58.randomString(5));
+        }
+    }
+
+}
diff --git a/.../env/src/test/java/org/apache/shardingsphere/test/e2e/env/runtime/E2ETestEnvironment.java b/.../env/src/test/java/org/apache/shardingsphere/test/e2e/env/runtime/E2ETestEnvironment.java
@@ -19,14 +19,17 @@
 
 import com.google.common.base.Splitter;
 import lombok.Getter;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.shardingsphere.test.e2e.env.runtime.cluster.ClusterEnvironment;
 import org.apache.shardingsphere.test.e2e.env.runtime.scenario.path.ScenarioCommonPath;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.Collection;
+import java.util.Map;
 import java.util.Properties;
 import java.util.TimeZone;
+import java.util.stream.Collectors;
 
 /**
  * E2E test environment.
@@ -46,6 +49,8 @@ public final class E2ETestEnvironment {
 
     private final boolean smoke;
 
+    private final Map<String, String> placeholderAndReplacementsMap;
+
     private E2ETestEnvironment() {
         Properties props = loadProperties();
         runModes = Splitter.on(",").trimResults().splitToList(props.getProperty("it.run.modes"));
@@ -54,6 +59,8 @@ private E2ETestEnvironment() {
         scenarios = getScenarios(props);
         smoke = Boolean.parseBoolean(props.getProperty("it.run.smoke"));
         clusterEnvironment = new ClusterEnvironment(props);
+        placeholderAndReplacementsMap = props.entrySet().stream().filter(entry -> entry.getKey().toString().startsWith("it.placeholder."))
+                .collect(Collectors.toMap(entry -> "${" + StringUtils.removeStart(String.valueOf(entry.getKey()), "it.placeholder.") + "}", entry -> String.valueOf(entry.getValue())));
     }
 
     @SuppressWarnings("AccessOfSystemProperties")

diff --git a/test/e2e/sql/src/test/resources/env/it-env.properties b/test/e2e/sql/src/test/resources/env/it-env.properties
@@ -33,3 +33,5 @@ it.cluster.adapters=proxy
 it.cluster.databases=MySQL
 
 #it.cluster.database.mysql.image=mysql:8.2.0
+
+it.placeholder.querywithplain=false
Original file line number	Diff line number	Diff line change
Expand Up		@@ -33,3 +33,5 @@ it.cluster.adapters=proxy
		it.cluster.databases=MySQL

		#it.cluster.database.mysql.image=mysql:8.2.0

		it.placeholder.querywithplain=false