diff --git a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java index 04731841a..cb06e25fd 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java @@ -49,6 +49,7 @@ import java.util.zip.ZipOutputStream; import static com.conveyal.datatools.manager.DataManager.getConfigPropertyAsText; +import static com.conveyal.datatools.manager.utils.HttpUtils.downloadFileFromURL; import static com.mongodb.client.model.Filters.and; import static com.mongodb.client.model.Filters.eq; @@ -209,7 +210,9 @@ public DeployJob.DeploySummary latest () { public String routerId; public String customBuildConfig; + public String customBuildConfigUrl; public String customRouterConfig; + public String customRouterConfigUrl; public List customFiles = new ArrayList<>(); @@ -414,14 +417,42 @@ public void dump (File output, boolean includeManifest, boolean includeOsm, bool out.close(); } - /** Generate build config for deployment as byte array (for writing to file output stream). */ - public byte[] generateBuildConfig() { - Project project = this.parentProject(); + /** Download config from provided URL. */ + public String downloadConfig(String configUrl) throws IOException { + if (configUrl != null) { + try { + // TODO: validate JSON? + return new String(downloadFileFromURL(new URL(configUrl)), StandardCharsets.UTF_8); + } catch (IOException e) { + String message = String.format("Could not download config file from %s.", configUrl); + LOG.error(message); + throw new IOException(message, e); + } + } + return null; + } + + /** Generate build config for deployment as byte array (for writing to file output stream). If an external build + * config is available and is successfully downloaded, use this instead of the deployment build config. If there is + * no deployment build config, use the project build config. */ + public byte[] generateBuildConfig() throws IOException { + String downloadedConfig = downloadConfig(customBuildConfigUrl); + if (downloadedConfig != null) { + customBuildConfig = downloadedConfig; + } return customBuildConfig != null ? customBuildConfig.getBytes(StandardCharsets.UTF_8) - : project.buildConfig != null - ? writeToBytes(project.buildConfig) - : null; + : getProjectBuildConfig(); + } + + /** + * If a project build config exists, return this as a byte array, or null if not available. + */ + private byte[] getProjectBuildConfig() { + Project project = parentProject(); + return project.buildConfig != null + ? writeToBytes(project.buildConfig) + : null; } public String generateBuildConfigAsString() { @@ -451,15 +482,18 @@ private String writeToString(O object) { /** Generate router config for deployment as string. */ public byte[] generateRouterConfig() throws IOException { - Project project = this.parentProject(); + String downloadedConfig = downloadConfig(customRouterConfigUrl); + if (downloadedConfig != null) { + customRouterConfig = downloadedConfig; + } byte[] customRouterConfigString = customRouterConfig != null - ? customRouterConfig.getBytes(StandardCharsets.UTF_8) - : null; + ? customRouterConfig.getBytes(StandardCharsets.UTF_8) + : null; - byte[] routerConfigString = project.routerConfig != null - ? writeToBytes(project.routerConfig) - : null; + byte[] routerConfigString = parentProject().routerConfig != null + ? writeToBytes(parentProject().routerConfig) + : null; // If both router configs are present, merge the JSON before returning // Merger code from: https://stackoverflow.com/questions/35747813/how-to-merge-two-json-strings-into-one-in-java diff --git a/src/main/java/com/conveyal/datatools/manager/utils/HttpUtils.java b/src/main/java/com/conveyal/datatools/manager/utils/HttpUtils.java index fdd6ed0d5..2f3461a9a 100644 --- a/src/main/java/com/conveyal/datatools/manager/utils/HttpUtils.java +++ b/src/main/java/com/conveyal/datatools/manager/utils/HttpUtils.java @@ -14,9 +14,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.net.URI; +import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -113,4 +116,21 @@ public static SimpleHttpResponse httpRequestRawResponse( return null; } } + + /** + * Download a file from a URL and return as a byte array. + */ + public static byte[] downloadFileFromURL(URL url) throws IOException { + try ( + InputStream inputStream = url.openStream(); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream() + ) { + byte[] chunk = new byte[4096]; + int bytesRead; + while ((bytesRead = inputStream.read(chunk)) >= 0) { + outputStream.write(chunk, 0, bytesRead); + } + return outputStream.toByteArray(); + } + } } diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/DeployJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/DeployJobTest.java index d3812276a..e0504d589 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/DeployJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/DeployJobTest.java @@ -27,6 +27,7 @@ import static com.zenika.snapshotmatcher.SnapshotMatcher.matchesSnapshot; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assumptions.assumeTrue; /** @@ -80,6 +81,15 @@ public static void setUp() throws IOException { Persistence.deployments.create(deployment); } + @Test + void canDownloadConfigs() throws IOException { + deployment.customBuildConfigUrl = "http://www.google.com"; + assertNotNull(deployment.downloadConfig(deployment.customBuildConfigUrl)); + + deployment.customRouterConfigUrl = "http://www.google.com"; + assertNotNull(deployment.downloadConfig(deployment.customRouterConfigUrl)); + } + /** * Tests that the otp-runner manifest and user data for a graph build + run server instance can be generated * properly