From f36daf2fccb30af5888df29363d03fa3d9473e2b Mon Sep 17 00:00:00 2001 From: Dai MIKURUBE Date: Fri, 1 Mar 2024 16:34:38 +0900 Subject: [PATCH 1/2] Upgrade the base Embulk SPI to 0.11, embulk-util-timestamp to 0.2.2 --- build.gradle | 20 +++++------ gradle.lockfile | 15 ++++----- .../elasticsearch/ElasticsearchTestUtils.java | 5 ++- .../TestElasticsearchHttpClient.java | 2 +- .../TestElasticsearchOutputPlugin.java | 33 ++++++++++--------- .../TestElasticsearchOutputPluginJSON.java | 18 +++++----- 6 files changed, 46 insertions(+), 47 deletions(-) diff --git a/build.gradle b/build.gradle index 409098f..9bc3265 100644 --- a/build.gradle +++ b/build.gradle @@ -33,10 +33,9 @@ java { } dependencies { - compileOnly "org.embulk:embulk-api:0.10.28" - compileOnly "org.embulk:embulk-spi:0.10.28" + compileOnly "org.embulk:embulk-spi:0.11" - implementation("org.embulk:embulk-util-config:0.3.2") { + implementation("org.embulk:embulk-util-config:0.3.4") { // They conflict with embulk-core. They are once excluded here, // and added explicitly with versions exactly the same with embulk-core:0.10.28. exclude group: "com.fasterxml.jackson.core", module: "jackson-annotations" @@ -50,21 +49,22 @@ dependencies { // and added explicitly with versions exactly the same with embulk-core:0.10.28. implementation "com.fasterxml.jackson.core:jackson-annotations:2.6.7" implementation "com.fasterxml.jackson.core:jackson-core:2.6.7" - implementation "com.fasterxml.jackson.core:jackson-databind:2.6.7" + implementation "com.fasterxml.jackson.core:jackson-databind:2.6.7.5" implementation "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7" implementation "javax.validation:validation-api:1.1.0.Final" implementation "org.embulk:embulk-base-restclient:0.10.1" implementation "org.embulk:embulk-util-retryhelper-jetty92:0.8.2" - implementation "org.embulk:embulk-util-timestamp:0.2.1" + implementation "org.embulk:embulk-util-timestamp:0.2.2" testImplementation "junit:junit:4.13.2" - testImplementation "org.embulk:embulk-core:0.10.28" - testImplementation "org.embulk:embulk-core:0.10.28:tests" - testImplementation "org.embulk:embulk-standards:0.10.28" - testImplementation "org.embulk:embulk-junit4:0.10.28" - testImplementation "org.embulk:embulk-deps:0.10.28" + testImplementation "org.embulk:embulk-core:0.11.2" + testImplementation "org.embulk:embulk-deps:0.11.2" + testImplementation "org.embulk:embulk-junit4:0.11.2" + testImplementation "org.embulk:embulk-input-file:0.11.0" + testImplementation "org.embulk:embulk-parser-csv:0.11.4" + testImplementation "com.google.guava:guava:18.0" } embulkPlugin { diff --git a/gradle.lockfile b/gradle.lockfile index 5178d4d..af76559 100644 --- a/gradle.lockfile +++ b/gradle.lockfile @@ -3,22 +3,21 @@ # This file is expected to be part of source control. com.fasterxml.jackson.core:jackson-annotations:2.6.7=compileClasspath,runtimeClasspath com.fasterxml.jackson.core:jackson-core:2.6.7=compileClasspath,runtimeClasspath -com.fasterxml.jackson.core:jackson-databind:2.6.7=compileClasspath,runtimeClasspath +com.fasterxml.jackson.core:jackson-databind:2.6.7.5=compileClasspath,runtimeClasspath com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7=compileClasspath,runtimeClasspath javax.validation:validation-api:1.1.0.Final=compileClasspath,runtimeClasspath org.eclipse.jetty:jetty-client:9.2.14.v20151106=compileClasspath,runtimeClasspath org.eclipse.jetty:jetty-http:9.2.14.v20151106=compileClasspath,runtimeClasspath org.eclipse.jetty:jetty-io:9.2.14.v20151106=compileClasspath,runtimeClasspath org.eclipse.jetty:jetty-util:9.2.14.v20151106=compileClasspath,runtimeClasspath -org.embulk:embulk-api:0.10.28=compileClasspath org.embulk:embulk-base-restclient:0.10.1=compileClasspath,runtimeClasspath -org.embulk:embulk-spi:0.10.28=compileClasspath -org.embulk:embulk-util-config:0.3.2=compileClasspath,runtimeClasspath +org.embulk:embulk-spi:0.11=compileClasspath +org.embulk:embulk-util-config:0.3.4=compileClasspath,runtimeClasspath org.embulk:embulk-util-json:0.1.1=compileClasspath,runtimeClasspath org.embulk:embulk-util-retryhelper-jetty92:0.8.2=compileClasspath,runtimeClasspath org.embulk:embulk-util-retryhelper:0.8.2=compileClasspath,runtimeClasspath -org.embulk:embulk-util-rubytime:0.3.2=runtimeClasspath -org.embulk:embulk-util-timestamp:0.2.1=compileClasspath,runtimeClasspath -org.msgpack:msgpack-core:0.8.11=compileClasspath -org.slf4j:slf4j-api:1.7.30=compileClasspath +org.embulk:embulk-util-rubytime:0.3.3=compileClasspath,runtimeClasspath +org.embulk:embulk-util-timestamp:0.2.2=compileClasspath,runtimeClasspath +org.msgpack:msgpack-core:0.8.24=compileClasspath +org.slf4j:slf4j-api:2.0.7=compileClasspath empty= diff --git a/src/test/java/org/embulk/output/elasticsearch/ElasticsearchTestUtils.java b/src/test/java/org/embulk/output/elasticsearch/ElasticsearchTestUtils.java index 877cd1a..08c6c81 100644 --- a/src/test/java/org/embulk/output/elasticsearch/ElasticsearchTestUtils.java +++ b/src/test/java/org/embulk/output/elasticsearch/ElasticsearchTestUtils.java @@ -18,7 +18,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.embulk.EmbulkTestRuntime; import org.embulk.config.ConfigSource; import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.PluginTask; import org.embulk.spi.Schema; @@ -99,9 +98,9 @@ public ConfigSource config() .set("maximum_retries", 2); } - public ConfigSource oldParserConfig(final EmbulkTestRuntime runtime) + public ConfigSource oldParserConfig() { - return runtime.getExec().newConfigSource() + return ElasticsearchOutputPlugin.CONFIG_MAPPER_FACTORY.newConfigSource() .set("parser", parserConfig(schemaConfig())) .getNested("parser"); } diff --git a/src/test/java/org/embulk/output/elasticsearch/TestElasticsearchHttpClient.java b/src/test/java/org/embulk/output/elasticsearch/TestElasticsearchHttpClient.java index e89c62e..019d3f9 100644 --- a/src/test/java/org/embulk/output/elasticsearch/TestElasticsearchHttpClient.java +++ b/src/test/java/org/embulk/output/elasticsearch/TestElasticsearchHttpClient.java @@ -17,12 +17,12 @@ package org.embulk.output.elasticsearch; import org.eclipse.jetty.http.HttpMethod; -import org.embulk.EmbulkTestRuntime; import org.embulk.config.ConfigException; import org.embulk.config.ConfigSource; import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.PluginTask; import org.embulk.spi.Exec; import org.embulk.spi.time.Timestamp; +import org.embulk.test.EmbulkTestRuntime; import org.embulk.util.config.ConfigMapper; import org.embulk.util.config.ConfigMapperFactory; import org.junit.Before; diff --git a/src/test/java/org/embulk/output/elasticsearch/TestElasticsearchOutputPlugin.java b/src/test/java/org/embulk/output/elasticsearch/TestElasticsearchOutputPlugin.java index 4a54715..1ecdf11 100644 --- a/src/test/java/org/embulk/output/elasticsearch/TestElasticsearchOutputPlugin.java +++ b/src/test/java/org/embulk/output/elasticsearch/TestElasticsearchOutputPlugin.java @@ -19,7 +19,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; import org.eclipse.jetty.http.HttpMethod; -import org.embulk.EmbulkTestRuntime; import org.embulk.config.ConfigException; import org.embulk.config.ConfigSource; import org.embulk.config.TaskReport; @@ -27,15 +26,16 @@ import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.AuthMethod; import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.Mode; import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.PluginTask; -import org.embulk.spi.Exec; import org.embulk.spi.OutputPlugin; import org.embulk.spi.Page; -import org.embulk.spi.PageTestUtils; import org.embulk.spi.Schema; import org.embulk.spi.TransactionalPageOutput; import org.embulk.spi.time.Timestamp; -import org.embulk.standards.CsvParserPlugin; +import org.embulk.parser.csv.CsvParserPlugin; +import org.embulk.test.EmbulkTestRuntime; +import org.embulk.test.PageTestUtils; import org.embulk.util.config.ConfigMapper; +import org.embulk.util.config.ConfigMapperFactory; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -57,6 +57,7 @@ public class TestElasticsearchOutputPlugin { + private static final ConfigMapperFactory CONFIG_MAPPER_FACTORY = ElasticsearchOutputPlugin.CONFIG_MAPPER_FACTORY; private static final ConfigMapper CONFIG_MAPPER = ElasticsearchOutputPlugin.CONFIG_MAPPER; @Rule @@ -85,7 +86,7 @@ public void testDefaultValues() @Test public void testDefaultValuesNull() { - final ConfigSource config = runtime.getExec().newConfigSource() + final ConfigSource config = CONFIG_MAPPER_FACTORY.newConfigSource() .set("in", utils.inputConfig()) .set("parser", utils.parserConfig(utils.schemaConfig())) .set("type", "elasticsearch") @@ -98,14 +99,14 @@ public void testDefaultValuesNull() .set("bulk_size", ES_BULK_SIZE) .set("concurrent_requests", ES_CONCURRENT_REQUESTS ); - Schema schema = utils.oldParserConfig(runtime).loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema(); + Schema schema = CONFIG_MAPPER.map(utils.oldParserConfig(), CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema(); try { plugin.transaction(config, schema, 0, new OutputPlugin.Control() { @Override public List run(TaskSource taskSource) { - return Lists.newArrayList(Exec.newTaskReport()); + return Lists.newArrayList(CONFIG_MAPPER_FACTORY.newTaskReport()); } }); } @@ -120,13 +121,13 @@ public List run(TaskSource taskSource) public void testTransaction() { ConfigSource config = utils.config(); - Schema schema = utils.oldParserConfig(runtime).loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema(); + Schema schema = CONFIG_MAPPER.map(utils.oldParserConfig(), CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema(); plugin.transaction(config, schema, 0, new OutputPlugin.Control() { @Override public List run(TaskSource taskSource) { - return Lists.newArrayList(Exec.newTaskReport()); + return Lists.newArrayList(CONFIG_MAPPER_FACTORY.newTaskReport()); } }); // no error happens @@ -136,14 +137,14 @@ public List run(TaskSource taskSource) public void testResume() { ConfigSource config = utils.config(); - Schema schema = utils.oldParserConfig(runtime).loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema(); + Schema schema = CONFIG_MAPPER.map(utils.oldParserConfig(), CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema(); final PluginTask task = CONFIG_MAPPER.map(config, PluginTask.class); plugin.resume(task.dump(), schema, 0, new OutputPlugin.Control() { @Override public List run(TaskSource taskSource) { - return Lists.newArrayList(Exec.newTaskReport()); + return Lists.newArrayList(CONFIG_MAPPER_FACTORY.newTaskReport()); } }); } @@ -152,9 +153,9 @@ public List run(TaskSource taskSource) public void testCleanup() { ConfigSource config = utils.config(); - Schema schema = utils.oldParserConfig(runtime).loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema(); + Schema schema = CONFIG_MAPPER.map(utils.oldParserConfig(), CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema(); final PluginTask task = CONFIG_MAPPER.map(config, PluginTask.class); - plugin.cleanup(task.dump(), schema, 0, Arrays.asList(Exec.newTaskReport())); + plugin.cleanup(task.dump(), schema, 0, Arrays.asList(CONFIG_MAPPER_FACTORY.newTaskReport())); // no error happens } @@ -162,13 +163,13 @@ public void testCleanup() public void testOutputByOpen() throws Exception { ConfigSource config = utils.config(); - Schema schema = utils.oldParserConfig(runtime).loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema(); + Schema schema = CONFIG_MAPPER.map(utils.oldParserConfig(), CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema(); final PluginTask task = CONFIG_MAPPER.map(config, PluginTask.class); plugin.transaction(config, schema, 0, new OutputPlugin.Control() { @Override public List run(TaskSource taskSource) { - return Lists.newArrayList(Exec.newTaskReport()); + return Lists.newArrayList(CONFIG_MAPPER_FACTORY.newTaskReport()); } }); TransactionalPageOutput output = plugin.open(task.dump(), schema, 0); @@ -214,7 +215,7 @@ public List run(TaskSource taskSource) public void testOpenAbort() { ConfigSource config = utils.config(); - Schema schema = utils.oldParserConfig(runtime).loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema(); + Schema schema = CONFIG_MAPPER.map(utils.oldParserConfig(), CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema(); final PluginTask task = CONFIG_MAPPER.map(config, PluginTask.class); TransactionalPageOutput output = plugin.open(task.dump(), schema, 0); output.abort(); diff --git a/src/test/java/org/embulk/output/elasticsearch/TestElasticsearchOutputPluginJSON.java b/src/test/java/org/embulk/output/elasticsearch/TestElasticsearchOutputPluginJSON.java index 63292e9..841fc4a 100644 --- a/src/test/java/org/embulk/output/elasticsearch/TestElasticsearchOutputPluginJSON.java +++ b/src/test/java/org/embulk/output/elasticsearch/TestElasticsearchOutputPluginJSON.java @@ -19,7 +19,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; import org.eclipse.jetty.http.HttpMethod; -import org.embulk.EmbulkTestRuntime; import org.embulk.config.ConfigException; import org.embulk.config.ConfigSource; import org.embulk.config.TaskReport; @@ -30,9 +29,10 @@ import org.embulk.spi.Exec; import org.embulk.spi.OutputPlugin; import org.embulk.spi.Page; -import org.embulk.spi.PageTestUtils; import org.embulk.spi.Schema; import org.embulk.spi.TransactionalPageOutput; +import org.embulk.test.EmbulkTestRuntime; +import org.embulk.test.PageTestUtils; import org.embulk.util.config.ConfigMapper; import org.embulk.util.config.ConfigMapperFactory; import org.junit.Before; @@ -87,7 +87,7 @@ public void testTransaction() @Override public List run(TaskSource taskSource) { - return Lists.newArrayList(Exec.newTaskReport()); + return Lists.newArrayList(CONFIG_MAPPER_FACTORY.newTaskReport()); } }); // no error happens @@ -104,7 +104,7 @@ public void testResume() @Override public List run(TaskSource taskSource) { - return Lists.newArrayList(Exec.newTaskReport()); + return Lists.newArrayList(CONFIG_MAPPER_FACTORY.newTaskReport()); } }); } @@ -115,7 +115,7 @@ public void testCleanup() ConfigSource config = utils.configJSON(); Schema schema = utils.JSONSchema(); final PluginTask task = CONFIG_MAPPER.map(config, PluginTask.class); - plugin.cleanup(task.dump(), schema, 0, Arrays.asList(Exec.newTaskReport())); + plugin.cleanup(task.dump(), schema, 0, Arrays.asList(CONFIG_MAPPER_FACTORY.newTaskReport())); // no error happens } @@ -129,12 +129,12 @@ public void testOutputByOpen() throws Exception @Override public List run(TaskSource taskSource) { - return Lists.newArrayList(Exec.newTaskReport()); + return Lists.newArrayList(CONFIG_MAPPER_FACTORY.newTaskReport()); } }); TransactionalPageOutput output = plugin.open(task.dump(), schema, 0); - List pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), schema, 1L, 32864L, "2015-01-27 19:23:49", "2015-01-27", true, 123.45, "embulk"); + List pages = PageTestUtils.buildPage(Exec.getBufferAllocator(), schema, 1L, 32864L, "2015-01-27 19:23:49", "2015-01-27", true, 123.45, "embulk"); assertThat(pages.size(), is(1)); for (Page page : pages) { output.add(page); @@ -182,12 +182,12 @@ public void testOutputByOpenWithNulls() throws Exception @Override public List run(TaskSource taskSource) { - return Lists.newArrayList(Exec.newTaskReport()); + return Lists.newArrayList(CONFIG_MAPPER_FACTORY.newTaskReport()); } }); TransactionalPageOutput output = plugin.open(task.dump(), schema, 0); - List pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), schema, 2L, null, null, "2015-01-27", true, 123.45, "embulk"); + List pages = PageTestUtils.buildPage(Exec.getBufferAllocator(), schema, 2L, null, null, "2015-01-27", true, 123.45, "embulk"); assertThat(pages.size(), is(1)); for (Page page : pages) { output.add(page); From cf5b5cf6d1e88e07dd96c00364c3b6e04d6afef6 Mon Sep 17 00:00:00 2001 From: Dai MIKURUBE Date: Fri, 1 Mar 2024 16:48:38 +0900 Subject: [PATCH 2/2] Remove jcenter() --- build.gradle | 3 --- 1 file changed, 3 deletions(-) diff --git a/build.gradle b/build.gradle index 9bc3265..d30faf9 100644 --- a/build.gradle +++ b/build.gradle @@ -9,9 +9,6 @@ plugins { repositories { mavenCentral() - - // Tests need EmbulkTestRuntime. EmbulkTestRuntime needs old-style embulk-standards till 0.10.28. 0.10.28 needs jcenter. - jcenter() } group = "org.embulk"