Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Upgrade the base Embulk SPI to 0.11, embulk-util-timestamp to 0.2.2 #80

Merged
merged 3 commits into from
Jul 26, 2024
Merged

Upgrade the base Embulk SPI to 0.11, embulk-util-timestamp to 0.2.2 #80

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f36daf2
Upgrade the base Embulk SPI to 0.11, embulk-util-timestamp to 0.2.2
dmikurube Mar 1, 2024
cf5b5cf
Remove jcenter()
dmikurube Mar 1, 2024
31b2140
Merge branch 'master' into spi-0.11-timestamp-0.2.2
dmikurube Jul 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 10 additions & 13 deletions build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,6 @@ plugins {

repositories {
mavenCentral()

// Tests need EmbulkTestRuntime. EmbulkTestRuntime needs old-style embulk-standards till 0.10.28. 0.10.28 needs jcenter.
jcenter()
}

group = "org.embulk"
Expand All @@ -33,10 +30,9 @@ java {
}

dependencies {
compileOnly "org.embulk:embulk-api:0.10.28"
compileOnly "org.embulk:embulk-spi:0.10.28"
compileOnly "org.embulk:embulk-spi:0.11"

implementation("org.embulk:embulk-util-config:0.3.2") {
implementation("org.embulk:embulk-util-config:0.3.4") {
// They conflict with embulk-core. They are once excluded here,
// and added explicitly with versions exactly the same with embulk-core:0.10.28.
exclude group: "com.fasterxml.jackson.core", module: "jackson-annotations"
Expand All @@ -50,21 +46,22 @@ dependencies {
// and added explicitly with versions exactly the same with embulk-core:0.10.28.
implementation "com.fasterxml.jackson.core:jackson-annotations:2.6.7"
implementation "com.fasterxml.jackson.core:jackson-core:2.6.7"
implementation "com.fasterxml.jackson.core:jackson-databind:2.6.7"
implementation "com.fasterxml.jackson.core:jackson-databind:2.6.7.5"
implementation "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7"
implementation "javax.validation:validation-api:1.1.0.Final"

implementation "org.embulk:embulk-base-restclient:0.10.1"
implementation "org.embulk:embulk-util-retryhelper-jetty94:0.9.0"

implementation "org.embulk:embulk-util-timestamp:0.2.1"
implementation "org.embulk:embulk-util-timestamp:0.2.2"

testImplementation "junit:junit:4.13.2"
testImplementation "org.embulk:embulk-core:0.10.28"
testImplementation "org.embulk:embulk-core:0.10.28:tests"
testImplementation "org.embulk:embulk-standards:0.10.28"
testImplementation "org.embulk:embulk-junit4:0.10.28"
testImplementation "org.embulk:embulk-deps:0.10.28"
testImplementation "org.embulk:embulk-core:0.11.2"
testImplementation "org.embulk:embulk-deps:0.11.2"
testImplementation "org.embulk:embulk-junit4:0.11.2"
testImplementation "org.embulk:embulk-input-file:0.11.0"
testImplementation "org.embulk:embulk-parser-csv:0.11.4"
testImplementation "com.google.guava:guava:18.0"
}

embulkPlugin {
Expand Down
15 changes: 7 additions & 8 deletions gradle.lockfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,22 +3,21 @@
# This file is expected to be part of source control.
com.fasterxml.jackson.core:jackson-annotations:2.6.7=compileClasspath,runtimeClasspath
com.fasterxml.jackson.core:jackson-core:2.6.7=compileClasspath,runtimeClasspath
com.fasterxml.jackson.core:jackson-databind:2.6.7=compileClasspath,runtimeClasspath
com.fasterxml.jackson.core:jackson-databind:2.6.7.5=compileClasspath,runtimeClasspath
com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7=compileClasspath,runtimeClasspath
javax.validation:validation-api:1.1.0.Final=compileClasspath,runtimeClasspath
org.eclipse.jetty:jetty-client:9.4.51.v20230217=compileClasspath,runtimeClasspath
org.eclipse.jetty:jetty-http:9.4.51.v20230217=compileClasspath,runtimeClasspath
org.eclipse.jetty:jetty-io:9.4.51.v20230217=compileClasspath,runtimeClasspath
org.eclipse.jetty:jetty-util:9.4.51.v20230217=compileClasspath,runtimeClasspath
org.embulk:embulk-api:0.10.28=compileClasspath
org.embulk:embulk-base-restclient:0.10.1=compileClasspath,runtimeClasspath
org.embulk:embulk-spi:0.10.28=compileClasspath
org.embulk:embulk-util-config:0.3.2=compileClasspath,runtimeClasspath
org.embulk:embulk-spi:0.11=compileClasspath
org.embulk:embulk-util-config:0.3.4=compileClasspath,runtimeClasspath
org.embulk:embulk-util-json:0.1.1=compileClasspath,runtimeClasspath
org.embulk:embulk-util-retryhelper-jetty94:0.9.0=compileClasspath,runtimeClasspath
org.embulk:embulk-util-retryhelper:0.9.0=compileClasspath,runtimeClasspath
org.embulk:embulk-util-rubytime:0.3.2=runtimeClasspath
org.embulk:embulk-util-timestamp:0.2.1=compileClasspath,runtimeClasspath
org.msgpack:msgpack-core:0.8.11=compileClasspath
org.slf4j:slf4j-api:1.7.30=compileClasspath
org.embulk:embulk-util-rubytime:0.3.3=compileClasspath,runtimeClasspath
org.embulk:embulk-util-timestamp:0.2.2=compileClasspath,runtimeClasspath
org.msgpack:msgpack-core:0.8.24=compileClasspath
org.slf4j:slf4j-api:2.0.7=compileClasspath
empty=
5 changes: 2 additions & 3 deletions src/test/java/org/embulk/output/elasticsearch/ElasticsearchTestUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import org.embulk.EmbulkTestRuntime;
import org.embulk.config.ConfigSource;
import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.PluginTask;
import org.embulk.spi.Schema;
Expand Down Expand Up @@ -99,9 +98,9 @@ public ConfigSource config()
.set("maximum_retries", 2);
}

public ConfigSource oldParserConfig(final EmbulkTestRuntime runtime)
public ConfigSource oldParserConfig()
{
return runtime.getExec().newConfigSource()
return ElasticsearchOutputPlugin.CONFIG_MAPPER_FACTORY.newConfigSource()
.set("parser", parserConfig(schemaConfig()))
.getNested("parser");
}
Expand Down
2 changes: 1 addition & 1 deletion src/test/java/org/embulk/output/elasticsearch/TestElasticsearchHttpClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,12 @@
package org.embulk.output.elasticsearch;

import org.eclipse.jetty.http.HttpMethod;
import org.embulk.EmbulkTestRuntime;
import org.embulk.config.ConfigException;
import org.embulk.config.ConfigSource;
import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.PluginTask;
import org.embulk.spi.Exec;
import org.embulk.spi.time.Timestamp;
import org.embulk.test.EmbulkTestRuntime;
import org.embulk.util.config.ConfigMapper;
import org.embulk.util.config.ConfigMapperFactory;
import org.junit.Before;
Expand Down
33 changes: 17 additions & 16 deletions src/test/java/org/embulk/output/elasticsearch/TestElasticsearchOutputPlugin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,23 +19,23 @@
import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.collect.Lists;
import org.eclipse.jetty.http.HttpMethod;
import org.embulk.EmbulkTestRuntime;
import org.embulk.config.ConfigException;
import org.embulk.config.ConfigSource;
import org.embulk.config.TaskReport;
import org.embulk.config.TaskSource;
import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.AuthMethod;
import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.Mode;
import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.PluginTask;
import org.embulk.spi.Exec;
import org.embulk.spi.OutputPlugin;
import org.embulk.spi.Page;
import org.embulk.spi.PageTestUtils;
import org.embulk.spi.Schema;
import org.embulk.spi.TransactionalPageOutput;
import org.embulk.spi.time.Timestamp;
import org.embulk.standards.CsvParserPlugin;
import org.embulk.parser.csv.CsvParserPlugin;
import org.embulk.test.EmbulkTestRuntime;
import org.embulk.test.PageTestUtils;
import org.embulk.util.config.ConfigMapper;
import org.embulk.util.config.ConfigMapperFactory;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
Expand All @@ -57,6 +57,7 @@

public class TestElasticsearchOutputPlugin
{
private static final ConfigMapperFactory CONFIG_MAPPER_FACTORY = ElasticsearchOutputPlugin.CONFIG_MAPPER_FACTORY;
private static final ConfigMapper CONFIG_MAPPER = ElasticsearchOutputPlugin.CONFIG_MAPPER;

@Rule
Expand Down Expand Up @@ -85,7 +86,7 @@ public void testDefaultValues()
@Test
public void testDefaultValuesNull()
{
final ConfigSource config = runtime.getExec().newConfigSource()
final ConfigSource config = CONFIG_MAPPER_FACTORY.newConfigSource()
.set("in", utils.inputConfig())
.set("parser", utils.parserConfig(utils.schemaConfig()))
.set("type", "elasticsearch")
Expand All @@ -98,14 +99,14 @@ public void testDefaultValuesNull()
.set("bulk_size", ES_BULK_SIZE)
.set("concurrent_requests", ES_CONCURRENT_REQUESTS
);
Schema schema = utils.oldParserConfig(runtime).loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
Schema schema = CONFIG_MAPPER.map(utils.oldParserConfig(), CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
try {
plugin.transaction(config, schema, 0, new OutputPlugin.Control()
{
@Override
public List<TaskReport> run(TaskSource taskSource)
{
return Lists.newArrayList(Exec.newTaskReport());
return Lists.newArrayList(CONFIG_MAPPER_FACTORY.newTaskReport());
}
});
}
Expand All @@ -120,13 +121,13 @@ public List<TaskReport> run(TaskSource taskSource)
public void testTransaction()
{
ConfigSource config = utils.config();
Schema schema = utils.oldParserConfig(runtime).loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
Schema schema = CONFIG_MAPPER.map(utils.oldParserConfig(), CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
plugin.transaction(config, schema, 0, new OutputPlugin.Control()
{
@Override
public List<TaskReport> run(TaskSource taskSource)
{
return Lists.newArrayList(Exec.newTaskReport());
return Lists.newArrayList(CONFIG_MAPPER_FACTORY.newTaskReport());
}
});
// no error happens
Expand All @@ -136,14 +137,14 @@ public List<TaskReport> run(TaskSource taskSource)
public void testResume()
{
ConfigSource config = utils.config();
Schema schema = utils.oldParserConfig(runtime).loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
Schema schema = CONFIG_MAPPER.map(utils.oldParserConfig(), CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
final PluginTask task = CONFIG_MAPPER.map(config, PluginTask.class);
plugin.resume(task.dump(), schema, 0, new OutputPlugin.Control()
{
@Override
public List<TaskReport> run(TaskSource taskSource)
{
return Lists.newArrayList(Exec.newTaskReport());
return Lists.newArrayList(CONFIG_MAPPER_FACTORY.newTaskReport());
}
});
}
Expand All @@ -152,23 +153,23 @@ public List<TaskReport> run(TaskSource taskSource)
public void testCleanup()
{
ConfigSource config = utils.config();
Schema schema = utils.oldParserConfig(runtime).loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
Schema schema = CONFIG_MAPPER.map(utils.oldParserConfig(), CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
final PluginTask task = CONFIG_MAPPER.map(config, PluginTask.class);
plugin.cleanup(task.dump(), schema, 0, Arrays.asList(Exec.newTaskReport()));
plugin.cleanup(task.dump(), schema, 0, Arrays.asList(CONFIG_MAPPER_FACTORY.newTaskReport()));
// no error happens
}

@Test
public void testOutputByOpen() throws Exception
{
ConfigSource config = utils.config();
Schema schema = utils.oldParserConfig(runtime).loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
Schema schema = CONFIG_MAPPER.map(utils.oldParserConfig(), CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
final PluginTask task = CONFIG_MAPPER.map(config, PluginTask.class);
plugin.transaction(config, schema, 0, new OutputPlugin.Control() {
@Override
public List<TaskReport> run(TaskSource taskSource)
{
return Lists.newArrayList(Exec.newTaskReport());
return Lists.newArrayList(CONFIG_MAPPER_FACTORY.newTaskReport());
}
});
TransactionalPageOutput output = plugin.open(task.dump(), schema, 0);
Expand Down Expand Up @@ -214,7 +215,7 @@ public List<TaskReport> run(TaskSource taskSource)
public void testOpenAbort()
{
ConfigSource config = utils.config();
Schema schema = utils.oldParserConfig(runtime).loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
Schema schema = CONFIG_MAPPER.map(utils.oldParserConfig(), CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
final PluginTask task = CONFIG_MAPPER.map(config, PluginTask.class);
TransactionalPageOutput output = plugin.open(task.dump(), schema, 0);
output.abort();
Expand Down
18 changes: 9 additions & 9 deletions src/test/java/org/embulk/output/elasticsearch/TestElasticsearchOutputPluginJSON.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.collect.Lists;
import org.eclipse.jetty.http.HttpMethod;
import org.embulk.EmbulkTestRuntime;
import org.embulk.config.ConfigException;
import org.embulk.config.ConfigSource;
import org.embulk.config.TaskReport;
Expand All @@ -30,9 +29,10 @@
import org.embulk.spi.Exec;
import org.embulk.spi.OutputPlugin;
import org.embulk.spi.Page;
import org.embulk.spi.PageTestUtils;
import org.embulk.spi.Schema;
import org.embulk.spi.TransactionalPageOutput;
import org.embulk.test.EmbulkTestRuntime;
import org.embulk.test.PageTestUtils;
import org.embulk.util.config.ConfigMapper;
import org.embulk.util.config.ConfigMapperFactory;
import org.junit.Before;
Expand Down Expand Up @@ -87,7 +87,7 @@ public void testTransaction()
@Override
public List<TaskReport> run(TaskSource taskSource)
{
return Lists.newArrayList(Exec.newTaskReport());
return Lists.newArrayList(CONFIG_MAPPER_FACTORY.newTaskReport());
}
});
// no error happens
Expand All @@ -104,7 +104,7 @@ public void testResume()
@Override
public List<TaskReport> run(TaskSource taskSource)
{
return Lists.newArrayList(Exec.newTaskReport());
return Lists.newArrayList(CONFIG_MAPPER_FACTORY.newTaskReport());
}
});
}
Expand All @@ -115,7 +115,7 @@ public void testCleanup()
ConfigSource config = utils.configJSON();
Schema schema = utils.JSONSchema();
final PluginTask task = CONFIG_MAPPER.map(config, PluginTask.class);
plugin.cleanup(task.dump(), schema, 0, Arrays.asList(Exec.newTaskReport()));
plugin.cleanup(task.dump(), schema, 0, Arrays.asList(CONFIG_MAPPER_FACTORY.newTaskReport()));
// no error happens
}

Expand All @@ -129,12 +129,12 @@ public void testOutputByOpen() throws Exception
@Override
public List<TaskReport> run(TaskSource taskSource)
{
return Lists.newArrayList(Exec.newTaskReport());
return Lists.newArrayList(CONFIG_MAPPER_FACTORY.newTaskReport());
}
});
TransactionalPageOutput output = plugin.open(task.dump(), schema, 0);

List<Page> pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), schema, 1L, 32864L, "2015-01-27 19:23:49", "2015-01-27", true, 123.45, "embulk");
List<Page> pages = PageTestUtils.buildPage(Exec.getBufferAllocator(), schema, 1L, 32864L, "2015-01-27 19:23:49", "2015-01-27", true, 123.45, "embulk");
assertThat(pages.size(), is(1));
for (Page page : pages) {
output.add(page);
Expand Down Expand Up @@ -182,12 +182,12 @@ public void testOutputByOpenWithNulls() throws Exception
@Override
public List<TaskReport> run(TaskSource taskSource)
{
return Lists.newArrayList(Exec.newTaskReport());
return Lists.newArrayList(CONFIG_MAPPER_FACTORY.newTaskReport());
}
});
TransactionalPageOutput output = plugin.open(task.dump(), schema, 0);

List<Page> pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), schema, 2L, null, null, "2015-01-27", true, 123.45, "embulk");
List<Page> pages = PageTestUtils.buildPage(Exec.getBufferAllocator(), schema, 2L, null, null, "2015-01-27", true, 123.45, "embulk");
assertThat(pages.size(), is(1));
for (Page page : pages) {
output.add(page);
Expand Down
Loading