From e34e35313ef1023a316230262f5da091dd2a10d9 Mon Sep 17 00:00:00 2001 From: yuki-tashiro <43487435+yu-kioo@users.noreply.github.com> Date: Wed, 7 Aug 2024 09:41:20 +0000 Subject: [PATCH 1/9] add user-agent parameter --- README.md | 3 +++ .../java/org/embulk/input/DatabricksInputPlugin.java | 12 ++++++++++++ 2 files changed, 15 insertions(+) mode change 100644 => 100755 src/main/java/org/embulk/input/DatabricksInputPlugin.java diff --git a/README.md b/README.md index f63446a..b28d9d4 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,9 @@ Databricks input plugin for Embulk loads records from Databricks. - **driver_path**: path to the jar file of the JDBC driver. If not set, [the bundled JDBC driver](https://docs.databricks.com/en/integrations/jdbc/index.html) will be used. (string, optional) - **options**: extra JDBC properties (hash, default: {}) +- **user_agent**: set user agent property to JDBC connection. If 'UserAgentEntry' property is specified in the **options**, it will be overwritten by this value. (hash, optional) + - **product_name**: product name of user agent (string) + - **product_version**: product version of user agent (string) - **server_hostname**: The Databricks compute resource’s Server Hostname value, see [Compute settings for the Databricks JDBC Driver](https://docs.databricks.com/en/integrations/jdbc/compute.html). (string, required) - **http_path**: The Databricks compute resource’s HTTP Path value, see [Compute settings for the Databricks JDBC Driver](https://docs.databricks.com/en/integrations/jdbc/compute.html). (string, required) - **personal_access_token**: The Databaricks personal_access_token, see [Authentication settings for the Databricks JDBC Driver](https://docs.databricks.com/en/integrations/jdbc/authentication.html#authentication-pat). (string, required) diff --git a/src/main/java/org/embulk/input/DatabricksInputPlugin.java b/src/main/java/org/embulk/input/DatabricksInputPlugin.java old mode 100644 new mode 100755 index 0511ac3..7408779 --- a/src/main/java/org/embulk/input/DatabricksInputPlugin.java +++ b/src/main/java/org/embulk/input/DatabricksInputPlugin.java @@ -3,6 +3,7 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; +import java.util.Map; import java.util.Optional; import java.util.Properties; import org.embulk.config.ConfigException; @@ -39,6 +40,10 @@ public interface DatabricksPluginTask extends PluginTask { @Config("schema_name") @ConfigDefault("null") public Optional getSchemaName(); + + @Config("user_agent") + @ConfigDefault("null") + public Optional> getUserAgent(); } @Override @@ -75,6 +80,13 @@ protected JdbcInputConnection newConnection(PluginTask task) throws SQLException props.put("ConnSchema", t.getSchemaName().get()); } props.putAll(t.getOptions()); + // overwrite UserAgentEntry property if the same property is set in options + if (t.getUserAgent().isPresent()) { + String product_name = t.getUserAgent().get().get("product_name"); + String product_version = t.getUserAgent().get().get("product_version"); + + props.put("UserAgentEntry", product_name + "/" + product_version); + } logConnectionProperties(url, props); Connection c = DriverManager.getConnection(url, props); From 15c247c33ba6100f5c5021ba0286f29afe59ca71 Mon Sep 17 00:00:00 2001 From: yuki-tashiro <43487435+yu-kioo@users.noreply.github.com> Date: Thu, 8 Aug 2024 03:04:42 +0000 Subject: [PATCH 2/9] add default value to user-agent parameter --- README.md | 4 ++-- .../embulk/input/DatabricksInputPlugin.java | 20 ++++++++++++++----- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index b28d9d4..69ffeaf 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,8 @@ Databricks input plugin for Embulk loads records from Databricks. - **driver_path**: path to the jar file of the JDBC driver. If not set, [the bundled JDBC driver](https://docs.databricks.com/en/integrations/jdbc/index.html) will be used. (string, optional) - **options**: extra JDBC properties (hash, default: {}) - **user_agent**: set user agent property to JDBC connection. If 'UserAgentEntry' property is specified in the **options**, it will be overwritten by this value. (hash, optional) - - **product_name**: product name of user agent (string) - - **product_version**: product version of user agent (string) + - **product_name**: product name of user agent (string, default: "unknown") + - **product_version**: product version of user agent (string, default: "0.0.0") - **server_hostname**: The Databricks compute resource’s Server Hostname value, see [Compute settings for the Databricks JDBC Driver](https://docs.databricks.com/en/integrations/jdbc/compute.html). (string, required) - **http_path**: The Databricks compute resource’s HTTP Path value, see [Compute settings for the Databricks JDBC Driver](https://docs.databricks.com/en/integrations/jdbc/compute.html). (string, required) - **personal_access_token**: The Databaricks personal_access_token, see [Authentication settings for the Databricks JDBC Driver](https://docs.databricks.com/en/integrations/jdbc/authentication.html#authentication-pat). (string, required) diff --git a/src/main/java/org/embulk/input/DatabricksInputPlugin.java b/src/main/java/org/embulk/input/DatabricksInputPlugin.java index 7408779..c8ccbda 100755 --- a/src/main/java/org/embulk/input/DatabricksInputPlugin.java +++ b/src/main/java/org/embulk/input/DatabricksInputPlugin.java @@ -3,7 +3,6 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; -import java.util.Map; import java.util.Optional; import java.util.Properties; import org.embulk.config.ConfigException; @@ -12,6 +11,7 @@ import org.embulk.spi.Schema; import org.embulk.util.config.Config; import org.embulk.util.config.ConfigDefault; +import org.embulk.util.config.Task; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,8 +42,18 @@ public interface DatabricksPluginTask extends PluginTask { public Optional getSchemaName(); @Config("user_agent") - @ConfigDefault("null") - public Optional> getUserAgent(); + @ConfigDefault("{}") + public Optional getUserAgent(); + + public interface UserAgentEntry extends Task { + @Config("product_name") + @ConfigDefault("\"unknown\"") + public String getProductName(); + + @Config("product_version") + @ConfigDefault("\"0.0.0\"") + public String getProductVersion(); + } } @Override @@ -82,8 +92,8 @@ protected JdbcInputConnection newConnection(PluginTask task) throws SQLException props.putAll(t.getOptions()); // overwrite UserAgentEntry property if the same property is set in options if (t.getUserAgent().isPresent()) { - String product_name = t.getUserAgent().get().get("product_name"); - String product_version = t.getUserAgent().get().get("product_version"); + String product_name = t.getUserAgent().get().getProductName(); + String product_version = t.getUserAgent().get().getProductVersion(); props.put("UserAgentEntry", product_name + "/" + product_version); } From 1f12088b3b6c247bbd3ffe19cb26e277c13a1820 Mon Sep 17 00:00:00 2001 From: yuki-tashiro <43487435+yu-kioo@users.noreply.github.com> Date: Thu, 8 Aug 2024 08:44:16 +0000 Subject: [PATCH 3/9] remove optional signature and align interface name with getter --- src/main/java/org/embulk/input/DatabricksInputPlugin.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/embulk/input/DatabricksInputPlugin.java b/src/main/java/org/embulk/input/DatabricksInputPlugin.java index c8ccbda..ff20378 100755 --- a/src/main/java/org/embulk/input/DatabricksInputPlugin.java +++ b/src/main/java/org/embulk/input/DatabricksInputPlugin.java @@ -43,9 +43,9 @@ public interface DatabricksPluginTask extends PluginTask { @Config("user_agent") @ConfigDefault("{}") - public Optional getUserAgent(); + public UserAgent getUserAgent(); - public interface UserAgentEntry extends Task { + public interface UserAgent extends Task { @Config("product_name") @ConfigDefault("\"unknown\"") public String getProductName(); From 87d849713159b0973ecdd513ae1f704eddf9bb97 Mon Sep 17 00:00:00 2001 From: yuki-tashiro <43487435+yu-kioo@users.noreply.github.com> Date: Thu, 8 Aug 2024 08:55:07 +0000 Subject: [PATCH 4/9] fix wrong code --- .../java/org/embulk/input/DatabricksInputPlugin.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/embulk/input/DatabricksInputPlugin.java b/src/main/java/org/embulk/input/DatabricksInputPlugin.java index ff20378..edf14e2 100755 --- a/src/main/java/org/embulk/input/DatabricksInputPlugin.java +++ b/src/main/java/org/embulk/input/DatabricksInputPlugin.java @@ -43,9 +43,9 @@ public interface DatabricksPluginTask extends PluginTask { @Config("user_agent") @ConfigDefault("{}") - public UserAgent getUserAgent(); + public UserAgentEntry getUserAgentEntry(); - public interface UserAgent extends Task { + public interface UserAgentEntry extends Task { @Config("product_name") @ConfigDefault("\"unknown\"") public String getProductName(); @@ -91,9 +91,9 @@ protected JdbcInputConnection newConnection(PluginTask task) throws SQLException } props.putAll(t.getOptions()); // overwrite UserAgentEntry property if the same property is set in options - if (t.getUserAgent().isPresent()) { - String product_name = t.getUserAgent().get().getProductName(); - String product_version = t.getUserAgent().get().getProductVersion(); + if (t.getUserAgentEntry() != null) { + String product_name = t.getUserAgentEntry().getProductName(); + String product_version = t.getUserAgentEntry().getProductVersion(); props.put("UserAgentEntry", product_name + "/" + product_version); } From 35b629834c5fce90d29ada71eec7168f35fe14f0 Mon Sep 17 00:00:00 2001 From: yuki-tashiro <43487435+yu-kioo@users.noreply.github.com> Date: Wed, 14 Aug 2024 04:29:39 +0000 Subject: [PATCH 5/9] fix condition --- .../embulk/input/DatabricksInputPlugin.java | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/embulk/input/DatabricksInputPlugin.java b/src/main/java/org/embulk/input/DatabricksInputPlugin.java index edf14e2..ed39d37 100755 --- a/src/main/java/org/embulk/input/DatabricksInputPlugin.java +++ b/src/main/java/org/embulk/input/DatabricksInputPlugin.java @@ -44,16 +44,19 @@ public interface DatabricksPluginTask extends PluginTask { @Config("user_agent") @ConfigDefault("{}") public UserAgentEntry getUserAgentEntry(); + } - public interface UserAgentEntry extends Task { - @Config("product_name") - @ConfigDefault("\"unknown\"") - public String getProductName(); + public interface UserAgentEntry extends Task { + String defaultProductName = "unknown"; + String defaultProductVersion = "0.0.0"; - @Config("product_version") - @ConfigDefault("\"0.0.0\"") - public String getProductVersion(); - } + @Config("product_name") + @ConfigDefault("\"unknown\"") + public String getProductName(); + + @Config("product_version") + @ConfigDefault("\"0.0.0\"") + public String getProductVersion(); } @Override @@ -90,12 +93,13 @@ protected JdbcInputConnection newConnection(PluginTask task) throws SQLException props.put("ConnSchema", t.getSchemaName().get()); } props.putAll(t.getOptions()); - // overwrite UserAgentEntry property if the same property is set in options - if (t.getUserAgentEntry() != null) { - String product_name = t.getUserAgentEntry().getProductName(); - String product_version = t.getUserAgentEntry().getProductVersion(); - props.put("UserAgentEntry", product_name + "/" + product_version); + String productName = t.getUserAgentEntry().getProductName(); + String productVersion = t.getUserAgentEntry().getProductVersion(); + boolean isSetUserAgentEntryInOptions = props.containsKey("UserAgentEntry"); + // overwrite UserAgentEntry property if the same property is set in options + if (isSetUserAgentEntry(isSetUserAgentEntryInOptions, productName, productVersion)) { + props.put("UserAgentEntry", productName + "/" + productVersion); } logConnectionProperties(url, props); @@ -132,4 +136,9 @@ protected Schema setupTask(JdbcInputConnection con, PluginTask task) throws SQLE } return super.setupTask(con, task); } + + private boolean isSetUserAgentEntry(boolean isSetUserAgentEntryInOptions, String productName, String productVersion) { + boolean isDefaultUserAgent = (productName.equals(UserAgentEntry.defaultProductName) || productVersion.equals(UserAgentEntry.defaultProductVersion)); + return !(isSetUserAgentEntryInOptions && isDefaultUserAgent); + } } From ac7804e3886cf7cc6033228c49533057f34e5e48 Mon Sep 17 00:00:00 2001 From: yuki-tashiro <43487435+yu-kioo@users.noreply.github.com> Date: Wed, 14 Aug 2024 04:53:48 +0000 Subject: [PATCH 6/9] format --- src/main/java/org/embulk/input/DatabricksInputPlugin.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/embulk/input/DatabricksInputPlugin.java b/src/main/java/org/embulk/input/DatabricksInputPlugin.java index ed39d37..3a5c2e7 100755 --- a/src/main/java/org/embulk/input/DatabricksInputPlugin.java +++ b/src/main/java/org/embulk/input/DatabricksInputPlugin.java @@ -138,7 +138,8 @@ protected Schema setupTask(JdbcInputConnection con, PluginTask task) throws SQLE } private boolean isSetUserAgentEntry(boolean isSetUserAgentEntryInOptions, String productName, String productVersion) { - boolean isDefaultUserAgent = (productName.equals(UserAgentEntry.defaultProductName) || productVersion.equals(UserAgentEntry.defaultProductVersion)); + boolean isDefaultUserAgent = (productName.equals(UserAgentEntry.defaultProductName) + || productVersion.equals(UserAgentEntry.defaultProductVersion)); return !(isSetUserAgentEntryInOptions && isDefaultUserAgent); } } From 817855f624bc0145a02c06574e79685fc2089d6b Mon Sep 17 00:00:00 2001 From: yuki-tashiro <43487435+yu-kioo@users.noreply.github.com> Date: Wed, 14 Aug 2024 04:56:02 +0000 Subject: [PATCH 7/9] format --- src/main/java/org/embulk/input/DatabricksInputPlugin.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/embulk/input/DatabricksInputPlugin.java b/src/main/java/org/embulk/input/DatabricksInputPlugin.java index 3a5c2e7..d31a063 100755 --- a/src/main/java/org/embulk/input/DatabricksInputPlugin.java +++ b/src/main/java/org/embulk/input/DatabricksInputPlugin.java @@ -137,9 +137,11 @@ protected Schema setupTask(JdbcInputConnection con, PluginTask task) throws SQLE return super.setupTask(con, task); } - private boolean isSetUserAgentEntry(boolean isSetUserAgentEntryInOptions, String productName, String productVersion) { - boolean isDefaultUserAgent = (productName.equals(UserAgentEntry.defaultProductName) - || productVersion.equals(UserAgentEntry.defaultProductVersion)); + private boolean isSetUserAgentEntry( + boolean isSetUserAgentEntryInOptions, String productName, String productVersion) { + boolean isDefaultUserAgent = + (productName.equals(UserAgentEntry.defaultProductName) + || productVersion.equals(UserAgentEntry.defaultProductVersion)); return !(isSetUserAgentEntryInOptions && isDefaultUserAgent); } } From c8c429d5117e2ee0d8649c9c0084638393cb782e Mon Sep 17 00:00:00 2001 From: yuki-tashiro <43487435+yu-kioo@users.noreply.github.com> Date: Thu, 15 Aug 2024 02:27:49 +0000 Subject: [PATCH 8/9] remove condition --- .../embulk/input/DatabricksInputPlugin.java | 35 +++++++------------ 1 file changed, 12 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/embulk/input/DatabricksInputPlugin.java b/src/main/java/org/embulk/input/DatabricksInputPlugin.java index d31a063..f41620f 100755 --- a/src/main/java/org/embulk/input/DatabricksInputPlugin.java +++ b/src/main/java/org/embulk/input/DatabricksInputPlugin.java @@ -44,19 +44,19 @@ public interface DatabricksPluginTask extends PluginTask { @Config("user_agent") @ConfigDefault("{}") public UserAgentEntry getUserAgentEntry(); - } - public interface UserAgentEntry extends Task { - String defaultProductName = "unknown"; - String defaultProductVersion = "0.0.0"; + public interface UserAgentEntry extends Task { + String defaultProductName = "unknown"; + String defaultProductVersion = "0.0.0"; - @Config("product_name") - @ConfigDefault("\"unknown\"") - public String getProductName(); + @Config("product_name") + @ConfigDefault("\"unknown\"") + public String getProductName(); - @Config("product_version") - @ConfigDefault("\"0.0.0\"") - public String getProductVersion(); + @Config("product_version") + @ConfigDefault("\"0.0.0\"") + public String getProductVersion(); + } } @Override @@ -94,13 +94,10 @@ protected JdbcInputConnection newConnection(PluginTask task) throws SQLException } props.putAll(t.getOptions()); + // overwrite UserAgentEntry property if the same property is set in options String productName = t.getUserAgentEntry().getProductName(); String productVersion = t.getUserAgentEntry().getProductVersion(); - boolean isSetUserAgentEntryInOptions = props.containsKey("UserAgentEntry"); - // overwrite UserAgentEntry property if the same property is set in options - if (isSetUserAgentEntry(isSetUserAgentEntryInOptions, productName, productVersion)) { - props.put("UserAgentEntry", productName + "/" + productVersion); - } + props.put("UserAgentEntry", productName + "/" + productVersion); logConnectionProperties(url, props); Connection c = DriverManager.getConnection(url, props); @@ -136,12 +133,4 @@ protected Schema setupTask(JdbcInputConnection con, PluginTask task) throws SQLE } return super.setupTask(con, task); } - - private boolean isSetUserAgentEntry( - boolean isSetUserAgentEntryInOptions, String productName, String productVersion) { - boolean isDefaultUserAgent = - (productName.equals(UserAgentEntry.defaultProductName) - || productVersion.equals(UserAgentEntry.defaultProductVersion)); - return !(isSetUserAgentEntryInOptions && isDefaultUserAgent); - } } From a7cff7b500b50a8cdab6b11e8ec9aaeafc926260 Mon Sep 17 00:00:00 2001 From: yuki-tashiro <43487435+yu-kioo@users.noreply.github.com> Date: Thu, 15 Aug 2024 02:31:00 +0000 Subject: [PATCH 9/9] remove cosntant --- src/main/java/org/embulk/input/DatabricksInputPlugin.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/org/embulk/input/DatabricksInputPlugin.java b/src/main/java/org/embulk/input/DatabricksInputPlugin.java index f41620f..9d5de09 100755 --- a/src/main/java/org/embulk/input/DatabricksInputPlugin.java +++ b/src/main/java/org/embulk/input/DatabricksInputPlugin.java @@ -46,9 +46,6 @@ public interface DatabricksPluginTask extends PluginTask { public UserAgentEntry getUserAgentEntry(); public interface UserAgentEntry extends Task { - String defaultProductName = "unknown"; - String defaultProductVersion = "0.0.0"; - @Config("product_name") @ConfigDefault("\"unknown\"") public String getProductName();