diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 41639f697bc7b..c24280170f277 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -46,6 +46,7 @@ 1. Encrypt: Fix merge exception without encrypt rule in database - [#33708](https://github.com/apache/shardingsphere/pull/33708) 1. SQL Parser: Fix mysql parse zone unreserved keyword error - [#33720](https://github.com/apache/shardingsphere/pull/33720) 1. Proxy: Fix BatchUpdateException when execute INSERT INTO ON DUPLICATE KEY UPDATE in proxy adapter - [#33796](https://github.com/apache/shardingsphere/pull/33796) +1. Infra: Fix the issue that ShardingSphere cannot connect to HiveServer2 using remote Hive Metastore Server - [#33837](https://github.com/apache/shardingsphere/pull/33837) ### Change Logs diff --git a/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.cn.md b/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.cn.md index 97fc19a4f6a05..ab5a4e35a6f89 100644 --- a/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.cn.md +++ b/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.cn.md @@ -40,6 +40,17 @@ ShardingSphere 对 HiveServer2 JDBC Driver 的支持位于可选模块中。 hive-service 4.0.1 + + org.apache.hadoop + hadoop-mapreduce-client-core + 3.3.6 + + + * + * + + + ``` @@ -81,6 +92,17 @@ ShardingSphere 对 HiveServer2 JDBC Driver 的支持位于可选模块中。 + + org.apache.hadoop + hadoop-mapreduce-client-core + 3.3.6 + + + * + * + + + ``` @@ -427,8 +449,31 @@ ShardingSphere 仅针对 HiveServer2 `4.0.1` 进行集成测试。 ### Hadoop 限制 用户仅可使用 Hadoop `3.3.6` 来作为 HiveServer2 JDBC Driver `4.0.1` 的底层 Hadoop 依赖。 -HiveServer2 JDBC Driver `4.0.1` 不支持 Hadoop `3.4.1`， -参考 https://github.com/apache/hive/pull/5500 。 +HiveServer2 JDBC Driver `4.0.1` 不支持 Hadoop `3.4.1`， 参考 https://github.com/apache/hive/pull/5500 。 + +对于 HiveServer2 JDBC Driver `org.apache.hive:hive-jdbc:4.0.1` 或 `classifier` 为 `standalone` 的 `org.apache.hive:hive-jdbc:4.0.1`， +实际上并不额外依赖 `org.apache.hadoop:hadoop-mapreduce-client-core:3.3.6`。 + +但 `org.apache.shardingsphere:shardingsphere-infra-database-hive` 的 +`org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader` 会使用 `org.apache.hadoop.hive.conf.HiveConf`， +这进一步使用了 `org.apache.hadoop:hadoop-mapreduce-client-core:3.3.6` 的 `org.apache.hadoop.mapred.JobConf` 类。 + +ShardingSphere 仅需要使用 `org.apache.hadoop.mapred.JobConf` 类， +因此排除 `org.apache.hadoop:hadoop-mapreduce-client-core:3.3.6` 的所有额外依赖是合理行为。 + +```xml + + org.apache.hadoop + hadoop-mapreduce-client-core + 3.3.6 + + + * + * + + + +``` ### SQL 限制 diff --git a/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.en.md b/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.en.md index 3cbcff19cb1aa..b9de7a2799798 100644 --- a/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.en.md +++ b/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.en.md @@ -41,6 +41,17 @@ The possible Maven dependencies are as follows. hive-service 4.0.1 + + org.apache.hadoop + hadoop-mapreduce-client-core + 3.3.6 + + + * + * + + + ``` @@ -83,6 +94,17 @@ The following is an example of a possible configuration, + + org.apache.hadoop + hadoop-mapreduce-client-core + 3.3.6 + + + * + * + + + ``` @@ -433,8 +455,31 @@ Reference https://issues.apache.org/jira/browse/HIVE-28418. ### Hadoop Limitations Users can only use Hadoop `3.3.6` as the underlying Hadoop dependency of HiveServer2 JDBC Driver `4.0.1`. -HiveServer2 JDBC Driver `4.0.1` does not support Hadoop `3.4.1`, -Reference https://github.com/apache/hive/pull/5500. +HiveServer2 JDBC Driver `4.0.1` does not support Hadoop `3.4.1`. Reference https://github.com/apache/hive/pull/5500 . + +For HiveServer2 JDBC Driver `org.apache.hive:hive-jdbc:4.0.1` or `org.apache.hive:hive-jdbc:4.0.1` with `classifier` as `standalone`, +there is actually no additional dependency on `org.apache.hadoop:hadoop-mapreduce-client-core:3.3.6`. + +But `org.apache.shardingsphere:shardingsphere-infra-database-hive`'s +`org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader` uses `org.apache.hadoop.hive.conf.HiveConf`, +which further uses `org.apache.hadoop:hadoop-mapreduce-client-core:3.3.6`'s `org.apache.hadoop.mapred.JobConf` class. + +ShardingSphere only needs to use the `org.apache.hadoop.mapred.JobConf` class, +so it is reasonable to exclude all additional dependencies of `org.apache.hadoop:hadoop-mapreduce-client-core:3.3.6`. + +```xml + + org.apache.hadoop + hadoop-mapreduce-client-core + 3.3.6 + + + * + * + + + +``` ### SQL Limitations diff --git a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.hadoop/hadoop-common/3.3.6/reflect-config.json b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.hadoop/hadoop-common/3.3.6/reflect-config.json new file mode 100644 index 0000000000000..2920c4f51630d --- /dev/null +++ b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.hadoop/hadoop-common/3.3.6/reflect-config.json @@ -0,0 +1,8 @@ +[ +{ + "condition":{"typeReachable":"org.apache.hadoop.security.UserGroupInformation"}, + "name":"org.apache.hadoop.security.UserGroupInformation$UgiMetrics", + "allDeclaredFields": true, + "allDeclaredMethods": true +} +] diff --git a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.hadoop/hadoop-common/3.3.6/resource-config.json b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.hadoop/hadoop-common/3.3.6/resource-config.json index a8e2a9038aa3d..30741259e0273 100644 --- a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.hadoop/hadoop-common/3.3.6/resource-config.json +++ b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.hadoop/hadoop-common/3.3.6/resource-config.json @@ -3,12 +3,6 @@ "includes":[{ "condition":{"typeReachable":"org.apache.hadoop.conf.Configuration"}, "pattern":"\\Qhadoop-site.xml\\E" - }, { - "condition":{"typeReachable":"org.apache.hadoop.conf.Configuration"}, - "pattern":"\\Qcore-default.xml\\E" - }, { - "condition":{"typeReachable":"org.apache.hadoop.conf.Configuration"}, - "pattern":"\\Qcore-site.xml\\E" }]}, "bundles":[] } diff --git a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.hive/hive-jdbc/4.0.1/reflect-config.json b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.hive/hive-jdbc/4.0.1/reflect-config.json new file mode 100644 index 0000000000000..3dd25918d6120 --- /dev/null +++ b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.hive/hive-jdbc/4.0.1/reflect-config.json @@ -0,0 +1,11 @@ +[ +{ + "condition":{"typeReachable":"org.apache.hadoop.hive.metastore.HiveMetaStoreClient"}, + "name":"org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl", + "methods":[{"name":"","parameterTypes":["org.apache.hadoop.conf.Configuration"] }] +}, +{ + "condition":{"typeReachable":"org.apache.hadoop.hive.metastore.conf.MetastoreConf"}, + "name":"org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl" +} +] diff --git a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/proxy-config.json b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/proxy-config.json index d230828e54b7c..f2fb40f70406e 100644 --- a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/proxy-config.json +++ b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/proxy-config.json @@ -3,6 +3,10 @@ "condition":{"typeReachable":"org.apache.shardingsphere.proxy.initializer.BootstrapInitializer"}, "interfaces":["java.sql.Connection"] }, + { + "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"}, + "interfaces":["org.apache.hadoop.metrics2.MetricsSystem$Callback"] + }, { "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.datasource.ShardingSphereDataSource"}, "interfaces":["org.apache.hive.service.rpc.thrift.TCLIService$Iface"] diff --git a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/reflect-config.json b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/reflect-config.json index a6d7095620daf..3988f03fe6983 100644 --- a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/reflect-config.json +++ b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/reflect-config.json @@ -16,7 +16,7 @@ "name":"[Lcom.github.dockerjava.api.model.VolumesFrom;" }, { - "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.connection.DriverDatabaseConnectionManager$$Lambda/0x00007f547fcab290"}, + "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.connection.DriverDatabaseConnectionManager$$Lambda/0x00007f1137caca88"}, "name":"[Lcom.zaxxer.hikari.util.ConcurrentBag$IConcurrentBagEntry;" }, { @@ -75,10 +75,6 @@ "condition":{"typeReachable":"org.apache.shardingsphere.mode.repository.standalone.jdbc.JDBCRepository"}, "name":"[Lcom.zaxxer.hikari.util.ConcurrentBag$IConcurrentBagEntry;" }, -{ - "condition":{"typeReachable":"org.apache.shardingsphere.proxy.backend.connector.jdbc.datasource.JDBCBackendDataSource"}, - "name":"[Lcom.zaxxer.hikari.util.ConcurrentBag$IConcurrentBagEntry;" -}, { "condition":{"typeReachable":"org.apache.shardingsphere.proxy.frontend.postgresql.command.query.extended.Portal"}, "name":"[Lcom.zaxxer.hikari.util.ConcurrentBag$IConcurrentBagEntry;" @@ -99,6 +95,10 @@ "condition":{"typeReachable":"org.apache.shardingsphere.mode.repository.standalone.jdbc.JDBCRepository"}, "name":"[Ljava.sql.Statement;" }, +{ + "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"}, + "name":"com.sun.security.auth.UnixPrincipal" +}, { "condition":{"typeReachable":"org.apache.shardingsphere.infra.util.yaml.YamlEngine"}, "name":"java.beans.PropertyVetoException" @@ -283,6 +283,12 @@ "condition":{"typeReachable":"org.apache.shardingsphere.infra.expr.groovy.GroovyInlineExpressionParser"}, "name":"java.lang.Number" }, +{ + "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"}, + "name":"java.lang.Object", + "allDeclaredFields":true, + "queryAllDeclaredMethods":true +}, { "condition":{"typeReachable":"org.apache.shardingsphere.infra.expr.groovy.GroovyInlineExpressionParser"}, "name":"java.lang.Object", @@ -1105,6 +1111,11 @@ "condition":{"typeReachable":"org.apache.shardingsphere.infra.spi.type.ordered.OrderedSPILoader"}, "name":"org.apache.shardingsphere.encrypt.checker.config.EncryptRuleConfigurationChecker" }, +{ + "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSphereStatement"}, + "name":"org.apache.shardingsphere.encrypt.checker.sql.EncryptSupportedSQLCheckersBuilder", + "methods":[{"name":"","parameterTypes":[] }] +}, { "condition":{"typeReachable":"org.apache.shardingsphere.proxy.backend.connector.StandardDatabaseConnector"}, "name":"org.apache.shardingsphere.encrypt.checker.sql.EncryptSupportedSQLCheckersBuilder", @@ -1307,7 +1318,7 @@ "methods":[{"name":"","parameterTypes":[] }] }, { - "condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.ShardingRuleConfigurationChecker"}, + "condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.config.ShardingRuleConfigurationChecker"}, "name":"org.apache.shardingsphere.infra.algorithm.keygen.snowflake.SnowflakeKeyGenerateAlgorithm", "methods":[{"name":"","parameterTypes":[] }] }, @@ -1322,7 +1333,7 @@ "methods":[{"name":"","parameterTypes":[] }] }, { - "condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.ShardingRuleConfigurationChecker"}, + "condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.config.ShardingRuleConfigurationChecker"}, "name":"org.apache.shardingsphere.infra.algorithm.keygen.uuid.UUIDKeyGenerateAlgorithm", "methods":[{"name":"","parameterTypes":[] }] }, @@ -2069,7 +2080,7 @@ "queryAllDeclaredMethods":true }, { - "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.listener.DatabaseMetaDataChangedListener$$Lambda/0x00007f547fb27268"}, + "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.listener.DatabaseMetaDataChangedListener$$Lambda/0x00007f1137b2dc80"}, "name":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.dispatch.MetaDataChangedSubscriber" }, { @@ -2675,7 +2686,7 @@ "methods":[{"name":"","parameterTypes":[] }] }, { - "condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.ShardingRuleConfigurationChecker"}, + "condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.config.ShardingRuleConfigurationChecker"}, "name":"org.apache.shardingsphere.sharding.algorithm.sharding.classbased.ClassBasedShardingAlgorithm", "methods":[{"name":"","parameterTypes":[] }] }, @@ -2685,7 +2696,7 @@ "methods":[{"name":"","parameterTypes":[] }] }, { - "condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.ShardingRuleConfigurationChecker"}, + "condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.config.ShardingRuleConfigurationChecker"}, "name":"org.apache.shardingsphere.sharding.algorithm.sharding.datetime.AutoIntervalShardingAlgorithm", "methods":[{"name":"","parameterTypes":[] }] }, @@ -2695,7 +2706,7 @@ "methods":[{"name":"","parameterTypes":[] }] }, { - "condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.ShardingRuleConfigurationChecker"}, + "condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.config.ShardingRuleConfigurationChecker"}, "name":"org.apache.shardingsphere.sharding.algorithm.sharding.datetime.IntervalShardingAlgorithm", "methods":[{"name":"","parameterTypes":[] }] }, @@ -2705,7 +2716,7 @@ "methods":[{"name":"","parameterTypes":[] }] }, { - "condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.ShardingRuleConfigurationChecker"}, + "condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.config.ShardingRuleConfigurationChecker"}, "name":"org.apache.shardingsphere.sharding.algorithm.sharding.hint.HintInlineShardingAlgorithm", "methods":[{"name":"","parameterTypes":[] }] }, @@ -2715,7 +2726,7 @@ "methods":[{"name":"","parameterTypes":[] }] }, { - "condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.ShardingRuleConfigurationChecker"}, + "condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.config.ShardingRuleConfigurationChecker"}, "name":"org.apache.shardingsphere.sharding.algorithm.sharding.inline.ComplexInlineShardingAlgorithm", "methods":[{"name":"","parameterTypes":[] }] }, @@ -2725,7 +2736,7 @@ "methods":[{"name":"","parameterTypes":[] }] }, { - "condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.ShardingRuleConfigurationChecker"}, + "condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.config.ShardingRuleConfigurationChecker"}, "name":"org.apache.shardingsphere.sharding.algorithm.sharding.inline.InlineShardingAlgorithm", "methods":[{"name":"","parameterTypes":[] }] }, @@ -2735,7 +2746,7 @@ "methods":[{"name":"","parameterTypes":[] }] }, { - "condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.ShardingRuleConfigurationChecker"}, + "condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.config.ShardingRuleConfigurationChecker"}, "name":"org.apache.shardingsphere.sharding.algorithm.sharding.mod.HashModShardingAlgorithm", "methods":[{"name":"","parameterTypes":[] }] }, @@ -2745,7 +2756,7 @@ "methods":[{"name":"","parameterTypes":[] }] }, { - "condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.ShardingRuleConfigurationChecker"}, + "condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.config.ShardingRuleConfigurationChecker"}, "name":"org.apache.shardingsphere.sharding.algorithm.sharding.mod.ModShardingAlgorithm", "methods":[{"name":"","parameterTypes":[] }] }, @@ -2755,7 +2766,7 @@ "methods":[{"name":"","parameterTypes":[] }] }, { - "condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.ShardingRuleConfigurationChecker"}, + "condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.config.ShardingRuleConfigurationChecker"}, "name":"org.apache.shardingsphere.sharding.algorithm.sharding.range.BoundaryBasedRangeShardingAlgorithm", "methods":[{"name":"","parameterTypes":[] }] }, @@ -2765,7 +2776,7 @@ "methods":[{"name":"","parameterTypes":[] }] }, { - "condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.ShardingRuleConfigurationChecker"}, + "condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.config.ShardingRuleConfigurationChecker"}, "name":"org.apache.shardingsphere.sharding.algorithm.sharding.range.VolumeBasedRangeShardingAlgorithm", "methods":[{"name":"","parameterTypes":[] }] }, @@ -2780,7 +2791,17 @@ }, { "condition":{"typeReachable":"org.apache.shardingsphere.infra.spi.type.ordered.OrderedSPILoader"}, - "name":"org.apache.shardingsphere.sharding.checker.ShardingRuleConfigurationChecker" + "name":"org.apache.shardingsphere.sharding.checker.config.ShardingRuleConfigurationChecker" +}, +{ + "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSphereStatement"}, + "name":"org.apache.shardingsphere.sharding.checker.sql.ShardingSupportedSQLCheckersBuilder", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "condition":{"typeReachable":"org.apache.shardingsphere.proxy.backend.connector.StandardDatabaseConnector"}, + "name":"org.apache.shardingsphere.sharding.checker.sql.ShardingSupportedSQLCheckersBuilder", + "methods":[{"name":"","parameterTypes":[] }] }, { "condition":{"typeReachable":"org.apache.shardingsphere.sqlfederation.engine.SQLFederationEngine"}, @@ -3056,6 +3077,16 @@ "condition":{"typeReachable":"org.apache.shardingsphere.infra.util.yaml.YamlEngine"}, "name":"org.apache.shardingsphere.sharding.yaml.engine.construct.NoneShardingStrategyConfigurationYamlConstruct" }, +{ + "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSphereStatement"}, + "name":"org.apache.shardingsphere.single.checker.sql.SingleSupportedSQLCheckersBuilder", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "condition":{"typeReachable":"org.apache.shardingsphere.proxy.backend.connector.StandardDatabaseConnector"}, + "name":"org.apache.shardingsphere.single.checker.sql.SingleSupportedSQLCheckersBuilder", + "methods":[{"name":"","parameterTypes":[] }] +}, { "condition":{"typeReachable":"org.apache.shardingsphere.sqlfederation.engine.SQLFederationEngine"}, "name":"org.apache.shardingsphere.single.decider.SingleSQLFederationDecider" @@ -3328,6 +3359,11 @@ "name":"org.apache.shardingsphere.sql.parser.statement.clickhouse.dml.ClickHouseInsertStatement", "methods":[{"name":"","parameterTypes":[] }] }, +{ + "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSpherePreparedStatement"}, + "name":"org.apache.shardingsphere.sql.parser.statement.clickhouse.dml.ClickHouseSelectStatement", + "methods":[{"name":"","parameterTypes":[] }] +}, { "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSphereStatement"}, "name":"org.apache.shardingsphere.sql.parser.statement.clickhouse.dml.ClickHouseSelectStatement", @@ -3338,6 +3374,11 @@ "name":"org.apache.shardingsphere.sql.parser.statement.hive.dml.HiveInsertStatement", "methods":[{"name":"","parameterTypes":[] }] }, +{ + "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSpherePreparedStatement"}, + "name":"org.apache.shardingsphere.sql.parser.statement.hive.dml.HiveSelectStatement", + "methods":[{"name":"","parameterTypes":[] }] +}, { "condition":{"typeReachable":"org.apache.shardingsphere.proxy.backend.handler.ProxyBackendHandlerFactory"}, "name":"org.apache.shardingsphere.sql.parser.statement.mysql.dml.MySQLDeleteStatement", @@ -3353,6 +3394,11 @@ "name":"org.apache.shardingsphere.sql.parser.statement.mysql.dml.MySQLInsertStatement", "methods":[{"name":"","parameterTypes":[] }] }, +{ + "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSpherePreparedStatement"}, + "name":"org.apache.shardingsphere.sql.parser.statement.mysql.dml.MySQLSelectStatement", + "methods":[{"name":"","parameterTypes":[] }] +}, { "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSphereStatement"}, "name":"org.apache.shardingsphere.sql.parser.statement.mysql.dml.MySQLSelectStatement", @@ -3368,6 +3414,11 @@ "name":"org.apache.shardingsphere.sql.parser.statement.opengauss.dml.OpenGaussInsertStatement", "methods":[{"name":"","parameterTypes":[] }] }, +{ + "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSpherePreparedStatement"}, + "name":"org.apache.shardingsphere.sql.parser.statement.opengauss.dml.OpenGaussSelectStatement", + "methods":[{"name":"","parameterTypes":[] }] +}, { "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSphereStatement"}, "name":"org.apache.shardingsphere.sql.parser.statement.opengauss.dml.OpenGaussSelectStatement", @@ -3388,6 +3439,11 @@ "name":"org.apache.shardingsphere.sql.parser.statement.postgresql.dml.PostgreSQLInsertStatement", "methods":[{"name":"","parameterTypes":[] }] }, +{ + "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSpherePreparedStatement"}, + "name":"org.apache.shardingsphere.sql.parser.statement.postgresql.dml.PostgreSQLSelectStatement", + "methods":[{"name":"","parameterTypes":[] }] +}, { "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSphereStatement"}, "name":"org.apache.shardingsphere.sql.parser.statement.postgresql.dml.PostgreSQLSelectStatement", @@ -3403,6 +3459,11 @@ "name":"org.apache.shardingsphere.sql.parser.statement.sqlserver.dml.SQLServerInsertStatement", "methods":[{"name":"","parameterTypes":[] }] }, +{ + "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSpherePreparedStatement"}, + "name":"org.apache.shardingsphere.sql.parser.statement.sqlserver.dml.SQLServerSelectStatement", + "methods":[{"name":"","parameterTypes":[] }] +}, { "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSphereStatement"}, "name":"org.apache.shardingsphere.sql.parser.statement.sqlserver.dml.SQLServerSelectStatement", @@ -3588,10 +3649,5 @@ { "condition":{"typeReachable":"org.apache.shardingsphere.proxy.initializer.BootstrapInitializer"}, "name":"org.apache.shardingsphere.transaction.yaml.config.YamlTransactionRuleConfigurationCustomizer" -}, -{ - "condition":{"typeReachable":"org.apache.shardingsphere.infra.executor.kernel.ExecutorEngine"}, - "name":"sun.security.provider.SecureRandom", - "methods":[{"name":"","parameterTypes":[] }, {"name":"","parameterTypes":["java.security.SecureRandomParameters"] }] } ] \ No newline at end of file diff --git a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/resource-config.json b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/resource-config.json index c42c1ecf3f6d5..4355d10bc0b25 100644 --- a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/resource-config.json +++ b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/resource-config.json @@ -79,7 +79,7 @@ "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.connection.DriverDatabaseConnectionManager"}, "pattern":"\\QMETA-INF/services/com.clickhouse.client.ClickHouseClient\\E" }, { - "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.connection.DriverDatabaseConnectionManager$$Lambda/0x00007f547fb36cb8"}, + "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.connection.DriverDatabaseConnectionManager$$Lambda/0x00007f1137b39f20"}, "pattern":"\\QMETA-INF/services/com.clickhouse.client.ClickHouseClient\\E" }, { "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.datasource.ShardingSphereDataSource"}, @@ -114,6 +114,9 @@ }, { "condition":{"typeReachable":"org.apache.shardingsphere.db.protocol.mysql.constant.MySQLCharacterSet"}, "pattern":"\\QMETA-INF/services/java.nio.charset.spi.CharsetProvider\\E" + }, { + "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"}, + "pattern":"\\QMETA-INF/services/java.nio.charset.spi.CharsetProvider\\E" }, { "condition":{"typeReachable":"org.apache.shardingsphere.proxy.backend.postgresql.handler.admin.executor.variable.charset.PostgreSQLCharacterSets"}, "pattern":"\\QMETA-INF/services/java.nio.charset.spi.CharsetProvider\\E" @@ -123,12 +126,18 @@ }, { "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.datasource.ShardingSphereDataSource"}, "pattern":"\\QMETA-INF/services/javax.xml.datatype.DatatypeFactory\\E" + }, { + "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"}, + "pattern":"\\QMETA-INF/services/javax.xml.parsers.DocumentBuilderFactory\\E" }, { "condition":{"typeReachable":"org.apache.shardingsphere.mode.repository.standalone.jdbc.sql.JDBCRepositorySQLLoader"}, "pattern":"\\QMETA-INF/services/javax.xml.stream.XMLInputFactory\\E" }, { "condition":{"typeReachable":"org.apache.shardingsphere.mode.repository.standalone.jdbc.sql.JDBCRepositorySQLLoader"}, "pattern":"\\QMETA-INF/services/javax.xml.stream.XMLOutputFactory\\E" + }, { + "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"}, + "pattern":"\\QMETA-INF/services/javax.xml.transform.TransformerFactory\\E" }, { "condition":{"typeReachable":"org.apache.shardingsphere.transaction.base.seata.at.SeataATShardingSphereTransactionManager"}, "pattern":"\\QMETA-INF/services/org.apache.seata.config.ConfigurationProvider\\E" @@ -235,7 +244,7 @@ "condition":{"typeReachable":"org.apache.shardingsphere.proxy.backend.handler.ProxyBackendHandlerFactory"}, "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.executor.checker.SQLExecutionChecker\\E" }, { - "condition":{"typeReachable":"org.apache.shardingsphere.infra.executor.kernel.ExecutorEngine"}, + "condition":{"typeReachable":"org.apache.shardingsphere.proxy.frontend.mysql.command.query.text.query.MySQLComQueryPacketExecutor"}, "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.executor.sql.hook.SQLExecutionHook\\E" }, { "condition":{"typeReachable":"org.apache.shardingsphere.infra.executor.sql.prepare.AbstractExecutionPrepareEngine"}, @@ -387,15 +396,42 @@ }, { "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.datasource.ShardingSphereDataSource"}, "pattern":"\\Qcontainer-license-acceptance.txt\\E" + }, { + "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"}, + "pattern":"\\Qcore-default.xml\\E" + }, { + "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"}, + "pattern":"\\Qcore-site.xml\\E" }, { "condition":{"typeReachable":"org.apache.shardingsphere.infra.autogen.version.ShardingSphereVersion"}, "pattern":"\\Qcurrent-git-commit.properties\\E" + }, { + "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"}, + "pattern":"\\Qhive-default.xml\\E" + }, { + "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"}, + "pattern":"\\Qhive-site.xml\\E" + }, { + "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"}, + "pattern":"\\Qhivemetastore-site.xml\\E" + }, { + "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"}, + "pattern":"\\Qhiveserver2-site.xml\\E" }, { "condition":{"typeReachable":"org.apache.shardingsphere.transaction.xa.atomikos.manager.AtomikosTransactionManagerProvider"}, "pattern":"\\Qjta.properties\\E" }, { "condition":{"typeReachable":"org.apache.shardingsphere.transaction.base.seata.at.SeataATShardingSphereTransactionManager"}, "pattern":"\\Qlib/sqlparser/druid.jar\\E" + }, { + "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"}, + "pattern":"\\Qmapred-default.xml\\E" + }, { + "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"}, + "pattern":"\\Qmapred-site.xml\\E" + }, { + "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"}, + "pattern":"\\Qorg/apache/hadoop/hive/conf/HiveConf.class\\E" }, { "condition":{"typeReachable":"org.apache.shardingsphere.proxy.initializer.BootstrapInitializer"}, "pattern":"\\Qorg/h2/util/data.zip\\E" @@ -1968,6 +2004,9 @@ }, { "condition":{"typeReachable":"org.apache.shardingsphere.infra.url.classpath.ClassPathURLLoader"}, "pattern":"\\Qtest-native/yaml/jdbc/databases/hive/iceberg.yaml\\E" + }, { + "condition":{"typeReachable":"org.apache.shardingsphere.infra.url.classpath.ClassPathURLLoader"}, + "pattern":"\\Qtest-native/yaml/jdbc/databases/hive/standalone-hms.yaml\\E" }, { "condition":{"typeReachable":"org.apache.shardingsphere.infra.url.classpath.ClassPathURLLoader"}, "pattern":"\\Qtest-native/yaml/jdbc/databases/hive/zsd.yaml\\E" @@ -2022,9 +2061,36 @@ }, { "condition":{"typeReachable":"org.apache.shardingsphere.mode.repository.cluster.etcd.EtcdRepository"}, "pattern":"\\Qvertx-default-jul-logging.properties\\E" + }, { + "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"}, + "pattern":"\\Qyarn-default.xml\\E" + }, { + "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"}, + "pattern":"\\Qyarn-site.xml\\E" + }, { + "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"}, + "pattern":"java.xml:\\Qcom/sun/org/apache/xml/internal/serializer/Encodings.properties\\E" + }, { + "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"}, + "pattern":"java.xml:\\Qcom/sun/org/apache/xml/internal/serializer/XMLEntities.properties\\E" + }, { + "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"}, + "pattern":"java.xml:\\Qcom/sun/org/apache/xml/internal/serializer/XMLEntities_zh.properties\\E" + }, { + "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"}, + "pattern":"java.xml:\\Qcom/sun/org/apache/xml/internal/serializer/XMLEntities_zh_CN.properties\\E" + }, { + "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"}, + "pattern":"java.xml:\\Qcom/sun/org/apache/xml/internal/serializer/XMLEntities_zh_Hans.properties\\E" + }, { + "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"}, + "pattern":"java.xml:\\Qcom/sun/org/apache/xml/internal/serializer/XMLEntities_zh_Hans_CN.properties\\E" }]}, "bundles":[{ "name":"com.microsoft.sqlserver.jdbc.SQLServerResource", "locales":["zh-CN"] + }, { + "name":"com.sun.org.apache.xml.internal.serializer.XMLEntities", + "locales":["zh-CN"] }] } \ No newline at end of file diff --git a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/shardingsphere-infra-reachability-metadata/reflect-config.json b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/shardingsphere-infra-reachability-metadata/reflect-config.json index 9bb9fe20748d3..4597bf4a5967c 100644 --- a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/shardingsphere-infra-reachability-metadata/reflect-config.json +++ b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/shardingsphere-infra-reachability-metadata/reflect-config.json @@ -332,11 +332,6 @@ "name":"org.apache.shardingsphere.infra.yaml.data.pojo.YamlShardingSphereRowData", "allPublicMethods": true }, -{ - "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSpherePreparedStatement"}, - "name":"org.apache.shardingsphere.sql.parser.statement.hive.dml.HiveSelectStatement", - "methods":[{"name":"","parameterTypes":[] }] -}, { "condition":{"typeReachable":"sun.security.provider.SecureRandom"}, "name":"sun.security.provider.SecureRandom", diff --git a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/shardingsphere-infra-reachability-metadata/resource-config.json b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/shardingsphere-infra-reachability-metadata/resource-config.json index 8f35719b5074e..4b68f0d1a8746 100644 --- a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/shardingsphere-infra-reachability-metadata/resource-config.json +++ b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/shardingsphere-infra-reachability-metadata/resource-config.json @@ -21,6 +21,6 @@ "locales":["en"] }, { "name":"org.opengauss.translation.messages", - "locales":["zh"] + "locales":["en", "zh"] }] } diff --git a/pom.xml b/pom.xml index 01d49458c9385..b5c5a366841d6 100644 --- a/pom.xml +++ b/pom.xml @@ -129,6 +129,7 @@ 0.6.3 4.0.1 1.5.0 + 3.3.6 0.288.1 4.0.3 diff --git a/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/PipelineContainerComposer.java b/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/PipelineContainerComposer.java index 2a9251c52d4c4..8b306e2f61a51 100644 --- a/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/PipelineContainerComposer.java +++ b/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/PipelineContainerComposer.java @@ -61,6 +61,7 @@ import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -392,7 +393,7 @@ public void proxyExecuteWithLog(final String sql, final int sleepSeconds) throws connection.createStatement().execute(each); } } - Awaitility.await().pollDelay(Math.max(sleepSeconds, 0L), TimeUnit.SECONDS).until(() -> true); + Awaitility.await().timeout(Duration.ofMinutes(1L)).pollDelay(Math.max(sleepSeconds, 0L), TimeUnit.SECONDS).until(() -> true); } /** diff --git a/test/native/native-image-filter/extra-filter.json b/test/native/native-image-filter/extra-filter.json index c4df37dbd7733..24857b0391c25 100644 --- a/test/native/native-image-filter/extra-filter.json +++ b/test/native/native-image-filter/extra-filter.json @@ -6,6 +6,7 @@ {"excludeClasses": "com.**"}, {"includeClasses": "com.oracle.svm.core.**"}, {"includeClasses": "com.sun.management.**"}, + {"includeClasses": "com.sun.security.auth.UnixPrincipal"}, {"excludeClasses": "ch.qos.logback.**"}, {"excludeClasses": "groovy.**"}, {"excludeClasses": "io.**"}, diff --git a/test/native/pom.xml b/test/native/pom.xml index 41a0d8be2796d..61b583aabeb56 100644 --- a/test/native/pom.xml +++ b/test/native/pom.xml @@ -190,6 +190,17 @@ + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + + + * + * + + + org.testcontainers junit-jupiter diff --git a/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/hive/StandaloneMetastoreTest.java b/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/hive/StandaloneMetastoreTest.java new file mode 100644 index 0000000000000..3b9afd28e15a8 --- /dev/null +++ b/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/hive/StandaloneMetastoreTest.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shardingsphere.test.natived.jdbc.databases.hive; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import org.apache.shardingsphere.test.natived.commons.TestShardingService; +import org.awaitility.Awaitility; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledInNativeImage; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import javax.sql.DataSource; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Duration; +import java.util.Properties; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; + +@SuppressWarnings({"SqlDialectInspection", "SqlNoDataSourceInspection", "resource"}) +@EnabledInNativeImage +@Testcontainers +class StandaloneMetastoreTest { + + private static final Network NETWORK = Network.newNetwork(); + + @Container + public static final GenericContainer HMS_CONTAINER = new GenericContainer<>("apache/hive:4.0.1") + .withEnv("SERVICE_NAME", "metastore") + .withNetwork(NETWORK) + .withNetworkAliases("metastore"); + + @Container + public static final GenericContainer HS2_CONTAINER = new GenericContainer<>("apache/hive:4.0.1") + .withEnv("SERVICE_NAME", "hiveserver2") + .withEnv("SERVICE_OPTS", "-Dhive.metastore.uris=thrift://metastore:9083") + .withNetwork(NETWORK) + .withExposedPorts(10000) + .dependsOn(HMS_CONTAINER); + + private static final String SYSTEM_PROP_KEY_PREFIX = "fixture.test-native.yaml.database.hive.hms."; + + // Due to https://issues.apache.org/jira/browse/HIVE-28317 , the `initFile` parameter of HiveServer2 JDBC Driver must be an absolute path. + private static final String ABSOLUTE_PATH = Paths.get("src/test/resources/test-native/sql/test-native-databases-hive-iceberg.sql").toAbsolutePath().toString(); + + private String jdbcUrlPrefix; + + @BeforeAll + static void beforeAll() { + assertThat(System.getProperty(SYSTEM_PROP_KEY_PREFIX + "ds0.jdbc-url"), is(nullValue())); + assertThat(System.getProperty(SYSTEM_PROP_KEY_PREFIX + "ds1.jdbc-url"), is(nullValue())); + assertThat(System.getProperty(SYSTEM_PROP_KEY_PREFIX + "ds2.jdbc-url"), is(nullValue())); + } + + @AfterAll + static void afterAll() { + NETWORK.close(); + System.clearProperty(SYSTEM_PROP_KEY_PREFIX + "ds0.jdbc-url"); + System.clearProperty(SYSTEM_PROP_KEY_PREFIX + "ds1.jdbc-url"); + System.clearProperty(SYSTEM_PROP_KEY_PREFIX + "ds2.jdbc-url"); + } + + @Test + void assertShardingInLocalTransactions() throws SQLException { + jdbcUrlPrefix = "jdbc:hive2://localhost:" + HS2_CONTAINER.getMappedPort(10000) + "/"; + DataSource dataSource = createDataSource(); + TestShardingService testShardingService = new TestShardingService(dataSource); + testShardingService.processSuccessInHive(); + } + + private Connection openConnection() throws SQLException { + Properties props = new Properties(); + return DriverManager.getConnection(jdbcUrlPrefix, props); + } + + private DataSource createDataSource() throws SQLException { + Awaitility.await().atMost(Duration.ofMinutes(1L)).ignoreExceptions().until(() -> { + openConnection().close(); + return true; + }); + try ( + Connection connection = openConnection(); + Statement statement = connection.createStatement()) { + statement.executeUpdate("CREATE DATABASE demo_ds_0"); + statement.executeUpdate("CREATE DATABASE demo_ds_1"); + statement.executeUpdate("CREATE DATABASE demo_ds_2"); + } + HikariConfig config = new HikariConfig(); + config.setDriverClassName("org.apache.shardingsphere.driver.ShardingSphereDriver"); + config.setJdbcUrl("jdbc:shardingsphere:classpath:test-native/yaml/jdbc/databases/hive/standalone-hms.yaml?placeholder-type=system_props"); + System.setProperty(SYSTEM_PROP_KEY_PREFIX + "ds0.jdbc-url", jdbcUrlPrefix + "demo_ds_0" + ";initFile=" + ABSOLUTE_PATH); + System.setProperty(SYSTEM_PROP_KEY_PREFIX + "ds1.jdbc-url", jdbcUrlPrefix + "demo_ds_1" + ";initFile=" + ABSOLUTE_PATH); + System.setProperty(SYSTEM_PROP_KEY_PREFIX + "ds2.jdbc-url", jdbcUrlPrefix + "demo_ds_2" + ";initFile=" + ABSOLUTE_PATH); + return new HikariDataSource(config); + } +} diff --git a/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/hive/ZookeeperServiceDiscoveryTest.java b/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/hive/ZookeeperServiceDiscoveryTest.java index ee9effbbc38ae..9bcaf48dc3a16 100644 --- a/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/hive/ZookeeperServiceDiscoveryTest.java +++ b/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/hive/ZookeeperServiceDiscoveryTest.java @@ -71,7 +71,7 @@ class ZookeeperServiceDiscoveryTest { */ @SuppressWarnings("unused") @Container - private static final GenericContainer HIVE_SERVER2_1_CONTAINER = new FixedHostPortGenericContainer<>("apache/hive:4.0.1") + private static final GenericContainer HS2_1_CONTAINER = new FixedHostPortGenericContainer<>("apache/hive:4.0.1") .withNetwork(NETWORK) .withEnv("SERVICE_NAME", "hiveserver2") .withEnv("SERVICE_OPTS", "-Dhive.server2.support.dynamic.service.discovery=true" + " " @@ -116,10 +116,10 @@ void assertShardingInLocalTransactions() throws SQLException { DataSource dataSource = createDataSource(); TestShardingService testShardingService = new TestShardingService(dataSource); testShardingService.processSuccessInHive(); - HIVE_SERVER2_1_CONTAINER.stop(); + HS2_1_CONTAINER.stop(); int randomPortSecond = InstanceSpec.getRandomPort(); try ( - GenericContainer hiveServer2SecondContainer = new FixedHostPortGenericContainer<>("apache/hive:4.0.1") + GenericContainer hs2SecondContainer = new FixedHostPortGenericContainer<>("apache/hive:4.0.1") .withNetwork(NETWORK) .withEnv("SERVICE_NAME", "hiveserver2") .withEnv("SERVICE_OPTS", "-Dhive.server2.support.dynamic.service.discovery=true" + " " @@ -128,8 +128,8 @@ void assertShardingInLocalTransactions() throws SQLException { + "-Dhive.server2.thrift.port=" + randomPortSecond) .withFixedExposedPort(randomPortSecond, randomPortSecond) .dependsOn(ZOOKEEPER_CONTAINER)) { - hiveServer2SecondContainer.start(); - extracted(hiveServer2SecondContainer.getMappedPort(randomPortSecond)); + hs2SecondContainer.start(); + extracted(hs2SecondContainer.getMappedPort(randomPortSecond)); testShardingService.processSuccessInHive(); } } @@ -140,7 +140,7 @@ private Connection openConnection() throws SQLException { } private DataSource createDataSource() throws SQLException { - extracted(HIVE_SERVER2_1_CONTAINER.getMappedPort(RANDOM_PORT_FIRST)); + extracted(HS2_1_CONTAINER.getMappedPort(RANDOM_PORT_FIRST)); HikariConfig config = new HikariConfig(); config.setDriverClassName("org.apache.shardingsphere.driver.ShardingSphereDriver"); config.setJdbcUrl("jdbc:shardingsphere:classpath:test-native/yaml/jdbc/databases/hive/zsd.yaml?placeholder-type=system_props"); diff --git a/test/native/src/test/resources/META-INF/native-image/shardingsphere-test-native-test-metadata/reflect-config.json b/test/native/src/test/resources/META-INF/native-image/shardingsphere-test-native-test-metadata/reflect-config.json index ca72894c15a1b..44c372fa4dd88 100644 --- a/test/native/src/test/resources/META-INF/native-image/shardingsphere-test-native-test-metadata/reflect-config.json +++ b/test/native/src/test/resources/META-INF/native-image/shardingsphere-test-native-test-metadata/reflect-config.json @@ -168,5 +168,13 @@ "allDeclaredConstructors": true, "allDeclaredMethods": true, "allPublicMethods": true +}, +{ + "condition":{"typeReachable":"org.apache.shardingsphere.test.natived.jdbc.databases.hive.StandaloneMetastoreTest"}, + "name":"org.apache.shardingsphere.test.natived.jdbc.databases.hive.StandaloneMetastoreTest", + "allDeclaredFields": true, + "allDeclaredConstructors": true, + "allDeclaredMethods": true, + "allPublicMethods": true } ] diff --git a/test/native/src/test/resources/test-native/yaml/jdbc/databases/hive/standalone-hms.yaml b/test/native/src/test/resources/test-native/yaml/jdbc/databases/hive/standalone-hms.yaml new file mode 100644 index 0000000000000..4ee01badefc24 --- /dev/null +++ b/test/native/src/test/resources/test-native/yaml/jdbc/databases/hive/standalone-hms.yaml @@ -0,0 +1,64 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +dataSources: + ds_0: + dataSourceClassName: com.zaxxer.hikari.HikariDataSource + driverClassName: org.apache.hive.jdbc.HiveDriver + jdbcUrl: $${fixture.test-native.yaml.database.hive.hms.ds0.jdbc-url::} + ds_1: + dataSourceClassName: com.zaxxer.hikari.HikariDataSource + driverClassName: org.apache.hive.jdbc.HiveDriver + jdbcUrl: $${fixture.test-native.yaml.database.hive.hms.ds1.jdbc-url::} + ds_2: + dataSourceClassName: com.zaxxer.hikari.HikariDataSource + driverClassName: org.apache.hive.jdbc.HiveDriver + jdbcUrl: $${fixture.test-native.yaml.database.hive.hms.ds2.jdbc-url::} + +rules: +- !SHARDING + tables: + t_order: + actualDataNodes: ds_0.t_order, ds_1.t_order, ds_2.t_order + keyGenerateStrategy: + column: order_id + keyGeneratorName: snowflake + t_order_item: + actualDataNodes: ds_0.t_order_item, ds_1.t_order_item, ds_2.t_order_item + keyGenerateStrategy: + column: order_item_id + keyGeneratorName: snowflake + defaultDatabaseStrategy: + standard: + shardingColumn: user_id + shardingAlgorithmName: inline + shardingAlgorithms: + inline: + type: CLASS_BASED + props: + strategy: STANDARD + algorithmClassName: org.apache.shardingsphere.test.natived.commons.algorithm.ClassBasedInlineShardingAlgorithmFixture + keyGenerators: + snowflake: + type: SNOWFLAKE + auditors: + sharding_key_required_auditor: + type: DML_SHARDING_CONDITIONS + +- !BROADCAST + tables: + - t_address