From ab2aead57db584f9ecd736b2d648d7631420a977 Mon Sep 17 00:00:00 2001 From: jiangML <1060319118@qq.com> Date: Tue, 24 Dec 2024 14:58:19 +0800 Subject: [PATCH 1/3] Refactor export and import metadata --- .../ExportDatabaseConfigurationExecutor.java | 13 +- .../ral/queryable/ExportMetaDataExecutor.java | 93 +------------ .../ImportDatabaseConfigurationExecutor.java | 16 ++- .../ral/updatable/ImportMetaDataExecutor.java | 51 ++----- .../util/ClusterExportMetaDataGenerator.java | 125 ++++++++++++++++++ .../util/DatabaseExportMetaDataGenerator.java | 108 +++++++++++++++ .../proxy/backend/util/ExportUtils.java | 67 ---------- .../backend/util/MetaDataImportExecutor.java | 96 ++++++++++++++ 8 files changed, 362 insertions(+), 207 deletions(-) create mode 100644 proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/ClusterExportMetaDataGenerator.java create mode 100644 proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/DatabaseExportMetaDataGenerator.java create mode 100644 proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/MetaDataImportExecutor.java diff --git a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportDatabaseConfigurationExecutor.java b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportDatabaseConfigurationExecutor.java index b95585bb75733..b7e482aeb57fb 100644 --- a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportDatabaseConfigurationExecutor.java +++ b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportDatabaseConfigurationExecutor.java @@ -24,6 +24,7 @@ import org.apache.shardingsphere.infra.merge.result.impl.local.LocalDataQueryResultRow; import org.apache.shardingsphere.infra.metadata.database.ShardingSphereDatabase; import org.apache.shardingsphere.mode.manager.ContextManager; +import org.apache.shardingsphere.proxy.backend.util.DatabaseExportMetaDataGenerator; import org.apache.shardingsphere.proxy.backend.util.ExportUtils; import java.util.Collection; @@ -44,13 +45,13 @@ public Collection getColumnNames(final ExportDatabaseConfigurationStatem @Override public Collection getRows(final ExportDatabaseConfigurationStatement sqlStatement, final ContextManager contextManager) { - String exportedData = ExportUtils.generateExportDatabaseData(database); - if (!sqlStatement.getFilePath().isPresent()) { - return Collections.singleton(new LocalDataQueryResultRow(exportedData)); + String exportedData = new DatabaseExportMetaDataGenerator(database).generateYAMLFormat(); + if (sqlStatement.getFilePath().isPresent()) { + String filePath = sqlStatement.getFilePath().get(); + ExportUtils.exportToFile(filePath, exportedData); + return Collections.singleton(new LocalDataQueryResultRow(String.format("Successfully exported to: '%s'", filePath))); } - String filePath = sqlStatement.getFilePath().get(); - ExportUtils.exportToFile(filePath, exportedData); - return Collections.singleton(new LocalDataQueryResultRow(String.format("Successfully exported to: '%s'", filePath))); + return Collections.singleton(new LocalDataQueryResultRow(exportedData)); } @Override diff --git a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportMetaDataExecutor.java b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportMetaDataExecutor.java index cda276c5496f9..cad9e0a6785a7 100644 --- a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportMetaDataExecutor.java +++ b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportMetaDataExecutor.java @@ -20,32 +20,15 @@ import org.apache.commons.codec.binary.Base64; import org.apache.shardingsphere.distsql.handler.engine.query.DistSQLQueryExecutor; import org.apache.shardingsphere.distsql.statement.ral.queryable.export.ExportMetaDataStatement; -import org.apache.shardingsphere.globalclock.provider.GlobalClockProvider; -import org.apache.shardingsphere.globalclock.rule.GlobalClockRule; -import org.apache.shardingsphere.infra.config.rule.RuleConfiguration; import org.apache.shardingsphere.infra.merge.result.impl.local.LocalDataQueryResultRow; -import org.apache.shardingsphere.infra.metadata.ShardingSphereMetaData; -import org.apache.shardingsphere.infra.metadata.database.ShardingSphereDatabase; -import org.apache.shardingsphere.infra.spi.type.ordered.OrderedSPILoader; -import org.apache.shardingsphere.infra.util.json.JsonUtils; -import org.apache.shardingsphere.infra.util.yaml.YamlEngine; -import org.apache.shardingsphere.infra.yaml.config.swapper.rule.YamlRuleConfigurationSwapper; import org.apache.shardingsphere.mode.manager.ContextManager; -import org.apache.shardingsphere.mode.metadata.decorator.RuleConfigurationPersistDecorateEngine; -import org.apache.shardingsphere.proxy.backend.context.ProxyContext; -import org.apache.shardingsphere.proxy.backend.distsql.export.ExportedClusterInfo; -import org.apache.shardingsphere.proxy.backend.distsql.export.ExportedMetaData; -import org.apache.shardingsphere.proxy.backend.distsql.export.ExportedSnapshotInfo; +import org.apache.shardingsphere.proxy.backend.util.ClusterExportMetaDataGenerator; import org.apache.shardingsphere.proxy.backend.util.ExportUtils; import java.time.LocalDateTime; import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Properties; /** * Export metadata executor. @@ -59,80 +42,14 @@ public Collection getColumnNames(final ExportMetaDataStatement sqlStatem @Override public Collection getRows(final ExportMetaDataStatement sqlStatement, final ContextManager contextManager) { - String exportedData = generateExportData(contextManager.getMetaDataContexts().getMetaData()); + String exportedData = new ClusterExportMetaDataGenerator(contextManager).generateJsonFormat(); + String instanceId = contextManager.getComputeNodeInstanceContext().getInstance().getMetaData().getId(); if (sqlStatement.getFilePath().isPresent()) { String filePath = sqlStatement.getFilePath().get(); ExportUtils.exportToFile(filePath, exportedData); - return Collections.singleton(new LocalDataQueryResultRow(contextManager.getComputeNodeInstanceContext().getInstance().getMetaData().getId(), LocalDateTime.now(), - String.format("Successfully exported to：'%s'", filePath))); - } - return Collections.singleton(new LocalDataQueryResultRow( - contextManager.getComputeNodeInstanceContext().getInstance().getMetaData().getId(), LocalDateTime.now(), Base64.encodeBase64String(exportedData.getBytes()))); - } - - private String generateExportData(final ShardingSphereMetaData metaData) { - ProxyContext proxyContext = ProxyContext.getInstance(); - ExportedMetaData exportedMetaData = new ExportedMetaData(); - exportedMetaData.setDatabases(getDatabases(proxyContext)); - exportedMetaData.setProps(generatePropsData(metaData.getProps().getProps())); - RuleConfigurationPersistDecorateEngine ruleConfigPersistDecorateEngine = - new RuleConfigurationPersistDecorateEngine(ProxyContext.getInstance().getContextManager().getComputeNodeInstanceContext()); - Collection ruleConfigs = ruleConfigPersistDecorateEngine.decorate(metaData.getGlobalRuleMetaData().getConfigurations()); - exportedMetaData.setRules(generateRulesData(ruleConfigs)); - ExportedClusterInfo exportedClusterInfo = new ExportedClusterInfo(); - exportedClusterInfo.setMetaData(exportedMetaData); - generateSnapshotInfo(metaData, exportedClusterInfo); - return JsonUtils.toJsonString(exportedClusterInfo); - } - - private Map getDatabases(final ProxyContext proxyContext) { - Collection databaseNames = proxyContext.getAllDatabaseNames(); - Map result = new LinkedHashMap<>(databaseNames.size(), 1F); - for (String each : databaseNames) { - ShardingSphereDatabase database = proxyContext.getContextManager().getDatabase(each); - if (database.getResourceMetaData().getAllInstanceDataSourceNames().isEmpty()) { - continue; - } - result.put(each, ExportUtils.generateExportDatabaseData(database)); - } - return result; - } - - private String generatePropsData(final Properties props) { - if (props.isEmpty()) { - return ""; - } - StringBuilder result = new StringBuilder(); - result.append("props:").append(System.lineSeparator()); - props.forEach((key, value) -> { - if (null != value && !"".equals(value)) { - result.append(" ").append(key).append(": ").append(value).append(System.lineSeparator()); - } - }); - return result.toString(); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - private String generateRulesData(final Collection rules) { - if (rules.isEmpty()) { - return ""; - } - StringBuilder result = new StringBuilder(); - result.append("rules:").append(System.lineSeparator()); - for (Entry entry : OrderedSPILoader.getServices(YamlRuleConfigurationSwapper.class, rules).entrySet()) { - result.append(YamlEngine.marshal(Collections.singletonList(entry.getValue().swapToYamlConfiguration(entry.getKey())))); - } - return result.toString(); - } - - private void generateSnapshotInfo(final ShardingSphereMetaData metaData, final ExportedClusterInfo exportedClusterInfo) { - GlobalClockRule globalClockRule = metaData.getGlobalRuleMetaData().getSingleRule(GlobalClockRule.class); - if (globalClockRule.getConfiguration().isEnabled()) { - ExportedSnapshotInfo snapshotInfo = new ExportedSnapshotInfo(); - snapshotInfo.setCsn(String.valueOf(globalClockRule.getGlobalClockProvider().map(GlobalClockProvider::getCurrentTimestamp).orElse(0L))); - snapshotInfo.setCreateTime(LocalDateTime.now()); - exportedClusterInfo.setSnapshotInfo(snapshotInfo); + return Collections.singleton(new LocalDataQueryResultRow(instanceId, LocalDateTime.now(), String.format("Successfully exported to：'%s'", filePath))); } + return Collections.singleton(new LocalDataQueryResultRow(instanceId, LocalDateTime.now(), Base64.encodeBase64String(exportedData.getBytes()))); } @Override diff --git a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportDatabaseConfigurationExecutor.java b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportDatabaseConfigurationExecutor.java index ec25631721540..8f18f91fd76b1 100644 --- a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportDatabaseConfigurationExecutor.java +++ b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportDatabaseConfigurationExecutor.java @@ -19,33 +19,35 @@ import org.apache.shardingsphere.distsql.handler.engine.update.DistSQLUpdateExecutor; import org.apache.shardingsphere.distsql.statement.ral.updatable.ImportDatabaseConfigurationStatement; +import org.apache.shardingsphere.infra.exception.generic.FileIOException; import org.apache.shardingsphere.infra.util.yaml.YamlEngine; import org.apache.shardingsphere.mode.manager.ContextManager; import org.apache.shardingsphere.proxy.backend.config.yaml.YamlProxyDatabaseConfiguration; -import org.apache.shardingsphere.infra.exception.generic.FileIOException; -import org.apache.shardingsphere.proxy.backend.util.YamlDatabaseConfigurationImportExecutor; +import org.apache.shardingsphere.proxy.backend.util.MetaDataImportExecutor; import java.io.File; import java.io.IOException; import java.sql.SQLException; +import java.util.Collections; /** * Import database configuration executor. */ public final class ImportDatabaseConfigurationExecutor implements DistSQLUpdateExecutor { - private final YamlDatabaseConfigurationImportExecutor databaseConfigImportExecutor = new YamlDatabaseConfigurationImportExecutor(); - @Override public void executeUpdate(final ImportDatabaseConfigurationStatement sqlStatement, final ContextManager contextManager) throws SQLException { + YamlProxyDatabaseConfiguration yamlConfig = getYamlProxyDatabaseConfiguration(sqlStatement); + new MetaDataImportExecutor(contextManager).importDatabaseConfigurations(Collections.singletonList(yamlConfig)); + } + + private YamlProxyDatabaseConfiguration getYamlProxyDatabaseConfiguration(final ImportDatabaseConfigurationStatement sqlStatement) { File file = new File(sqlStatement.getFilePath()); - YamlProxyDatabaseConfiguration yamlConfig; try { - yamlConfig = YamlEngine.unmarshal(file, YamlProxyDatabaseConfiguration.class); + return YamlEngine.unmarshal(file, YamlProxyDatabaseConfiguration.class); } catch (final IOException ignore) { throw new FileIOException(file); } - databaseConfigImportExecutor.importDatabaseConfiguration(yamlConfig); } @Override diff --git a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportMetaDataExecutor.java b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportMetaDataExecutor.java index dc6d9306bf9a4..62478b2930a16 100644 --- a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportMetaDataExecutor.java +++ b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportMetaDataExecutor.java @@ -21,69 +21,42 @@ import org.apache.commons.io.FileUtils; import org.apache.shardingsphere.distsql.handler.engine.update.DistSQLUpdateExecutor; import org.apache.shardingsphere.distsql.statement.ral.updatable.ImportMetaDataStatement; -import org.apache.shardingsphere.infra.config.rule.RuleConfiguration; +import org.apache.shardingsphere.infra.exception.generic.FileIOException; import org.apache.shardingsphere.infra.util.json.JsonUtils; -import org.apache.shardingsphere.infra.util.yaml.YamlEngine; -import org.apache.shardingsphere.infra.yaml.config.swapper.rule.YamlRuleConfigurationSwapperEngine; import org.apache.shardingsphere.mode.manager.ContextManager; -import org.apache.shardingsphere.proxy.backend.config.yaml.YamlProxyDatabaseConfiguration; -import org.apache.shardingsphere.proxy.backend.config.yaml.YamlProxyServerConfiguration; import org.apache.shardingsphere.proxy.backend.distsql.export.ExportedClusterInfo; import org.apache.shardingsphere.proxy.backend.distsql.export.ExportedMetaData; -import org.apache.shardingsphere.infra.exception.generic.FileIOException; -import org.apache.shardingsphere.proxy.backend.util.YamlDatabaseConfigurationImportExecutor; +import org.apache.shardingsphere.proxy.backend.util.MetaDataImportExecutor; import java.io.File; import java.io.IOException; import java.nio.charset.Charset; import java.sql.SQLException; -import java.util.Collection; /** * Import meta data executor. */ public final class ImportMetaDataExecutor implements DistSQLUpdateExecutor { - private final YamlRuleConfigurationSwapperEngine ruleConfigSwapperEngine = new YamlRuleConfigurationSwapperEngine(); - - private final YamlDatabaseConfigurationImportExecutor databaseConfigImportExecutor = new YamlDatabaseConfigurationImportExecutor(); - @Override public void executeUpdate(final ImportMetaDataStatement sqlStatement, final ContextManager contextManager) throws SQLException { - String jsonMetaDataConfig; - if (sqlStatement.getFilePath().isPresent()) { - File file = new File(sqlStatement.getFilePath().get()); - try { - jsonMetaDataConfig = FileUtils.readFileToString(file, Charset.defaultCharset()); - } catch (final IOException ignore) { - throw new FileIOException(file); - } - } else { - jsonMetaDataConfig = new String(Base64.decodeBase64(sqlStatement.getMetaDataValue())); - } + String jsonMetaDataConfig = sqlStatement.getFilePath().isPresent() ? getMetaDataFromFile(sqlStatement) : getMetaDataFromConsole(sqlStatement); ExportedClusterInfo exportedClusterInfo = JsonUtils.fromJsonString(jsonMetaDataConfig, ExportedClusterInfo.class); ExportedMetaData exportedMetaData = exportedClusterInfo.getMetaData(); - importServerConfiguration(contextManager, exportedMetaData); - importDatabase(exportedMetaData); + new MetaDataImportExecutor(contextManager).importClusterConfigurations(exportedMetaData); } - private void importServerConfiguration(final ContextManager contextManager, final ExportedMetaData exportedMetaData) throws SQLException { - YamlProxyServerConfiguration yamlServerConfig = YamlEngine.unmarshal(exportedMetaData.getRules() + System.lineSeparator() + exportedMetaData.getProps(), YamlProxyServerConfiguration.class); - if (null == yamlServerConfig) { - return; - } - Collection rules = ruleConfigSwapperEngine.swapToRuleConfigurations(yamlServerConfig.getRules()); - for (RuleConfiguration each : rules) { - contextManager.getPersistServiceFacade().getMetaDataManagerPersistService().alterGlobalRuleConfiguration(each); + private String getMetaDataFromFile(final ImportMetaDataStatement sqlStatement) { + File file = new File(sqlStatement.getFilePath().get()); + try { + return FileUtils.readFileToString(file, Charset.defaultCharset()); + } catch (final IOException ignore) { + throw new FileIOException(file); } - contextManager.getPersistServiceFacade().getMetaDataManagerPersistService().alterProperties(yamlServerConfig.getProps()); } - private void importDatabase(final ExportedMetaData exportedMetaData) throws SQLException { - for (String each : exportedMetaData.getDatabases().values()) { - YamlProxyDatabaseConfiguration yamlDatabaseConfig = YamlEngine.unmarshal(each, YamlProxyDatabaseConfiguration.class); - databaseConfigImportExecutor.importDatabaseConfiguration(yamlDatabaseConfig); - } + private String getMetaDataFromConsole(final ImportMetaDataStatement sqlStatement) { + return new String(Base64.decodeBase64(sqlStatement.getMetaDataValue())); } @Override diff --git a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/ClusterExportMetaDataGenerator.java b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/ClusterExportMetaDataGenerator.java new file mode 100644 index 0000000000000..7746420f10650 --- /dev/null +++ b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/ClusterExportMetaDataGenerator.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shardingsphere.proxy.backend.util; + +import lombok.RequiredArgsConstructor; +import org.apache.shardingsphere.globalclock.provider.GlobalClockProvider; +import org.apache.shardingsphere.globalclock.rule.GlobalClockRule; +import org.apache.shardingsphere.infra.config.rule.RuleConfiguration; +import org.apache.shardingsphere.infra.metadata.ShardingSphereMetaData; +import org.apache.shardingsphere.infra.metadata.database.ShardingSphereDatabase; +import org.apache.shardingsphere.infra.spi.type.ordered.OrderedSPILoader; +import org.apache.shardingsphere.infra.util.json.JsonUtils; +import org.apache.shardingsphere.infra.util.yaml.YamlEngine; +import org.apache.shardingsphere.infra.yaml.config.swapper.rule.YamlRuleConfigurationSwapper; +import org.apache.shardingsphere.mode.manager.ContextManager; +import org.apache.shardingsphere.mode.metadata.decorator.RuleConfigurationPersistDecorateEngine; +import org.apache.shardingsphere.proxy.backend.distsql.export.ExportedClusterInfo; +import org.apache.shardingsphere.proxy.backend.distsql.export.ExportedMetaData; +import org.apache.shardingsphere.proxy.backend.distsql.export.ExportedSnapshotInfo; + +import java.time.LocalDateTime; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Properties; +import java.util.stream.Collectors; + +/** + * Cluster export metadata generator. + */ +@RequiredArgsConstructor +public final class ClusterExportMetaDataGenerator { + + private final ContextManager contextManager; + + /** + * Generate Json format. + * + * @return export data + */ + public String generateJsonFormat() { + ShardingSphereMetaData metaData = contextManager.getMetaDataContexts().getMetaData(); + ExportedMetaData exportedMetaData = new ExportedMetaData(); + exportedMetaData.setDatabases(generatorDatabasesExportData()); + exportedMetaData.setProps(generatePropsData(metaData.getProps().getProps())); + exportedMetaData.setRules(generateRulesData(getGlobalRules(metaData))); + ExportedClusterInfo exportedClusterInfo = new ExportedClusterInfo(); + exportedClusterInfo.setMetaData(exportedMetaData); + generateSnapshotInfo(metaData, exportedClusterInfo); + return JsonUtils.toJsonString(exportedClusterInfo); + } + + private Map generatorDatabasesExportData() { + Collection databaseNames = contextManager.getMetaDataContexts().getMetaData().getAllDatabases().stream() + .map(ShardingSphereDatabase::getName).collect(Collectors.toList()); + Map result = new LinkedHashMap<>(databaseNames.size(), 1F); + for (String each : databaseNames) { + ShardingSphereDatabase database = contextManager.getDatabase(each); + if (database.getResourceMetaData().getAllInstanceDataSourceNames().isEmpty()) { + continue; + } + result.put(each, new DatabaseExportMetaDataGenerator(database).generateYAMLFormat()); + } + return result; + } + + private String generatePropsData(final Properties props) { + if (props.isEmpty()) { + return ""; + } + StringBuilder result = new StringBuilder(); + result.append("props:").append(System.lineSeparator()); + for (Entry entry : props.entrySet()) { + if (null != entry.getValue() && !"".equals(entry.getValue().toString())) { + result.append(" ").append(entry.getKey()).append(": ").append(entry.getValue()).append(System.lineSeparator()); + } + } + return result.toString(); + } + + private Collection getGlobalRules(final ShardingSphereMetaData metaData) { + RuleConfigurationPersistDecorateEngine ruleConfigPersistDecorateEngine = new RuleConfigurationPersistDecorateEngine(contextManager.getComputeNodeInstanceContext()); + return ruleConfigPersistDecorateEngine.decorate(metaData.getGlobalRuleMetaData().getConfigurations()); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private String generateRulesData(final Collection rules) { + if (rules.isEmpty()) { + return ""; + } + StringBuilder result = new StringBuilder(); + result.append("rules:").append(System.lineSeparator()); + for (Entry entry : OrderedSPILoader.getServices(YamlRuleConfigurationSwapper.class, rules).entrySet()) { + result.append(YamlEngine.marshal(Collections.singletonList(entry.getValue().swapToYamlConfiguration(entry.getKey())))); + } + return result.toString(); + } + + private void generateSnapshotInfo(final ShardingSphereMetaData metaData, final ExportedClusterInfo exportedClusterInfo) { + GlobalClockRule globalClockRule = metaData.getGlobalRuleMetaData().getSingleRule(GlobalClockRule.class); + if (globalClockRule.getConfiguration().isEnabled()) { + ExportedSnapshotInfo snapshotInfo = new ExportedSnapshotInfo(); + snapshotInfo.setCsn(String.valueOf(globalClockRule.getGlobalClockProvider().map(GlobalClockProvider::getCurrentTimestamp).orElse(0L))); + snapshotInfo.setCreateTime(LocalDateTime.now()); + exportedClusterInfo.setSnapshotInfo(snapshotInfo); + } + } +} diff --git a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/DatabaseExportMetaDataGenerator.java b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/DatabaseExportMetaDataGenerator.java new file mode 100644 index 0000000000000..570e9f9694416 --- /dev/null +++ b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/DatabaseExportMetaDataGenerator.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shardingsphere.proxy.backend.util; + +import lombok.RequiredArgsConstructor; +import org.apache.shardingsphere.infra.config.rule.RuleConfiguration; +import org.apache.shardingsphere.infra.config.rule.scope.DatabaseRuleConfiguration; +import org.apache.shardingsphere.infra.config.rule.scope.DatabaseRuleConfigurationEmptyChecker; +import org.apache.shardingsphere.infra.datasource.pool.props.domain.DataSourcePoolProperties; +import org.apache.shardingsphere.infra.metadata.database.ShardingSphereDatabase; +import org.apache.shardingsphere.infra.metadata.database.resource.unit.StorageUnit; +import org.apache.shardingsphere.infra.spi.type.ordered.OrderedSPILoader; +import org.apache.shardingsphere.infra.spi.type.typed.TypedSPILoader; +import org.apache.shardingsphere.infra.util.yaml.YamlEngine; +import org.apache.shardingsphere.infra.yaml.config.swapper.rule.YamlRuleConfigurationSwapper; + +import java.util.Collection; +import java.util.Collections; +import java.util.Map.Entry; + +/** + * Database export metadata generator. + */ +@RequiredArgsConstructor +public final class DatabaseExportMetaDataGenerator { + + private final ShardingSphereDatabase database; + + /** + * Generate YAML format. + * + * @return database configuration data of YAML format + */ + public String generateYAMLFormat() { + StringBuilder result = new StringBuilder(); + appendDatabaseName(database.getName(), result); + appendDataSourceConfigurations(database, result); + appendRuleConfigurations(database.getRuleMetaData().getConfigurations(), result); + return result.toString(); + } + + private void appendDatabaseName(final String databaseName, final StringBuilder stringBuilder) { + stringBuilder.append("databaseName: ").append(databaseName).append(System.lineSeparator()); + } + + private void appendDataSourceConfigurations(final ShardingSphereDatabase database, final StringBuilder stringBuilder) { + if (database.getResourceMetaData().getStorageUnits().isEmpty()) { + return; + } + stringBuilder.append("dataSources:").append(System.lineSeparator()); + for (Entry entry : database.getResourceMetaData().getStorageUnits().entrySet()) { + appendDataSourceConfiguration(entry.getKey(), entry.getValue().getDataSourcePoolProperties(), stringBuilder); + } + } + + private void appendDataSourceConfiguration(final String dataSourceName, final DataSourcePoolProperties props, final StringBuilder stringBuilder) { + stringBuilder.append(createIndentation(2)).append(dataSourceName).append(':').append(System.lineSeparator()); + for (Entry entry : props.getConnectionPropertySynonyms().getStandardProperties().entrySet()) { + String value = null == entry.getValue() ? "" : entry.getValue().toString(); + stringBuilder.append(createIndentation(4)).append(entry.getKey()).append(": ").append(value).append(System.lineSeparator()); + } + for (Entry entry : props.getPoolPropertySynonyms().getStandardProperties().entrySet()) { + String value = null == entry.getValue() ? "" : entry.getValue().toString(); + stringBuilder.append(createIndentation(4)).append(entry.getKey()).append(": ").append(value).append(System.lineSeparator()); + } + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private void appendRuleConfigurations(final Collection ruleConfigs, final StringBuilder stringBuilder) { + if (ruleConfigs.isEmpty()) { + return; + } + boolean hasAppendedRule = false; + for (Entry entry : OrderedSPILoader.getServices(YamlRuleConfigurationSwapper.class, ruleConfigs).entrySet()) { + if (TypedSPILoader.getService(DatabaseRuleConfigurationEmptyChecker.class, entry.getKey().getClass()).isEmpty((DatabaseRuleConfiguration) entry.getKey())) { + continue; + } + if (!hasAppendedRule) { + stringBuilder.append("rules:").append(System.lineSeparator()); + hasAppendedRule = true; + } + stringBuilder.append(YamlEngine.marshal(Collections.singletonList(entry.getValue().swapToYamlConfiguration(entry.getKey())))); + } + } + + private String createIndentation(final int count) { + StringBuilder result = new StringBuilder(); + for (int i = 0; i < count; i++) { + result.append(" "); + } + return result.toString(); + } +} diff --git a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/ExportUtils.java b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/ExportUtils.java index 433139ca1b50c..cc482d0566ac4 100644 --- a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/ExportUtils.java +++ b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/ExportUtils.java @@ -19,26 +19,13 @@ import lombok.AccessLevel; import lombok.NoArgsConstructor; -import org.apache.shardingsphere.infra.config.rule.RuleConfiguration; -import org.apache.shardingsphere.infra.config.rule.scope.DatabaseRuleConfiguration; -import org.apache.shardingsphere.infra.config.rule.scope.DatabaseRuleConfigurationEmptyChecker; -import org.apache.shardingsphere.infra.datasource.pool.props.domain.DataSourcePoolProperties; import org.apache.shardingsphere.infra.exception.generic.FileIOException; -import org.apache.shardingsphere.infra.metadata.database.ShardingSphereDatabase; -import org.apache.shardingsphere.infra.metadata.database.resource.unit.StorageUnit; -import org.apache.shardingsphere.infra.spi.type.ordered.OrderedSPILoader; -import org.apache.shardingsphere.infra.spi.type.typed.TypedSPILoader; -import org.apache.shardingsphere.infra.util.yaml.YamlEngine; -import org.apache.shardingsphere.infra.yaml.config.swapper.rule.YamlRuleConfigurationSwapper; import java.io.File; import java.io.IOException; import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Paths; -import java.util.Collection; -import java.util.Collections; -import java.util.Map.Entry; /** * Export utility class. @@ -66,58 +53,4 @@ public static void exportToFile(final String filePath, final String exportedData throw new FileIOException(file); } } - - /** - * Generate configuration data of ShardingSphere database. - * - * @param database ShardingSphere database - * @return configuration data - */ - public static String generateExportDatabaseData(final ShardingSphereDatabase database) { - StringBuilder result = new StringBuilder(); - appendDatabaseName(database.getName(), result); - appendDataSourceConfigurations(database, result); - appendRuleConfigurations(database.getRuleMetaData().getConfigurations(), result); - return result.toString(); - } - - private static void appendDatabaseName(final String databaseName, final StringBuilder stringBuilder) { - stringBuilder.append("databaseName: ").append(databaseName).append(System.lineSeparator()); - } - - private static void appendDataSourceConfigurations(final ShardingSphereDatabase database, final StringBuilder stringBuilder) { - if (database.getResourceMetaData().getStorageUnits().isEmpty()) { - return; - } - stringBuilder.append("dataSources:").append(System.lineSeparator()); - for (Entry entry : database.getResourceMetaData().getStorageUnits().entrySet()) { - appendDataSourceConfiguration(entry.getKey(), entry.getValue().getDataSourcePoolProperties(), stringBuilder); - } - } - - private static void appendDataSourceConfiguration(final String name, final DataSourcePoolProperties props, final StringBuilder stringBuilder) { - stringBuilder.append(" ").append(name).append(':').append(System.lineSeparator()); - props.getConnectionPropertySynonyms().getStandardProperties() - .forEach((key, value) -> stringBuilder.append(" ").append(key).append(": ").append(value).append(System.lineSeparator())); - for (Entry entry : props.getPoolPropertySynonyms().getStandardProperties().entrySet()) { - if (null != entry.getValue()) { - stringBuilder.append(" ").append(entry.getKey()).append(": ").append(entry.getValue()).append(System.lineSeparator()); - } - } - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - private static void appendRuleConfigurations(final Collection ruleConfigs, final StringBuilder stringBuilder) { - if (ruleConfigs.isEmpty() - || ruleConfigs.stream().allMatch(each -> TypedSPILoader.getService(DatabaseRuleConfigurationEmptyChecker.class, each.getClass()).isEmpty((DatabaseRuleConfiguration) each))) { - return; - } - stringBuilder.append("rules:").append(System.lineSeparator()); - for (Entry entry : OrderedSPILoader.getServices(YamlRuleConfigurationSwapper.class, ruleConfigs).entrySet()) { - if (TypedSPILoader.getService(DatabaseRuleConfigurationEmptyChecker.class, entry.getKey().getClass()).isEmpty((DatabaseRuleConfiguration) entry.getKey())) { - continue; - } - stringBuilder.append(YamlEngine.marshal(Collections.singletonList(entry.getValue().swapToYamlConfiguration(entry.getKey())))); - } - } } diff --git a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/MetaDataImportExecutor.java b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/MetaDataImportExecutor.java new file mode 100644 index 0000000000000..d65d9498348ce --- /dev/null +++ b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/MetaDataImportExecutor.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shardingsphere.proxy.backend.util; + +import lombok.RequiredArgsConstructor; +import org.apache.shardingsphere.infra.config.rule.RuleConfiguration; +import org.apache.shardingsphere.infra.util.yaml.YamlEngine; +import org.apache.shardingsphere.infra.yaml.config.swapper.rule.YamlRuleConfigurationSwapperEngine; +import org.apache.shardingsphere.mode.manager.ContextManager; +import org.apache.shardingsphere.proxy.backend.config.yaml.YamlProxyDatabaseConfiguration; +import org.apache.shardingsphere.proxy.backend.config.yaml.YamlProxyServerConfiguration; +import org.apache.shardingsphere.proxy.backend.distsql.export.ExportedMetaData; + +import java.sql.SQLException; +import java.util.Collection; +import java.util.LinkedList; + +/** + * Metadata import executor. + */ +@RequiredArgsConstructor +public final class MetaDataImportExecutor { + + private final YamlRuleConfigurationSwapperEngine ruleConfigSwapperEngine = new YamlRuleConfigurationSwapperEngine(); + + private final YamlDatabaseConfigurationImportExecutor databaseConfigImportExecutor = new YamlDatabaseConfigurationImportExecutor(); + + private final ContextManager contextManager; + + /** + * Import cluster configurations. + * + * @param exportedMetaData exported metadata + * @throws SQLException SQL exception + */ + public void importClusterConfigurations(final ExportedMetaData exportedMetaData) throws SQLException { + Collection databaseConfigs = getYamlProxyDatabaseConfigurations(exportedMetaData); + importServerConfiguration(exportedMetaData); + importDatabaseConfigurations(databaseConfigs); + } + + private void importServerConfiguration(final ExportedMetaData exportedMetaData) throws SQLException { + YamlProxyServerConfiguration yamlServerConfig = YamlEngine.unmarshal(exportedMetaData.getRules() + System.lineSeparator() + exportedMetaData.getProps(), YamlProxyServerConfiguration.class); + if (null == yamlServerConfig) { + return; + } + importGlobalRules(yamlServerConfig); + importProps(yamlServerConfig); + } + + private void importGlobalRules(final YamlProxyServerConfiguration yamlServerConfig) throws SQLException { + Collection rules = ruleConfigSwapperEngine.swapToRuleConfigurations(yamlServerConfig.getRules()); + for (RuleConfiguration each : rules) { + contextManager.getPersistServiceFacade().getMetaDataManagerPersistService().alterGlobalRuleConfiguration(each); + } + } + + private void importProps(final YamlProxyServerConfiguration yamlServerConfig) throws SQLException { + contextManager.getPersistServiceFacade().getMetaDataManagerPersistService().alterProperties(yamlServerConfig.getProps()); + } + + private Collection getYamlProxyDatabaseConfigurations(final ExportedMetaData exportedMetaData) { + Collection result = new LinkedList<>(); + for (String each : exportedMetaData.getDatabases().values()) { + result.add(YamlEngine.unmarshal(each, YamlProxyDatabaseConfiguration.class)); + } + return result; + } + + /** + * Import database configurations. + * + * @param databaseConfigs YAML proxy database configuration + * @throws SQLException SQL exception + */ + public void importDatabaseConfigurations(final Collection databaseConfigs) throws SQLException { + for (YamlProxyDatabaseConfiguration each : databaseConfigs) { + databaseConfigImportExecutor.importDatabaseConfiguration(each); + } + } +} From 9dac2f80603fc09b9b9137a9f5eaadf7103f5f1d Mon Sep 17 00:00:00 2001 From: jiangML <1060319118@qq.com> Date: Tue, 24 Dec 2024 20:37:53 +0800 Subject: [PATCH 2/3] Fix test error --- .../util/ClusterExportMetaDataGenerator.java | 12 +- .../backend/util/MetaDataImportExecutor.java | 7 +- ...mlDatabaseConfigurationImportExecutor.java | 22 +-- ...portDatabaseConfigurationExecutorTest.java | 4 +- .../queryable/ExportMetaDataExecutorTest.java | 98 +++++------ ...portDatabaseConfigurationExecutorTest.java | 69 +++----- .../updatable/ImportMetaDataExecutorTest.java | 156 ++++++++++-------- 7 files changed, 175 insertions(+), 193 deletions(-) diff --git a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/ClusterExportMetaDataGenerator.java b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/ClusterExportMetaDataGenerator.java index 7746420f10650..f788c8967f633 100644 --- a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/ClusterExportMetaDataGenerator.java +++ b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/ClusterExportMetaDataGenerator.java @@ -40,7 +40,6 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Properties; -import java.util.stream.Collectors; /** * Cluster export metadata generator. @@ -68,15 +67,12 @@ public String generateJsonFormat() { } private Map generatorDatabasesExportData() { - Collection databaseNames = contextManager.getMetaDataContexts().getMetaData().getAllDatabases().stream() - .map(ShardingSphereDatabase::getName).collect(Collectors.toList()); - Map result = new LinkedHashMap<>(databaseNames.size(), 1F); - for (String each : databaseNames) { - ShardingSphereDatabase database = contextManager.getDatabase(each); - if (database.getResourceMetaData().getAllInstanceDataSourceNames().isEmpty()) { + Map result = new LinkedHashMap<>(contextManager.getMetaDataContexts().getMetaData().getAllDatabases().size(), 1F); + for (ShardingSphereDatabase each : contextManager.getMetaDataContexts().getMetaData().getAllDatabases()) { + if (each.getResourceMetaData().getAllInstanceDataSourceNames().isEmpty()) { continue; } - result.put(each, new DatabaseExportMetaDataGenerator(database).generateYAMLFormat()); + result.put(each.getName(), new DatabaseExportMetaDataGenerator(each).generateYAMLFormat()); } return result; } diff --git a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/MetaDataImportExecutor.java b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/MetaDataImportExecutor.java index d65d9498348ce..a7e22f3d8298e 100644 --- a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/MetaDataImportExecutor.java +++ b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/MetaDataImportExecutor.java @@ -38,10 +38,15 @@ public final class MetaDataImportExecutor { private final YamlRuleConfigurationSwapperEngine ruleConfigSwapperEngine = new YamlRuleConfigurationSwapperEngine(); - private final YamlDatabaseConfigurationImportExecutor databaseConfigImportExecutor = new YamlDatabaseConfigurationImportExecutor(); + private final YamlDatabaseConfigurationImportExecutor databaseConfigImportExecutor; private final ContextManager contextManager; + public MetaDataImportExecutor(final ContextManager contextManager) { + this.contextManager = contextManager; + this.databaseConfigImportExecutor = new YamlDatabaseConfigurationImportExecutor(contextManager); + } + /** * Import cluster configurations. * diff --git a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/YamlDatabaseConfigurationImportExecutor.java b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/YamlDatabaseConfigurationImportExecutor.java index 33078e9e56ad7..54c141f8a640b 100644 --- a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/YamlDatabaseConfigurationImportExecutor.java +++ b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/YamlDatabaseConfigurationImportExecutor.java @@ -17,6 +17,7 @@ package org.apache.shardingsphere.proxy.backend.util; +import lombok.RequiredArgsConstructor; import org.apache.shardingsphere.distsql.handler.validate.DistSQLDataSourcePoolPropertiesValidator; import org.apache.shardingsphere.infra.config.rule.RuleConfiguration; import org.apache.shardingsphere.infra.config.rule.checker.RuleConfigurationCheckEngine; @@ -47,7 +48,6 @@ import org.apache.shardingsphere.proxy.backend.config.yaml.YamlProxyDataSourceConfiguration; import org.apache.shardingsphere.proxy.backend.config.yaml.YamlProxyDatabaseConfiguration; import org.apache.shardingsphere.proxy.backend.config.yaml.swapper.YamlProxyDataSourceConfigurationSwapper; -import org.apache.shardingsphere.proxy.backend.context.ProxyContext; import java.sql.SQLException; import java.util.Collection; @@ -62,12 +62,15 @@ /** * Yaml database configuration import executor. */ +@RequiredArgsConstructor public final class YamlDatabaseConfigurationImportExecutor { private final YamlProxyDataSourceConfigurationSwapper dataSourceConfigSwapper = new YamlProxyDataSourceConfigurationSwapper(); private final DistSQLDataSourcePoolPropertiesValidator validateHandler = new DistSQLDataSourcePoolPropertiesValidator(); + private final ContextManager contextManager; + /** * Import proxy database from yaml configuration. * @@ -94,11 +97,11 @@ private void checkDataSources(final String databaseName, final Map new DatabaseCreateExistsException(databaseName)); + ShardingSpherePreconditions.checkNotEmpty(databaseName, MissingRequiredDatabaseException::new); + ShardingSpherePreconditions.checkState(!contextManager.getMetaDataContexts().getMetaData().containsDatabase(databaseName), () -> new DatabaseCreateExistsException(databaseName)); } private void addDatabase(final String databaseName) throws SQLException { - ContextManager contextManager = ProxyContext.getInstance().getContextManager(); contextManager.getPersistServiceFacade().getMetaDataManagerPersistService().createDatabase(databaseName); DatabaseType protocolType = DatabaseTypeEngine.getProtocolType(Collections.emptyMap(), contextManager.getMetaDataContexts().getMetaData().getProps()); contextManager.getMetaDataContexts().getMetaData().addDatabase(databaseName, protocolType, contextManager.getMetaDataContexts().getMetaData().getProps()); @@ -112,12 +115,11 @@ private void importDataSources(final String databaseName, final Map storageUnits = ProxyContext.getInstance().getContextManager() - .getMetaDataContexts().getMetaData().getDatabase(databaseName).getResourceMetaData().getStorageUnits(); + Map storageUnits = contextManager.getMetaDataContexts().getMetaData().getDatabase(databaseName).getResourceMetaData().getStorageUnits(); Map toBeAddedStorageNode = StorageUnitNodeMapCreator.create(propsMap); for (Entry entry : propsMap.entrySet()) { storageUnits.put(entry.getKey(), new StorageUnit(toBeAddedStorageNode.get(entry.getKey()), entry.getValue(), DataSourcePoolCreator.create(entry.getValue()))); @@ -129,10 +131,10 @@ private void importRules(final String databaseName, final Collection ruleConfigs = new LinkedList<>(); - MetaDataContexts metaDataContexts = ProxyContext.getInstance().getContextManager().getMetaDataContexts(); + MetaDataContexts metaDataContexts = contextManager.getMetaDataContexts(); ShardingSphereDatabase database = metaDataContexts.getMetaData().getDatabase(databaseName); swapToRuleConfigs(yamlRuleConfigs).values().forEach(each -> addRule(ruleConfigs, each, database)); - ProxyContext.getInstance().getContextManager().getPersistServiceFacade().getMetaDataPersistService().getDatabaseRulePersistService() + contextManager.getPersistServiceFacade().getMetaDataPersistService().getDatabaseRulePersistService() .persist(metaDataContexts.getMetaData().getDatabase(databaseName).getName(), ruleConfigs); } @@ -145,7 +147,7 @@ private void addRule(final Collection ruleConfigs, final Rule @SuppressWarnings({"unchecked", "rawtypes"}) private ShardingSphereRule buildRule(final RuleConfiguration ruleConfig, final ShardingSphereDatabase database) { DatabaseRuleBuilder ruleBuilder = OrderedSPILoader.getServices(DatabaseRuleBuilder.class, Collections.singleton(ruleConfig)).get(ruleConfig); - ComputeNodeInstanceContext computeNodeInstanceContext = ProxyContext.getInstance().getContextManager().getComputeNodeInstanceContext(); + ComputeNodeInstanceContext computeNodeInstanceContext = contextManager.getComputeNodeInstanceContext(); return ruleBuilder.build(ruleConfig, database.getName(), database.getProtocolType(), database.getResourceMetaData(), database.getRuleMetaData().getRules(), computeNodeInstanceContext); } @@ -161,6 +163,6 @@ private Map swapToRuleConfigs(final Collection storageUnits = createStorageUnits(); when(database.getResourceMetaData().getStorageUnits()).thenReturn(storageUnits); @@ -91,6 +90,7 @@ private Map createStorageUnits() { @Test void assertExecuteWithEmptyDatabase() { + ShardingSphereDatabase database = mock(ShardingSphereDatabase.class, RETURNS_DEEP_STUBS); when(database.getName()).thenReturn("empty_db"); when(database.getResourceMetaData().getStorageUnits()).thenReturn(Collections.emptyMap()); when(database.getRuleMetaData().getConfigurations()).thenReturn(Collections.emptyList()); diff --git a/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportMetaDataExecutorTest.java b/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportMetaDataExecutorTest.java index 908fd91169ef7..6967b4e51f3c8 100644 --- a/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportMetaDataExecutorTest.java +++ b/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportMetaDataExecutorTest.java @@ -52,26 +52,18 @@ import org.apache.shardingsphere.proxy.backend.config.yaml.YamlProxyDataSourceConfiguration; import org.apache.shardingsphere.proxy.backend.config.yaml.YamlProxyDatabaseConfiguration; import org.apache.shardingsphere.proxy.backend.config.yaml.YamlProxyServerConfiguration; -import org.apache.shardingsphere.proxy.backend.context.ProxyContext; import org.apache.shardingsphere.proxy.backend.distsql.export.ExportedClusterInfo; import org.apache.shardingsphere.proxy.backend.distsql.export.ExportedMetaData; import org.apache.shardingsphere.test.fixture.jdbc.MockedDataSource; -import org.apache.shardingsphere.test.mock.AutoMockExtension; -import org.apache.shardingsphere.test.mock.StaticMockSettings; import org.apache.shardingsphere.test.util.PropertiesBuilder; import org.apache.shardingsphere.test.util.PropertiesBuilder.Property; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Answers; -import org.mockito.Mock; import javax.sql.DataSource; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.LinkedHashMap; -import java.util.LinkedList; import java.util.Map; import java.util.Map.Entry; import java.util.Properties; @@ -87,36 +79,21 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -@ExtendWith(AutoMockExtension.class) -@StaticMockSettings(ProxyContext.class) class ExportMetaDataExecutorTest { - private static final String EXPECTED_EMPTY_METADATA_VALUE = "eyJtZXRhX2RhdGEiOnsiZGF0YWJhc2VzIjp7ImVtcHR5X21ldGFkYXRhIjoiZGF0YWJhc2VOYW1lOiBudWxsXG5kYXRhU291cmNlczpcbn" - + "J1bGVzOlxuIn0sInByb3BzIjoiIiwicnVsZXMiOiJydWxlczpcbi0gIUdMT0JBTF9DTE9DS1xuICBlbmFibGVkOiBmYWxzZVxuICBwcm92aWRlcjogbG9jYWxcbiAgdHlwZTogVFNPXG4ifX0="; + private static final String EXPECTED_EMPTY_METADATA_VALUE = "eyJtZXRhX2RhdGEiOnsiZGF0YWJhc2VzIjp7ImVtcHR5X21ldGFkYXRhIjoiZGF0YWJhc2VOYW1lOiBlbXB0eV9tZXRhZGF0YVxuI" + + "n0sInByb3BzIjoiIiwicnVsZXMiOiJydWxlczpcbi0gIUdMT0JBTF9DTE9DS1xuICBlbmFibGVkOiBmYWxzZVxuICBwcm92aWRlcjogbG9jYWxcbiAgdHlwZTogVFNPXG4ifX0="; - private static final String EXPECTED_NOT_EMPTY_METADATA_VALUE = "eyJtZXRhX2RhdGEiOnsiZGF0YWJhc2VzIjp7Im5vcm1hbF9kYiI6ImRhdGFiYXNlTmFtZTogbm9ybWFsX2RiXG5kYXRhU291cm" - + "NlczpcbiAgZHNfMDpcbiAgICBwYXNzd29yZDogXG4gICAgdXJsOiBqZGJjOm9wZW5nYXVzczovLzEyNy4wLjAuMTo1NDMyL2RlbW9fZHNfMFxuICAgIHVzZXJuYW1lOiByb290XG4gICAgbWluUG9vb" - + "FNpemU6IDFcbiAgICBtYXhQb29sU2l6ZTogNTBcbiAgZHNfMTpcbiAgICBwYXNzd29yZDogXG4gICAgdXJsOiBqZGJjOm9wZW5nYXVzczovLzEyNy4wLjAuMTo1NDMyL2RlbW9fZHNfMVxuICAgIHVzZ" - + "XJuYW1lOiByb290XG4gICAgbWluUG9vbFNpemU6IDFcbiAgICBtYXhQb29sU2l6ZTogNTBcbiJ9LCJwcm9wcyI6InByb3BzOlxuICBzcWwtc2hvdzogdHJ1ZVxuIiwicnVsZXMiOiJydWxlczpcbi0g" - + "IUFVVEhPUklUWVxuICBwcml2aWxlZ2U6XG4gICAgdHlwZTogQUxMX1BFUk1JVFRFRFxuICB1c2VyczpcbiAgLSBhdXRoZW50aWNhdGlvbk1ldGhvZE5hbWU6ICcnXG4gICAgcGFzc3dvcmQ6IHJvb3Rc" - + "biAgICB1c2VyOiByb290QCVcbi0gIUdMT0JBTF9DTE9DS1xuICBlbmFibGVkOiBmYWxzZVxuICBwcm92aWRlcjogbG9jYWxcbiAgdHlwZTogVFNPXG4ifX0="; - - @Mock(answer = Answers.RETURNS_DEEP_STUBS) - private ShardingSphereDatabase database; - - @BeforeEach - void setUp() { - when(database.getProtocolType()).thenReturn(TypedSPILoader.getService(DatabaseType.class, "FIXTURE")); - } + private static final String EXPECTED_NOT_EMPTY_METADATA_VALUE = "eyJtZXRhX2RhdGEiOnsiZGF0YWJhc2VzIjp7Im5vcm1hbF9kYiI6ImRhdGFiYXNlTmFtZTogbm9ybWFsX2RiXG5kYXRhU291cmNlczpcbiAgZHNfMDpcbiA" + + "gICBwYXNzd29yZDogXG4gICAgdXJsOiBqZGJjOmgyOm1lbTpkZW1vX2RzXzA7REJfQ0xPU0VfREVMQVk9LTE7REFUQUJBU0VfVE9fVVBQRVI9ZmFsc2U7TU9ERT1NeVNRTFxuICAgIHVzZXJuYW1lOiByb290XG4gICAgbWluUG9" + + "vbFNpemU6IDFcbiAgICBtYXhQb29sU2l6ZTogNTBcbiAgZHNfMTpcbiAgICBwYXNzd29yZDogXG4gICAgdXJsOiBqZGJjOmgyOm1lbTpkZW1vX2RzXzE7REJfQ0xPU0VfREVMQVk9LTE7REFUQUJBU0VfVE9fVVBQRVI9ZmFsc2" + + "U7TU9ERT1NeVNRTFxuICAgIHVzZXJuYW1lOiByb290XG4gICAgbWluUG9vbFNpemU6IDFcbiAgICBtYXhQb29sU2l6ZTogNTBcbiJ9LCJwcm9wcyI6InByb3BzOlxuICBzcWwtc2hvdzogdHJ1ZVxuIiwicnVsZXMiOiJydWxlczpcbi0g" + + "IUFVVEhPUklUWVxuICBwcml2aWxlZ2U6XG4gICAgdHlwZTogQUxMX1BFUk1JVFRFRFxuICB1c2VyczpcbiAgLSBhZG1pbjogdHJ1ZVxuICAgIGF1dGhlbnRpY2F0aW9uTWV0aG9kTmFtZTogJydcbiAgIC" + + "BwYXNzd29yZDogcm9vdFxuICAgIHVzZXI6IHJvb3RAJVxuLSAhR0xPQkFMX0NMT0NLXG4gIGVuYWJsZWQ6IGZhbHNlXG4gIHByb3ZpZGVyOiBsb2NhbFxuICB0eXBlOiBUU09cbiJ9fQ=="; @Test void assertExecuteWithEmptyMetaData() { ContextManager contextManager = mockEmptyContextManager(); - when(ProxyContext.getInstance().getContextManager()).thenReturn(contextManager); - when(ProxyContext.getInstance().getAllDatabaseNames()).thenReturn(Collections.singleton("empty_metadata")); - when(database.getResourceMetaData().getAllInstanceDataSourceNames()).thenReturn(Collections.singleton("empty_metadata")); - when(database.getResourceMetaData().getStorageUnits()).thenReturn(Collections.emptyMap()); - when(database.getRuleMetaData().getConfigurations()).thenReturn(Collections.emptyList()); ExportMetaDataStatement sqlStatement = new ExportMetaDataStatement(null); Collection actual = new ExportMetaDataExecutor().getRows(sqlStatement, contextManager); assertThat(actual.size(), is(1)); @@ -125,43 +102,36 @@ void assertExecuteWithEmptyMetaData() { } private ContextManager mockEmptyContextManager() { + ShardingSphereDatabase database = mockEmptyShardingSphereDatabase(); ContextManager result = mock(ContextManager.class, RETURNS_DEEP_STUBS); - MetaDataContexts metaDataContexts = MetaDataContextsFactory.create(mock(MetaDataPersistService.class), new ShardingSphereMetaData(new LinkedList<>(), + MetaDataContexts metaDataContexts = MetaDataContextsFactory.create(mock(MetaDataPersistService.class), new ShardingSphereMetaData(Collections.singleton(database), new ResourceMetaData(Collections.emptyMap()), new RuleMetaData(Collections.singleton(new GlobalClockRule(new DefaultGlobalClockRuleConfigurationBuilder().build()))), new ConfigurationProperties(new Properties()))); when(result.getMetaDataContexts()).thenReturn(metaDataContexts); return result; } + private ShardingSphereDatabase mockEmptyShardingSphereDatabase() { + ShardingSphereDatabase result = mock(ShardingSphereDatabase.class, RETURNS_DEEP_STUBS); + when(result.getProtocolType()).thenReturn(TypedSPILoader.getService(DatabaseType.class, "FIXTURE")); + when(result.getName()).thenReturn("empty_metadata"); + when(result.getResourceMetaData().getAllInstanceDataSourceNames()).thenReturn(Collections.singleton("empty_metadata")); + when(result.getResourceMetaData().getStorageUnits()).thenReturn(Collections.emptyMap()); + when(result.getRuleMetaData().getConfigurations()).thenReturn(Collections.emptyList()); + return result; + } + @Test void assertExecute() { - when(database.getName()).thenReturn("normal_db"); - when(database.getResourceMetaData().getAllInstanceDataSourceNames()).thenReturn(Collections.singleton("empty_metadata")); - Map storageUnits = createStorageUnits(); - when(database.getResourceMetaData().getStorageUnits()).thenReturn(storageUnits); - when(database.getRuleMetaData().getConfigurations()).thenReturn(Collections.emptyList()); ContextManager contextManager = mockContextManager(); - when(ProxyContext.getInstance().getContextManager()).thenReturn(contextManager); - when(ProxyContext.getInstance().getAllDatabaseNames()).thenReturn(Collections.singleton("normal_db")); Collection actual = new ExportMetaDataExecutor().getRows(new ExportMetaDataStatement(null), contextManager); assertThat(actual.size(), is(1)); LocalDataQueryResultRow row = actual.iterator().next(); assertMetaData(row.getCell(3), EXPECTED_NOT_EMPTY_METADATA_VALUE); } - private Map createStorageUnits() { - Map propsMap = createDataSourceMap().entrySet().stream() - .collect(Collectors.toMap(Entry::getKey, entry -> DataSourcePoolPropertiesCreator.create(entry.getValue()), (oldValue, currentValue) -> oldValue, LinkedHashMap::new)); - Map result = new LinkedHashMap<>(propsMap.size(), 1F); - for (Entry entry : propsMap.entrySet()) { - StorageUnit storageUnit = mock(StorageUnit.class, RETURNS_DEEP_STUBS); - when(storageUnit.getDataSourcePoolProperties()).thenReturn(entry.getValue()); - result.put(entry.getKey(), storageUnit); - } - return result; - } - private ContextManager mockContextManager() { + ShardingSphereDatabase database = mockShardingSphereDatabase(); MetaDataContexts metaDataContexts = MetaDataContextsFactory.create(mock(MetaDataPersistService.class), new ShardingSphereMetaData(Collections.singleton(database), new ResourceMetaData(Collections.emptyMap()), new RuleMetaData(Arrays.asList(new AuthorityRule(new DefaultAuthorityRuleConfigurationBuilder().build()), @@ -173,7 +143,29 @@ private ContextManager mockContextManager() { ContextManager result = mock(ContextManager.class, RETURNS_DEEP_STUBS); when(result.getMetaDataContexts()).thenReturn(metaDataContexts); when(result.getComputeNodeInstanceContext()).thenReturn(computeNodeInstanceContext); - when(result.getDatabase("normal_db")).thenReturn(database); + return result; + } + + private ShardingSphereDatabase mockShardingSphereDatabase() { + Map storageUnits = createStorageUnits(); + ShardingSphereDatabase result = mock(ShardingSphereDatabase.class, RETURNS_DEEP_STUBS); + when(result.getProtocolType()).thenReturn(TypedSPILoader.getService(DatabaseType.class, "FIXTURE")); + when(result.getName()).thenReturn("normal_db"); + when(result.getResourceMetaData().getAllInstanceDataSourceNames()).thenReturn(storageUnits.keySet()); + when(result.getResourceMetaData().getStorageUnits()).thenReturn(storageUnits); + when(result.getRuleMetaData().getConfigurations()).thenReturn(Collections.emptyList()); + return result; + } + + private Map createStorageUnits() { + Map propsMap = createDataSourceMap().entrySet().stream() + .collect(Collectors.toMap(Entry::getKey, entry -> DataSourcePoolPropertiesCreator.create(entry.getValue()), (oldValue, currentValue) -> oldValue, LinkedHashMap::new)); + Map result = new LinkedHashMap<>(propsMap.size(), 1F); + for (Entry entry : propsMap.entrySet()) { + StorageUnit storageUnit = mock(StorageUnit.class, RETURNS_DEEP_STUBS); + when(storageUnit.getDataSourcePoolProperties()).thenReturn(entry.getValue()); + result.put(entry.getKey(), storageUnit); + } return result; } @@ -186,7 +178,7 @@ private Map createDataSourceMap() { private DataSource createDataSource(final String name) { MockedDataSource result = new MockedDataSource(); - result.setUrl(String.format("jdbc:opengauss://127.0.0.1:5432/%s", name)); + result.setUrl(String.format("jdbc:h2:mem:%s;DB_CLOSE_DELAY=-1;DATABASE_TO_UPPER=false;MODE=MySQL", name)); result.setUsername("root"); result.setPassword(""); result.setMaxPoolSize(50); diff --git a/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportDatabaseConfigurationExecutorTest.java b/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportDatabaseConfigurationExecutorTest.java index 7d49a7ac63914..204ddfbec561c 100644 --- a/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportDatabaseConfigurationExecutorTest.java +++ b/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportDatabaseConfigurationExecutorTest.java @@ -17,8 +17,6 @@ package org.apache.shardingsphere.proxy.backend.handler.distsql.ral.updatable; -import lombok.SneakyThrows; -import org.apache.shardingsphere.distsql.handler.validate.DistSQLDataSourcePoolPropertiesValidator; import org.apache.shardingsphere.distsql.statement.ral.updatable.ImportDatabaseConfigurationStatement; import org.apache.shardingsphere.infra.config.props.ConfigurationProperties; import org.apache.shardingsphere.infra.config.props.ConfigurationPropertyKey; @@ -33,18 +31,8 @@ import org.apache.shardingsphere.infra.spi.exception.ServiceProviderNotFoundException; import org.apache.shardingsphere.infra.spi.type.typed.TypedSPILoader; import org.apache.shardingsphere.mode.manager.ContextManager; -import org.apache.shardingsphere.proxy.backend.context.ProxyContext; -import org.apache.shardingsphere.proxy.backend.util.YamlDatabaseConfigurationImportExecutor; import org.apache.shardingsphere.test.fixture.jdbc.MockedDataSource; -import org.apache.shardingsphere.test.fixture.jdbc.MockedDriver; -import org.apache.shardingsphere.test.mock.AutoMockExtension; -import org.apache.shardingsphere.test.mock.StaticMockSettings; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.internal.configuration.plugins.Plugins; -import org.mockito.junit.jupiter.MockitoSettings; -import org.mockito.quality.Strictness; import javax.sql.DataSource; import java.net.URL; @@ -60,81 +48,68 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -@ExtendWith(AutoMockExtension.class) -@StaticMockSettings(ProxyContext.class) -@MockitoSettings(strictness = Strictness.LENIENT) class ImportDatabaseConfigurationExecutorTest { - private ImportDatabaseConfigurationExecutor executor; - - @BeforeAll - static void setUp() throws ClassNotFoundException { - Class.forName(MockedDriver.class.getName()); - } - @Test void assertImportDatabaseExecutorForSharding() throws SQLException { - assertExecute("sharding_db", "/conf/import/database-sharding.yaml"); + ContextManager contextManager = mockContextManager("sharding_db"); + assertExecute(contextManager, "/conf/import/database-sharding.yaml"); } @Test void assertImportDatabaseExecutorForReadwriteSplitting() throws SQLException { - assertExecute("readwrite_splitting_db", "/conf/import/database-readwrite-splitting.yaml"); + ContextManager contextManager = mockContextManager("readwrite_splitting_db"); + assertExecute(contextManager, "/conf/import/database-readwrite-splitting.yaml"); } @Test void assertImportDatabaseExecutorForEncrypt() throws SQLException { - assertExecute("encrypt_db", "/conf/import/database-encrypt.yaml"); + ContextManager contextManager = mockContextManager("encrypt_db"); + assertExecute(contextManager, "/conf/import/database-encrypt.yaml"); } @Test void assertImportDatabaseExecutorForShadow() throws SQLException { - assertExecute("shadow_db", "/conf/import/database-shadow.yaml"); + ContextManager contextManager = mockContextManager("shadow_db"); + assertExecute(contextManager, "/conf/import/database-shadow.yaml"); } @Test void assertImportDatabaseExecutorForMask() throws SQLException { - assertExecute("mask_db", "/conf/import/database-mask.yaml"); + ContextManager contextManager = mockContextManager("mask_db"); + assertExecute(contextManager, "/conf/import/database-mask.yaml"); } @Test void assertImportExistedDatabase() { - String databaseName = "sharding_db"; - when(ProxyContext.getInstance().databaseExists(databaseName)).thenReturn(true); - assertThrows(DatabaseCreateExistsException.class, () -> assertExecute(databaseName, "/conf/import/database-sharding.yaml")); + ContextManager contextManager = mockContextManager("sharding_db"); + when(contextManager.getMetaDataContexts().getMetaData().containsDatabase("sharding_db")).thenReturn(true); + assertThrows(DatabaseCreateExistsException.class, () -> assertExecute(contextManager, "/conf/import/database-sharding.yaml")); } @Test void assertImportEmptyDatabaseName() { - assertThrows(MissingRequiredDatabaseException.class, () -> assertExecute("sharding_db", "/conf/import/database-empty-database-name.yaml")); + ContextManager contextManager = mockContextManager("sharding_db"); + assertThrows(MissingRequiredDatabaseException.class, () -> assertExecute(contextManager, "/conf/import/database-empty-database-name.yaml")); } @Test void assertImportDuplicatedLogicTable() { - assertThrows(DuplicateRuleException.class, () -> assertExecute("sharding_db", "/conf/import/database-duplicated-logic-table.yaml")); + ContextManager contextManager = mockContextManager("sharding_db"); + assertThrows(DuplicateRuleException.class, () -> assertExecute(contextManager, "/conf/import/database-duplicated-logic-table.yaml")); } @Test void assertImportInvalidAlgorithm() { - assertThrows(ServiceProviderNotFoundException.class, () -> assertExecute("sharding_db", "/conf/import/database-invalid-algorithm.yaml")); + ContextManager contextManager = mockContextManager("sharding_db"); + assertThrows(ServiceProviderNotFoundException.class, () -> assertExecute(contextManager, "/conf/import/database-invalid-algorithm.yaml")); } - private void assertExecute(final String databaseName, final String filePath) throws SQLException { - init(databaseName); + private void assertExecute(final ContextManager contextManager, final String filePath) throws SQLException { + ImportDatabaseConfigurationExecutor executor = new ImportDatabaseConfigurationExecutor(); URL url = ImportDatabaseConfigurationExecutorTest.class.getResource(filePath); assertNotNull(url); - executor.executeUpdate(new ImportDatabaseConfigurationStatement(url.getPath()), mock(ContextManager.class)); - } - - @SneakyThrows({IllegalAccessException.class, NoSuchFieldException.class}) - private void init(final String databaseName) { - ContextManager contextManager = mockContextManager(databaseName); - when(ProxyContext.getInstance().getContextManager()).thenReturn(contextManager); - executor = new ImportDatabaseConfigurationExecutor(); - YamlDatabaseConfigurationImportExecutor databaseConfigImportExecutor = new YamlDatabaseConfigurationImportExecutor(); - Plugins.getMemberAccessor().set(ImportDatabaseConfigurationExecutor.class.getDeclaredField("databaseConfigImportExecutor"), executor, databaseConfigImportExecutor); - Plugins.getMemberAccessor().set( - YamlDatabaseConfigurationImportExecutor.class.getDeclaredField("validateHandler"), databaseConfigImportExecutor, mock(DistSQLDataSourcePoolPropertiesValidator.class)); + executor.executeUpdate(new ImportDatabaseConfigurationStatement(url.getPath()), contextManager); } private ContextManager mockContextManager(final String databaseName) { diff --git a/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportMetaDataExecutorTest.java b/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportMetaDataExecutorTest.java index 89c353b4e5d96..f0a7cb00bb60d 100644 --- a/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportMetaDataExecutorTest.java +++ b/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportMetaDataExecutorTest.java @@ -17,44 +17,51 @@ package org.apache.shardingsphere.proxy.backend.handler.distsql.ral.updatable; -import lombok.SneakyThrows; -import org.apache.groovy.util.Maps; -import org.apache.shardingsphere.distsql.handler.validate.DistSQLDataSourcePoolPropertiesValidator; +import org.apache.shardingsphere.authority.rule.AuthorityRule; +import org.apache.shardingsphere.authority.rule.builder.DefaultAuthorityRuleConfigurationBuilder; import org.apache.shardingsphere.distsql.statement.ral.updatable.ImportMetaDataStatement; +import org.apache.shardingsphere.globalclock.rule.GlobalClockRule; +import org.apache.shardingsphere.globalclock.rule.builder.DefaultGlobalClockRuleConfigurationBuilder; +import org.apache.shardingsphere.infra.config.mode.ModeConfiguration; import org.apache.shardingsphere.infra.config.props.ConfigurationProperties; import org.apache.shardingsphere.infra.config.props.ConfigurationPropertyKey; +import org.apache.shardingsphere.infra.database.core.type.DatabaseType; +import org.apache.shardingsphere.infra.datasource.pool.props.creator.DataSourcePoolPropertiesCreator; import org.apache.shardingsphere.infra.datasource.pool.props.domain.DataSourcePoolProperties; import org.apache.shardingsphere.infra.exception.dialect.exception.syntax.database.DatabaseCreateExistsException; import org.apache.shardingsphere.infra.exception.kernel.metadata.resource.storageunit.EmptyStorageUnitException; +import org.apache.shardingsphere.infra.instance.ComputeNodeInstance; +import org.apache.shardingsphere.infra.instance.ComputeNodeInstanceContext; +import org.apache.shardingsphere.infra.instance.metadata.InstanceMetaData; +import org.apache.shardingsphere.infra.lock.LockContext; +import org.apache.shardingsphere.infra.metadata.ShardingSphereMetaData; import org.apache.shardingsphere.infra.metadata.database.ShardingSphereDatabase; -import org.apache.shardingsphere.infra.metadata.database.resource.node.StorageNode; +import org.apache.shardingsphere.infra.metadata.database.resource.ResourceMetaData; import org.apache.shardingsphere.infra.metadata.database.resource.unit.StorageUnit; -import org.apache.shardingsphere.infra.metadata.database.schema.model.ShardingSphereColumn; -import org.apache.shardingsphere.infra.metadata.database.schema.model.ShardingSphereIndex; -import org.apache.shardingsphere.infra.metadata.database.schema.model.ShardingSphereSchema; -import org.apache.shardingsphere.infra.metadata.database.schema.model.ShardingSphereTable; +import org.apache.shardingsphere.infra.metadata.database.rule.RuleMetaData; +import org.apache.shardingsphere.infra.spi.type.typed.TypedSPILoader; +import org.apache.shardingsphere.infra.util.eventbus.EventBusContext; +import org.apache.shardingsphere.metadata.persist.MetaDataPersistService; import org.apache.shardingsphere.mode.manager.ContextManager; -import org.apache.shardingsphere.proxy.backend.context.ProxyContext; -import org.apache.shardingsphere.proxy.backend.util.YamlDatabaseConfigurationImportExecutor; +import org.apache.shardingsphere.mode.manager.standalone.workerid.StandaloneWorkerIdGenerator; +import org.apache.shardingsphere.mode.metadata.MetaDataContexts; +import org.apache.shardingsphere.mode.metadata.MetaDataContextsFactory; import org.apache.shardingsphere.test.fixture.jdbc.MockedDataSource; -import org.apache.shardingsphere.test.mock.AutoMockExtension; -import org.apache.shardingsphere.test.mock.StaticMockSettings; import org.apache.shardingsphere.test.util.PropertiesBuilder; import org.apache.shardingsphere.test.util.PropertiesBuilder.Property; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.internal.configuration.plugins.Plugins; -import org.mockito.junit.jupiter.MockitoSettings; -import org.mockito.quality.Strictness; +import javax.sql.DataSource; import java.sql.SQLException; -import java.util.Collection; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Map.Entry; import java.util.Objects; +import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -62,96 +69,101 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -@ExtendWith(AutoMockExtension.class) -@StaticMockSettings(ProxyContext.class) -@MockitoSettings(strictness = Strictness.LENIENT) class ImportMetaDataExecutorTest { - private static final String METADATA_VALUE = "eyJtZXRhX2RhdGEiOnsiZGF0YWJhc2VzIjp7InNoYXJkaW5nX2RiIjoiZGF0YWJhc2VOYW1lOiBzaGFyZGluZ19kYlxuZGF0YVNvdXJjZXM6XG4gIGRzXzA6XG4gICAgcGFzc3dvcmQ6IFxu" - + "ICAgIGRhdGFTb3VyY2VDbGFzc05hbWU6IG51bGxcbiAgICB1cmw6IGpkYmM6bXlzcWw6Ly8xMjcuMC4wLjE6MzMwNi9kZW1vX2RzXzA/dXNlU1NMPWZhbHNlXG4gICAgdXNlcm5hbWU6IHJvb3RcbiAgICBtaW5Qb29sU2l6ZTogMVxuICAgI" - + "GNvbm5lY3Rpb25UaW1lb3V0TWlsbGlzZWNvbmRzOiAzMDAwMFxuICAgIG1heExpZmV0aW1lTWlsbGlzZWNvbmRzOiAxODAwMDAwXG4gICAgaWRsZVRpbWVvdXRNaWxsaXNlY29uZHM6IDYwMDAwXG4gICAgbWF4UG9vbFNpemU6IDUwXG4gIG" - + "RzXzE6XG4gICAgcGFzc3dvcmQ6IFxuICAgIGRhdGFTb3VyY2VDbGFzc05hbWU6IG51bGxcbiAgICB1cmw6IGpkYmM6bXlzcWw6Ly8xMjcuMC4wLjE6MzMwNi9kZW1vX2RzXzE/dXNlU1NMPWZhbHNlXG4gICAgdXNlcm5hbWU6IHJvb3RcbiA" - + "gICBtaW5Qb29sU2l6ZTogMVxuICAgIGNvbm5lY3Rpb25UaW1lb3V0TWlsbGlzZWNvbmRzOiAzMDAwMFxuICAgIG1heExpZmV0aW1lTWlsbGlzZWNvbmRzOiAxODAwMDAwXG4gICAgaWRsZVRpbWVvdXRNaWxsaXNlY29uZHM6IDYwMDAwXG4g" - + "ICAgbWF4UG9vbFNpemU6IDUwXG5ydWxlczpcbiJ9LCJwcm9wcyI6InByb3BzOlxuICBzeXN0ZW0tbG9nLWxldmVsOiBJTkZPXG4gIHNxbC1zaG93OiBmYWxzZVxuIiwicnVsZXMiOiJydWxlczpcbi0gIUFVVEhPUklUWVxuICBwcml2aWxlZ" - + "2U6XG4gICAgdHlwZTogQUxMX1BFUk1JVFRFRFxuICB1c2VyczpcbiAgLSBhdXRoZW50aWNhdGlvbk1ldGhvZE5hbWU6ICcnXG4gICAgcGFzc3dvcmQ6IHJvb3RcbiAgICB1c2VyOiByb290QCVcbiJ9fQ=="; + private static final String METADATA_VALUE = "eyJtZXRhX2RhdGEiOnsiZGF0YWJhc2VzIjp7Im5vcm1hbF9kYiI6ImRhdGFiYXNlTmFtZTogbm9ybWFsX2RiXG5kYXRhU291cmNlczpcbiAgZHNfMDpcbiA" + + "gICBwYXNzd29yZDogXG4gICAgdXJsOiBqZGJjOmgyOm1lbTpkZW1vX2RzXzA7REJfQ0xPU0VfREVMQVk9LTE7REFUQUJBU0VfVE9fVVBQRVI9ZmFsc2U7TU9ERT1NeVNRTFxuICAgIHVzZXJuYW1lOiByb290XG4gICAgbWluUG9" + + "vbFNpemU6IDFcbiAgICBtYXhQb29sU2l6ZTogNTBcbiAgZHNfMTpcbiAgICBwYXNzd29yZDogXG4gICAgdXJsOiBqZGJjOmgyOm1lbTpkZW1vX2RzXzE7REJfQ0xPU0VfREVMQVk9LTE7REFUQUJBU0VfVE9fVVBQRVI9ZmFsc2" + + "U7TU9ERT1NeVNRTFxuICAgIHVzZXJuYW1lOiByb290XG4gICAgbWluUG9vbFNpemU6IDFcbiAgICBtYXhQb29sU2l6ZTogNTBcbiJ9LCJwcm9wcyI6InByb3BzOlxuICBzcWwtc2hvdzogdHJ1ZVxuIiwicnVsZXMiOiJydWxlczpcbi0g" + + "IUFVVEhPUklUWVxuICBwcml2aWxlZ2U6XG4gICAgdHlwZTogQUxMX1BFUk1JVFRFRFxuICB1c2VyczpcbiAgLSBhZG1pbjogdHJ1ZVxuICAgIGF1dGhlbnRpY2F0aW9uTWV0aG9kTmFtZTogJydcbiAgIC" + + "BwYXNzd29yZDogcm9vdFxuICAgIHVzZXI6IHJvb3RAJVxuLSAhR0xPQkFMX0NMT0NLXG4gIGVuYWJsZWQ6IGZhbHNlXG4gIHByb3ZpZGVyOiBsb2NhbFxuICB0eXBlOiBUU09cbiJ9fQ=="; - private static final String EMPTY = "empty_metadata"; - - private ImportMetaDataExecutor executor; + private static final String EMPTY_DATABASE_NAME = "empty_metadata"; private final Map featureMap = new HashMap<>(1, 1F); @BeforeEach void setup() { - featureMap.put(EMPTY, "/conf/import/empty-metadata.json"); + featureMap.put(EMPTY_DATABASE_NAME, "/conf/import/empty-metadata.json"); } @Test void assertImportEmptyMetaData() { - init(null); + ImportMetaDataExecutor executor = new ImportMetaDataExecutor(); ContextManager contextManager = mock(ContextManager.class, RETURNS_DEEP_STUBS); assertThrows(EmptyStorageUnitException.class, () -> executor.executeUpdate( - new ImportMetaDataStatement(null, Objects.requireNonNull(ImportMetaDataExecutorTest.class.getResource(featureMap.get(EMPTY))).getPath()), contextManager)); + new ImportMetaDataStatement(null, Objects.requireNonNull(ImportMetaDataExecutorTest.class.getResource(featureMap.get(EMPTY_DATABASE_NAME))).getPath()), contextManager)); } @Test void assertImportMetaDataFromJsonValue() throws SQLException { - init(EMPTY); ContextManager contextManager = mock(ContextManager.class, RETURNS_DEEP_STUBS); + ImportMetaDataExecutor executor = new ImportMetaDataExecutor(); executor.executeUpdate(new ImportMetaDataStatement(METADATA_VALUE, null), contextManager); - assertNotNull(contextManager.getDatabase("sharding_db")); + assertNotNull(contextManager.getDatabase("normal_db")); } @Test void assertImportExistedMetaDataFromFile() { - init(EMPTY); - ContextManager contextManager = mock(ContextManager.class, RETURNS_DEEP_STUBS); + ImportMetaDataExecutor executor = new ImportMetaDataExecutor(); + ContextManager contextManager = mockContextManager(); assertThrows(DatabaseCreateExistsException.class, () -> executor.executeUpdate( - new ImportMetaDataStatement(null, Objects.requireNonNull(ImportMetaDataExecutorTest.class.getResource(featureMap.get(EMPTY))).getPath()), contextManager)); + new ImportMetaDataStatement(null, Objects.requireNonNull(ImportMetaDataExecutorTest.class.getResource(featureMap.get(EMPTY_DATABASE_NAME))).getPath()), contextManager)); } - @SneakyThrows({IllegalAccessException.class, NoSuchFieldException.class}) - private void init(final String feature) { - executor = new ImportMetaDataExecutor(); - ContextManager contextManager = mockContextManager(feature); - when(ProxyContext.getInstance().getContextManager()).thenReturn(contextManager); - when(ProxyContext.getInstance().databaseExists(feature)).thenReturn(true); - YamlDatabaseConfigurationImportExecutor databaseConfigImportExecutor = new YamlDatabaseConfigurationImportExecutor(); - Plugins.getMemberAccessor().set(ImportMetaDataExecutor.class.getDeclaredField("databaseConfigImportExecutor"), executor, databaseConfigImportExecutor); - Plugins.getMemberAccessor().set( - YamlDatabaseConfigurationImportExecutor.class.getDeclaredField("validateHandler"), databaseConfigImportExecutor, mock(DistSQLDataSourcePoolPropertiesValidator.class)); + private ContextManager mockContextManager() { + ShardingSphereDatabase database = mockShardingSphereDatabase(); + MetaDataContexts metaDataContexts = MetaDataContextsFactory.create(mock(MetaDataPersistService.class), new ShardingSphereMetaData(Collections.singleton(database), + new ResourceMetaData(Collections.emptyMap()), + new RuleMetaData(Arrays.asList(new AuthorityRule(new DefaultAuthorityRuleConfigurationBuilder().build()), + new GlobalClockRule(new DefaultGlobalClockRuleConfigurationBuilder().build()))), + new ConfigurationProperties(PropertiesBuilder.build(new Property(ConfigurationPropertyKey.SQL_SHOW.getKey(), "true"))))); + ComputeNodeInstanceContext computeNodeInstanceContext = new ComputeNodeInstanceContext( + new ComputeNodeInstance(mock(InstanceMetaData.class)), new ModeConfiguration("Standalone", null), new EventBusContext()); + computeNodeInstanceContext.init(new StandaloneWorkerIdGenerator(), mock(LockContext.class)); + ContextManager result = mock(ContextManager.class, RETURNS_DEEP_STUBS); + when(result.getMetaDataContexts()).thenReturn(metaDataContexts); + when(result.getComputeNodeInstanceContext()).thenReturn(computeNodeInstanceContext); + return result; } - private ContextManager mockContextManager(final String feature) { - ContextManager result = mock(ContextManager.class, RETURNS_DEEP_STUBS); - when(result.getMetaDataContexts().getMetaData().getProps()) - .thenReturn(new ConfigurationProperties(PropertiesBuilder.build(new Property(ConfigurationPropertyKey.PROXY_FRONTEND_DATABASE_PROTOCOL_TYPE.getKey(), "MySQL")))); - if (null != feature) { - ShardingSphereDatabase database = mock(ShardingSphereDatabase.class, RETURNS_DEEP_STUBS); - when(database.getName()).thenReturn(feature); - when(database.getSchema("foo_db")).thenReturn(new ShardingSphereSchema("foo_db", createTables(), Collections.emptyList())); - Map storageUnits = createStorageUnits(); - when(database.getResourceMetaData().getStorageUnits()).thenReturn(storageUnits); - when(result.getMetaDataContexts().getMetaData().getAllDatabases()).thenReturn(Collections.singleton(database)); - when(result.getMetaDataContexts().getMetaData().getDatabase(feature)).thenReturn(database); - } + private ShardingSphereDatabase mockShardingSphereDatabase() { + Map storageUnits = createStorageUnits(); + ShardingSphereDatabase result = mock(ShardingSphereDatabase.class, RETURNS_DEEP_STUBS); + when(result.getProtocolType()).thenReturn(TypedSPILoader.getService(DatabaseType.class, "FIXTURE")); + when(result.getName()).thenReturn(EMPTY_DATABASE_NAME); + when(result.getResourceMetaData().getAllInstanceDataSourceNames()).thenReturn(storageUnits.keySet()); + when(result.getResourceMetaData().getStorageUnits()).thenReturn(storageUnits); + when(result.getRuleMetaData().getConfigurations()).thenReturn(Collections.emptyList()); return result; } private Map createStorageUnits() { - Map result = new LinkedHashMap<>(2, 1F); - DataSourcePoolProperties dataSourcePoolProps0 = mock(DataSourcePoolProperties.class, RETURNS_DEEP_STUBS); - when(dataSourcePoolProps0.getConnectionPropertySynonyms().getStandardProperties()).thenReturn(Maps.of("url", "jdbc:mock://127.0.0.1/ds_0", "username", "test")); - result.put("ds_0", new StorageUnit(mock(StorageNode.class), dataSourcePoolProps0, new MockedDataSource())); - DataSourcePoolProperties dataSourcePoolProps1 = mock(DataSourcePoolProperties.class, RETURNS_DEEP_STUBS); - when(dataSourcePoolProps1.getConnectionPropertySynonyms().getStandardProperties()).thenReturn(Maps.of("url", "jdbc:mock://127.0.0.1/ds_1", "username", "test")); - result.put("ds_1", new StorageUnit(mock(StorageNode.class), dataSourcePoolProps1, new MockedDataSource())); + Map propsMap = createDataSourceMap().entrySet().stream() + .collect(Collectors.toMap(Entry::getKey, entry -> DataSourcePoolPropertiesCreator.create(entry.getValue()), (oldValue, currentValue) -> oldValue, LinkedHashMap::new)); + Map result = new LinkedHashMap<>(propsMap.size(), 1F); + for (Entry entry : propsMap.entrySet()) { + StorageUnit storageUnit = mock(StorageUnit.class, RETURNS_DEEP_STUBS); + when(storageUnit.getDataSourcePoolProperties()).thenReturn(entry.getValue()); + result.put(entry.getKey(), storageUnit); + } return result; } - private Collection createTables() { - Collection columns = Collections.singleton(new ShardingSphereColumn("order_id", 0, false, false, false, true, false, false)); - Collection indexes = Collections.singleton(new ShardingSphereIndex("primary", Collections.emptyList(), false)); - return Collections.singletonList(new ShardingSphereTable("t_order", columns, indexes, Collections.emptyList())); + private Map createDataSourceMap() { + Map result = new LinkedHashMap<>(2, 1F); + result.put("ds_0", createDataSource("demo_ds_0")); + result.put("ds_1", createDataSource("demo_ds_1")); + return result; + } + + private DataSource createDataSource(final String name) { + MockedDataSource result = new MockedDataSource(); + result.setUrl(String.format("jdbc:h2:mem:%s;DB_CLOSE_DELAY=-1;DATABASE_TO_UPPER=false;MODE=MySQL", name)); + result.setUsername("root"); + result.setPassword(""); + result.setMaxPoolSize(50); + result.setMinPoolSize(1); + return result; } } From ccd17f66965b452a625ee051531953085dbf847b Mon Sep 17 00:00:00 2001 From: jiangML <1060319118@qq.com> Date: Tue, 24 Dec 2024 20:55:38 +0800 Subject: [PATCH 3/3] Fix checkstyle error --- .../proxy/backend/util/MetaDataImportExecutor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/MetaDataImportExecutor.java b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/MetaDataImportExecutor.java index a7e22f3d8298e..016b01f00b1e8 100644 --- a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/MetaDataImportExecutor.java +++ b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/MetaDataImportExecutor.java @@ -49,7 +49,7 @@ public MetaDataImportExecutor(final ContextManager contextManager) { /** * Import cluster configurations. - * + * * @param exportedMetaData exported metadata * @throws SQLException SQL exception */