Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor export and import metadata #34136

Merged
merged 3 commits into from
Dec 25, 2024
Merged

Refactor export and import metadata #34136

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ab2aead
Refactor export and import metadata
jiangML Dec 24, 2024
9dac2f8
Fix test error
jiangML Dec 24, 2024
ccd17f6
Fix checkstyle error
jiangML Dec 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions ...here/proxy/backend/handler/distsql/ral/queryable/ExportDatabaseConfigurationExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import org.apache.shardingsphere.infra.merge.result.impl.local.LocalDataQueryResultRow;
import org.apache.shardingsphere.infra.metadata.database.ShardingSphereDatabase;
import org.apache.shardingsphere.mode.manager.ContextManager;
import org.apache.shardingsphere.proxy.backend.util.DatabaseExportMetaDataGenerator;
import org.apache.shardingsphere.proxy.backend.util.ExportUtils;

import java.util.Collection;
Expand All @@ -44,13 +45,13 @@ public Collection<String> getColumnNames(final ExportDatabaseConfigurationStatem

@Override
public Collection<LocalDataQueryResultRow> getRows(final ExportDatabaseConfigurationStatement sqlStatement, final ContextManager contextManager) {
String exportedData = ExportUtils.generateExportDatabaseData(database);
if (!sqlStatement.getFilePath().isPresent()) {
return Collections.singleton(new LocalDataQueryResultRow(exportedData));
String exportedData = new DatabaseExportMetaDataGenerator(database).generateYAMLFormat();
if (sqlStatement.getFilePath().isPresent()) {
String filePath = sqlStatement.getFilePath().get();
ExportUtils.exportToFile(filePath, exportedData);
return Collections.singleton(new LocalDataQueryResultRow(String.format("Successfully exported to: '%s'", filePath)));
}
String filePath = sqlStatement.getFilePath().get();
ExportUtils.exportToFile(filePath, exportedData);
return Collections.singleton(new LocalDataQueryResultRow(String.format("Successfully exported to: '%s'", filePath)));
return Collections.singleton(new LocalDataQueryResultRow(exportedData));
}

@Override
Expand Down
93 changes: 5 additions & 88 deletions ...he/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportMetaDataExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,32 +20,15 @@
import org.apache.commons.codec.binary.Base64;
import org.apache.shardingsphere.distsql.handler.engine.query.DistSQLQueryExecutor;
import org.apache.shardingsphere.distsql.statement.ral.queryable.export.ExportMetaDataStatement;
import org.apache.shardingsphere.globalclock.provider.GlobalClockProvider;
import org.apache.shardingsphere.globalclock.rule.GlobalClockRule;
import org.apache.shardingsphere.infra.config.rule.RuleConfiguration;
import org.apache.shardingsphere.infra.merge.result.impl.local.LocalDataQueryResultRow;
import org.apache.shardingsphere.infra.metadata.ShardingSphereMetaData;
import org.apache.shardingsphere.infra.metadata.database.ShardingSphereDatabase;
import org.apache.shardingsphere.infra.spi.type.ordered.OrderedSPILoader;
import org.apache.shardingsphere.infra.util.json.JsonUtils;
import org.apache.shardingsphere.infra.util.yaml.YamlEngine;
import org.apache.shardingsphere.infra.yaml.config.swapper.rule.YamlRuleConfigurationSwapper;
import org.apache.shardingsphere.mode.manager.ContextManager;
import org.apache.shardingsphere.mode.metadata.decorator.RuleConfigurationPersistDecorateEngine;
import org.apache.shardingsphere.proxy.backend.context.ProxyContext;
import org.apache.shardingsphere.proxy.backend.distsql.export.ExportedClusterInfo;
import org.apache.shardingsphere.proxy.backend.distsql.export.ExportedMetaData;
import org.apache.shardingsphere.proxy.backend.distsql.export.ExportedSnapshotInfo;
import org.apache.shardingsphere.proxy.backend.util.ClusterExportMetaDataGenerator;
import org.apache.shardingsphere.proxy.backend.util.ExportUtils;

import java.time.LocalDateTime;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;

/**
* Export metadata executor.
Expand All @@ -59,80 +42,14 @@ public Collection<String> getColumnNames(final ExportMetaDataStatement sqlStatem

@Override
public Collection<LocalDataQueryResultRow> getRows(final ExportMetaDataStatement sqlStatement, final ContextManager contextManager) {
String exportedData = generateExportData(contextManager.getMetaDataContexts().getMetaData());
String exportedData = new ClusterExportMetaDataGenerator(contextManager).generateJsonFormat();
String instanceId = contextManager.getComputeNodeInstanceContext().getInstance().getMetaData().getId();
if (sqlStatement.getFilePath().isPresent()) {
String filePath = sqlStatement.getFilePath().get();
ExportUtils.exportToFile(filePath, exportedData);
return Collections.singleton(new LocalDataQueryResultRow(contextManager.getComputeNodeInstanceContext().getInstance().getMetaData().getId(), LocalDateTime.now(),
String.format("Successfully exported to：'%s'", filePath)));
}
return Collections.singleton(new LocalDataQueryResultRow(
contextManager.getComputeNodeInstanceContext().getInstance().getMetaData().getId(), LocalDateTime.now(), Base64.encodeBase64String(exportedData.getBytes())));
}

private String generateExportData(final ShardingSphereMetaData metaData) {
ProxyContext proxyContext = ProxyContext.getInstance();
ExportedMetaData exportedMetaData = new ExportedMetaData();
exportedMetaData.setDatabases(getDatabases(proxyContext));
exportedMetaData.setProps(generatePropsData(metaData.getProps().getProps()));
RuleConfigurationPersistDecorateEngine ruleConfigPersistDecorateEngine =
new RuleConfigurationPersistDecorateEngine(ProxyContext.getInstance().getContextManager().getComputeNodeInstanceContext());
Collection<RuleConfiguration> ruleConfigs = ruleConfigPersistDecorateEngine.decorate(metaData.getGlobalRuleMetaData().getConfigurations());
exportedMetaData.setRules(generateRulesData(ruleConfigs));
ExportedClusterInfo exportedClusterInfo = new ExportedClusterInfo();
exportedClusterInfo.setMetaData(exportedMetaData);
generateSnapshotInfo(metaData, exportedClusterInfo);
return JsonUtils.toJsonString(exportedClusterInfo);
}

private Map<String, String> getDatabases(final ProxyContext proxyContext) {
Collection<String> databaseNames = proxyContext.getAllDatabaseNames();
Map<String, String> result = new LinkedHashMap<>(databaseNames.size(), 1F);
for (String each : databaseNames) {
ShardingSphereDatabase database = proxyContext.getContextManager().getDatabase(each);
if (database.getResourceMetaData().getAllInstanceDataSourceNames().isEmpty()) {
continue;
}
result.put(each, ExportUtils.generateExportDatabaseData(database));
}
return result;
}

private String generatePropsData(final Properties props) {
if (props.isEmpty()) {
return "";
}
StringBuilder result = new StringBuilder();
result.append("props:").append(System.lineSeparator());
props.forEach((key, value) -> {
if (null != value && !"".equals(value)) {
result.append(" ").append(key).append(": ").append(value).append(System.lineSeparator());
}
});
return result.toString();
}

@SuppressWarnings({"rawtypes", "unchecked"})
private String generateRulesData(final Collection<RuleConfiguration> rules) {
if (rules.isEmpty()) {
return "";
}
StringBuilder result = new StringBuilder();
result.append("rules:").append(System.lineSeparator());
for (Entry<RuleConfiguration, YamlRuleConfigurationSwapper> entry : OrderedSPILoader.getServices(YamlRuleConfigurationSwapper.class, rules).entrySet()) {
result.append(YamlEngine.marshal(Collections.singletonList(entry.getValue().swapToYamlConfiguration(entry.getKey()))));
}
return result.toString();
}

private void generateSnapshotInfo(final ShardingSphereMetaData metaData, final ExportedClusterInfo exportedClusterInfo) {
GlobalClockRule globalClockRule = metaData.getGlobalRuleMetaData().getSingleRule(GlobalClockRule.class);
if (globalClockRule.getConfiguration().isEnabled()) {
ExportedSnapshotInfo snapshotInfo = new ExportedSnapshotInfo();
snapshotInfo.setCsn(String.valueOf(globalClockRule.getGlobalClockProvider().map(GlobalClockProvider::getCurrentTimestamp).orElse(0L)));
snapshotInfo.setCreateTime(LocalDateTime.now());
exportedClusterInfo.setSnapshotInfo(snapshotInfo);
return Collections.singleton(new LocalDataQueryResultRow(instanceId, LocalDateTime.now(), String.format("Successfully exported to：'%s'", filePath)));
}
return Collections.singleton(new LocalDataQueryResultRow(instanceId, LocalDateTime.now(), Base64.encodeBase64String(exportedData.getBytes())));
}

@Override
Expand Down
16 changes: 9 additions & 7 deletions ...here/proxy/backend/handler/distsql/ral/updatable/ImportDatabaseConfigurationExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,33 +19,35 @@

import org.apache.shardingsphere.distsql.handler.engine.update.DistSQLUpdateExecutor;
import org.apache.shardingsphere.distsql.statement.ral.updatable.ImportDatabaseConfigurationStatement;
import org.apache.shardingsphere.infra.exception.generic.FileIOException;
import org.apache.shardingsphere.infra.util.yaml.YamlEngine;
import org.apache.shardingsphere.mode.manager.ContextManager;
import org.apache.shardingsphere.proxy.backend.config.yaml.YamlProxyDatabaseConfiguration;
import org.apache.shardingsphere.infra.exception.generic.FileIOException;
import org.apache.shardingsphere.proxy.backend.util.YamlDatabaseConfigurationImportExecutor;
import org.apache.shardingsphere.proxy.backend.util.MetaDataImportExecutor;

import java.io.File;
import java.io.IOException;
import java.sql.SQLException;
import java.util.Collections;

/**
* Import database configuration executor.
*/
public final class ImportDatabaseConfigurationExecutor implements DistSQLUpdateExecutor<ImportDatabaseConfigurationStatement> {

private final YamlDatabaseConfigurationImportExecutor databaseConfigImportExecutor = new YamlDatabaseConfigurationImportExecutor();

@Override
public void executeUpdate(final ImportDatabaseConfigurationStatement sqlStatement, final ContextManager contextManager) throws SQLException {
YamlProxyDatabaseConfiguration yamlConfig = getYamlProxyDatabaseConfiguration(sqlStatement);
new MetaDataImportExecutor(contextManager).importDatabaseConfigurations(Collections.singletonList(yamlConfig));
}

private YamlProxyDatabaseConfiguration getYamlProxyDatabaseConfiguration(final ImportDatabaseConfigurationStatement sqlStatement) {
File file = new File(sqlStatement.getFilePath());
YamlProxyDatabaseConfiguration yamlConfig;
try {
yamlConfig = YamlEngine.unmarshal(file, YamlProxyDatabaseConfiguration.class);
return YamlEngine.unmarshal(file, YamlProxyDatabaseConfiguration.class);
} catch (final IOException ignore) {
throw new FileIOException(file);
}
databaseConfigImportExecutor.importDatabaseConfiguration(yamlConfig);
}

@Override
Expand Down
51 changes: 12 additions & 39 deletions ...he/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportMetaDataExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,69 +21,42 @@
import org.apache.commons.io.FileUtils;
import org.apache.shardingsphere.distsql.handler.engine.update.DistSQLUpdateExecutor;
import org.apache.shardingsphere.distsql.statement.ral.updatable.ImportMetaDataStatement;
import org.apache.shardingsphere.infra.config.rule.RuleConfiguration;
import org.apache.shardingsphere.infra.exception.generic.FileIOException;
import org.apache.shardingsphere.infra.util.json.JsonUtils;
import org.apache.shardingsphere.infra.util.yaml.YamlEngine;
import org.apache.shardingsphere.infra.yaml.config.swapper.rule.YamlRuleConfigurationSwapperEngine;
import org.apache.shardingsphere.mode.manager.ContextManager;
import org.apache.shardingsphere.proxy.backend.config.yaml.YamlProxyDatabaseConfiguration;
import org.apache.shardingsphere.proxy.backend.config.yaml.YamlProxyServerConfiguration;
import org.apache.shardingsphere.proxy.backend.distsql.export.ExportedClusterInfo;
import org.apache.shardingsphere.proxy.backend.distsql.export.ExportedMetaData;
import org.apache.shardingsphere.infra.exception.generic.FileIOException;
import org.apache.shardingsphere.proxy.backend.util.YamlDatabaseConfigurationImportExecutor;
import org.apache.shardingsphere.proxy.backend.util.MetaDataImportExecutor;

import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.sql.SQLException;
import java.util.Collection;

/**
* Import meta data executor.
*/
public final class ImportMetaDataExecutor implements DistSQLUpdateExecutor<ImportMetaDataStatement> {

private final YamlRuleConfigurationSwapperEngine ruleConfigSwapperEngine = new YamlRuleConfigurationSwapperEngine();

private final YamlDatabaseConfigurationImportExecutor databaseConfigImportExecutor = new YamlDatabaseConfigurationImportExecutor();

@Override
public void executeUpdate(final ImportMetaDataStatement sqlStatement, final ContextManager contextManager) throws SQLException {
String jsonMetaDataConfig;
if (sqlStatement.getFilePath().isPresent()) {
File file = new File(sqlStatement.getFilePath().get());
try {
jsonMetaDataConfig = FileUtils.readFileToString(file, Charset.defaultCharset());
} catch (final IOException ignore) {
throw new FileIOException(file);
}
} else {
jsonMetaDataConfig = new String(Base64.decodeBase64(sqlStatement.getMetaDataValue()));
}
String jsonMetaDataConfig = sqlStatement.getFilePath().isPresent() ? getMetaDataFromFile(sqlStatement) : getMetaDataFromConsole(sqlStatement);
ExportedClusterInfo exportedClusterInfo = JsonUtils.fromJsonString(jsonMetaDataConfig, ExportedClusterInfo.class);
ExportedMetaData exportedMetaData = exportedClusterInfo.getMetaData();
importServerConfiguration(contextManager, exportedMetaData);
importDatabase(exportedMetaData);
new MetaDataImportExecutor(contextManager).importClusterConfigurations(exportedMetaData);
}

private void importServerConfiguration(final ContextManager contextManager, final ExportedMetaData exportedMetaData) throws SQLException {
YamlProxyServerConfiguration yamlServerConfig = YamlEngine.unmarshal(exportedMetaData.getRules() + System.lineSeparator() + exportedMetaData.getProps(), YamlProxyServerConfiguration.class);
if (null == yamlServerConfig) {
return;
}
Collection<RuleConfiguration> rules = ruleConfigSwapperEngine.swapToRuleConfigurations(yamlServerConfig.getRules());
for (RuleConfiguration each : rules) {
contextManager.getPersistServiceFacade().getMetaDataManagerPersistService().alterGlobalRuleConfiguration(each);
private String getMetaDataFromFile(final ImportMetaDataStatement sqlStatement) {
File file = new File(sqlStatement.getFilePath().get());
try {
return FileUtils.readFileToString(file, Charset.defaultCharset());
} catch (final IOException ignore) {
throw new FileIOException(file);
}
contextManager.getPersistServiceFacade().getMetaDataManagerPersistService().alterProperties(yamlServerConfig.getProps());
}

private void importDatabase(final ExportedMetaData exportedMetaData) throws SQLException {
for (String each : exportedMetaData.getDatabases().values()) {
YamlProxyDatabaseConfiguration yamlDatabaseConfig = YamlEngine.unmarshal(each, YamlProxyDatabaseConfiguration.class);
databaseConfigImportExecutor.importDatabaseConfiguration(yamlDatabaseConfig);
}
private String getMetaDataFromConsole(final ImportMetaDataStatement sqlStatement) {
return new String(Base64.decodeBase64(sqlStatement.getMetaDataValue()));
}

@Override
Expand Down
Loading
Loading