diff --git a/ChangeLog.md b/ChangeLog.md index 7bc5d262..a2b879e3 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,5 +1,9 @@ # 更新历史 +### V1.7.4 (2021-07-08) +- 增加基础表和表字段过滤能力 +- 修复区块多笔交易场景下,事件解析只解析首条的bug + ### V1.7.3 (2021-05-21) - 服务方式启动整合编译插件,配置合约即可完成导出 - sdk修复部分bug diff --git a/WeBankBlockchain-Data-Export-sdk/WeBankBlockchain-Data-Export-common/src/main/java/com/webank/blockchain/data/export/common/bo/data/BlockRawDataBO.java b/WeBankBlockchain-Data-Export-sdk/WeBankBlockchain-Data-Export-common/src/main/java/com/webank/blockchain/data/export/common/bo/data/BlockRawDataBO.java index 9da22667..60ceb583 100644 --- a/WeBankBlockchain-Data-Export-sdk/WeBankBlockchain-Data-Export-common/src/main/java/com/webank/blockchain/data/export/common/bo/data/BlockRawDataBO.java +++ b/WeBankBlockchain-Data-Export-sdk/WeBankBlockchain-Data-Export-common/src/main/java/com/webank/blockchain/data/export/common/bo/data/BlockRawDataBO.java @@ -31,5 +31,4 @@ public class BlockRawDataBO { private String gasUsed; private String signatureList; private String transactionList; - protected Date depotUpdatetime; } diff --git a/WeBankBlockchain-Data-Export-sdk/WeBankBlockchain-Data-Export-common/src/main/java/com/webank/blockchain/data/export/common/entity/ExportConfig.java b/WeBankBlockchain-Data-Export-sdk/WeBankBlockchain-Data-Export-common/src/main/java/com/webank/blockchain/data/export/common/entity/ExportConfig.java index 98be5d32..e7327810 100644 --- a/WeBankBlockchain-Data-Export-sdk/WeBankBlockchain-Data-Export-common/src/main/java/com/webank/blockchain/data/export/common/entity/ExportConfig.java +++ b/WeBankBlockchain-Data-Export-sdk/WeBankBlockchain-Data-Export-common/src/main/java/com/webank/blockchain/data/export/common/entity/ExportConfig.java @@ -33,6 +33,11 @@ public class ExportConfig { */ private Map>> ignoreParam = new HashMap<>(); + /** + * ex: Map>> + */ + private Map> ignoreBasicDataTableParam = new HashMap<>(); + /** * ex: Map>> */ diff --git a/WeBankBlockchain-Data-Export-sdk/WeBankBlockchain-Data-Export-common/src/main/java/com/webank/blockchain/data/export/common/enums/DataType.java b/WeBankBlockchain-Data-Export-sdk/WeBankBlockchain-Data-Export-common/src/main/java/com/webank/blockchain/data/export/common/enums/DataType.java index 3ebfc737..5d51ab5b 100644 --- a/WeBankBlockchain-Data-Export-sdk/WeBankBlockchain-Data-Export-common/src/main/java/com/webank/blockchain/data/export/common/enums/DataType.java +++ b/WeBankBlockchain-Data-Export-sdk/WeBankBlockchain-Data-Export-common/src/main/java/com/webank/blockchain/data/export/common/enums/DataType.java @@ -36,6 +36,15 @@ public static List getDefault(){ return ListUtil.toList(NULL); } + public static DataType getDataType(String tableName){ + for(DataType dataType : DataType.values()){ + if (dataType.tableName.equals(tableName)){ + return dataType; + } + } + return NULL; + } + public static List getTables(List dataTypes){ if (CollectionUtil.isEmpty(dataTypes)) { return Collections.emptyList(); diff --git a/WeBankBlockchain-Data-Export-sdk/WeBankBlockchain-Data-Export-common/src/main/java/com/webank/blockchain/data/export/common/enums/IgnoreBasicDataParam.java b/WeBankBlockchain-Data-Export-sdk/WeBankBlockchain-Data-Export-common/src/main/java/com/webank/blockchain/data/export/common/enums/IgnoreBasicDataParam.java new file mode 100644 index 00000000..a29dc01d --- /dev/null +++ b/WeBankBlockchain-Data-Export-sdk/WeBankBlockchain-Data-Export-common/src/main/java/com/webank/blockchain/data/export/common/enums/IgnoreBasicDataParam.java @@ -0,0 +1,75 @@ +package com.webank.blockchain.data.export.common.enums; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * @author wesleywang + * @Description: + * @date 2021/6/23 + */ +@AllArgsConstructor +@Getter +public class IgnoreBasicDataParam { + + @Getter + public enum IgnoreBasicDataTable { + BLOCK_RAW_DATA_TABLE, + TX_RAW_DATA_TABLE, + TX_RECEIPT_RAW_DATA_TABLE; + + IgnoreBasicDataTable() { } + } + + @Getter + public enum BlockRawDataParams { + DB_HASH, + EXTRA_DATA, + GAS_LIMIT, + GAS_USED, + LOGS_BLOOM, + PARENT_HASH, + RECEIPTS_ROOT, + SEALER, + SEALER_LIST, + SIGNATURE_LIST, + STATE_ROOT, + TRANSACTION_LIST, + TRANSACTIONS_ROOT; + + BlockRawDataParams() { } + } + + @Getter + public enum TxReceiptRawDataParams { + FROM, + GAS_USED, + LOGS, + INPUT, + MESSAGE, + OUTPUT, + LOGS_BLOOM, + ROOT, + TO, + TX_INDEX, + TX_PROOF, + RECEIPT_PROOF; + + TxReceiptRawDataParams() { } + } + + @Getter + public enum TxRawDataParams { + FROM, + GAS, + GAS_PRICE, + INPUT, + NONCE, + VALUE, + TO; + + + TxRawDataParams() { } + } + +} diff --git a/WeBankBlockchain-Data-Export-sdk/WeBankBlockchain-Data-Export-parser/src/main/java/com/webank/blockchain/data/export/parser/handler/BlockCrawlerHandler.java b/WeBankBlockchain-Data-Export-sdk/WeBankBlockchain-Data-Export-parser/src/main/java/com/webank/blockchain/data/export/parser/handler/BlockCrawlerHandler.java index 80457bbf..678619cd 100644 --- a/WeBankBlockchain-Data-Export-sdk/WeBankBlockchain-Data-Export-parser/src/main/java/com/webank/blockchain/data/export/parser/handler/BlockCrawlerHandler.java +++ b/WeBankBlockchain-Data-Export-sdk/WeBankBlockchain-Data-Export-parser/src/main/java/com/webank/blockchain/data/export/parser/handler/BlockCrawlerHandler.java @@ -16,10 +16,16 @@ import com.webank.blockchain.data.export.common.bo.data.BlockDetailInfoBO; import com.webank.blockchain.data.export.common.bo.data.BlockDetailInfoBO.Status; import com.webank.blockchain.data.export.common.bo.data.BlockRawDataBO; +import com.webank.blockchain.data.export.common.entity.ExportConfig; +import com.webank.blockchain.data.export.common.entity.ExportConstant; +import com.webank.blockchain.data.export.common.enums.IgnoreBasicDataParam; import com.webank.blockchain.data.export.common.tools.DateUtils; import com.webank.blockchain.data.export.common.tools.JacksonUtils; import org.fisco.bcos.sdk.client.protocol.response.BcosBlock.Block; +import java.util.List; +import java.util.Map; + /** * BlockCrawlerHandler is responsible for crawling block info. * @@ -36,7 +42,6 @@ public class BlockCrawlerHandler { * transaction's count in current block, block hash and block timestamp. * * @param block - * @param blockHeight * @return boolean */ public static BlockDetailInfoBO handleBlockDetail(Block block) { @@ -50,24 +55,68 @@ public static BlockDetailInfoBO handleBlockDetail(Block block) { } public static BlockRawDataBO handleBlockRawData(Block block) { + ExportConfig config = ExportConstant.getCurrentContext().getConfig(); + Map> ignoreBasicDataTableParam = config.getIgnoreBasicDataTableParam(); BlockRawDataBO blockRawDataBO = new BlockRawDataBO(); blockRawDataBO.setBlockHeight(block.getNumber().longValue()); blockRawDataBO.setBlockHash(block.getHash()); blockRawDataBO.setBlockTimeStamp(DateUtils.hexStrToDate(block.getTimestamp())); - blockRawDataBO.setDbHash(block.getDbHash()); - blockRawDataBO.setExtraData(JacksonUtils.toJson(block.getExtraData())); - blockRawDataBO.setGasLimit(block.getGasLimit()); - blockRawDataBO.setGasUsed(block.getGasUsed()); - blockRawDataBO.setLogsBloom(block.getLogsBloom()); - blockRawDataBO.setParentHash(block.getParentHash()); - blockRawDataBO.setReceiptsRoot(block.getReceiptsRoot()); - blockRawDataBO.setSealer(block.getSealer()); - blockRawDataBO.setSealerList(JacksonUtils.toJson(block.getSealerList())); - blockRawDataBO.setSignatureList(JacksonUtils.toJson(block.getSignatureList())); - blockRawDataBO.setStateRoot(block.getStateRoot()); - blockRawDataBO.setTransactionsRoot(block.getTransactionsRoot()); - blockRawDataBO.setTransactionList(JacksonUtils.toJson(block.getTransactions())); - blockRawDataBO.setBlockObject(JacksonUtils.toJson(block)); + if (!ignoreBasicDataTableParam.containsKey(IgnoreBasicDataParam.IgnoreBasicDataTable.BLOCK_RAW_DATA_TABLE.name())) { + blockRawDataBO.setDbHash(block.getDbHash()); + blockRawDataBO.setExtraData(JacksonUtils.toJson(block.getExtraData())); + blockRawDataBO.setGasLimit(block.getGasLimit()); + blockRawDataBO.setGasUsed(block.getGasUsed()); + blockRawDataBO.setLogsBloom(block.getLogsBloom()); + blockRawDataBO.setParentHash(block.getParentHash()); + blockRawDataBO.setReceiptsRoot(block.getReceiptsRoot()); + blockRawDataBO.setSealer(block.getSealer()); + blockRawDataBO.setSealerList(JacksonUtils.toJson(block.getSealerList())); + blockRawDataBO.setSignatureList(JacksonUtils.toJson(block.getSignatureList())); + blockRawDataBO.setStateRoot(block.getStateRoot()); + blockRawDataBO.setTransactionsRoot(block.getTransactionsRoot()); + blockRawDataBO.setTransactionList(JacksonUtils.toJson(block.getTransactions())); + }else { + List params = ignoreBasicDataTableParam.get(IgnoreBasicDataParam.IgnoreBasicDataTable.BLOCK_RAW_DATA_TABLE.name()); + if (!params.contains(IgnoreBasicDataParam.BlockRawDataParams.DB_HASH.name())) { + blockRawDataBO.setDbHash(block.getDbHash()); + } + if (!params.contains(IgnoreBasicDataParam.BlockRawDataParams.EXTRA_DATA.name())) { + blockRawDataBO.setExtraData(JacksonUtils.toJson(block.getExtraData())); + } + if (!params.contains(IgnoreBasicDataParam.BlockRawDataParams.GAS_LIMIT.name())) { + blockRawDataBO.setGasLimit(block.getGasLimit()); + } + if (!params.contains(IgnoreBasicDataParam.BlockRawDataParams.GAS_USED.name())) { + blockRawDataBO.setGasUsed(block.getGasUsed()); + } + if (!params.contains(IgnoreBasicDataParam.BlockRawDataParams.LOGS_BLOOM.name())) { + blockRawDataBO.setLogsBloom(block.getLogsBloom()); + } + if (!params.contains(IgnoreBasicDataParam.BlockRawDataParams.PARENT_HASH.name())) { + blockRawDataBO.setParentHash(block.getParentHash()); + } + if (!params.contains(IgnoreBasicDataParam.BlockRawDataParams.RECEIPTS_ROOT.name())) { + blockRawDataBO.setReceiptsRoot(block.getReceiptsRoot()); + } + if (!params.contains(IgnoreBasicDataParam.BlockRawDataParams.SEALER.name())) { + blockRawDataBO.setSealer(block.getSealer()); + } + if (!params.contains(IgnoreBasicDataParam.BlockRawDataParams.SEALER_LIST.name())) { + blockRawDataBO.setSealerList(JacksonUtils.toJson(block.getSealerList())); + } + if (!params.contains(IgnoreBasicDataParam.BlockRawDataParams.SEALER_LIST.name())) { + blockRawDataBO.setSignatureList(JacksonUtils.toJson(block.getSignatureList())); + } + if (!params.contains(IgnoreBasicDataParam.BlockRawDataParams.STATE_ROOT.name())) { + blockRawDataBO.setStateRoot(block.getStateRoot()); + } + if (!params.contains(IgnoreBasicDataParam.BlockRawDataParams.TRANSACTIONS_ROOT.name())) { + blockRawDataBO.setTransactionsRoot(block.getTransactionsRoot()); + } + if (!params.contains(IgnoreBasicDataParam.BlockRawDataParams.TRANSACTION_LIST.name())) { + blockRawDataBO.setTransactionList(JacksonUtils.toJson(block.getTransactions())); + } + } return blockRawDataBO; } } diff --git a/WeBankBlockchain-Data-Export-sdk/WeBankBlockchain-Data-Export-parser/src/main/java/com/webank/blockchain/data/export/parser/handler/EventCrawlerHandler.java b/WeBankBlockchain-Data-Export-sdk/WeBankBlockchain-Data-Export-parser/src/main/java/com/webank/blockchain/data/export/parser/handler/EventCrawlerHandler.java index 5be8a764..5300e954 100644 --- a/WeBankBlockchain-Data-Export-sdk/WeBankBlockchain-Data-Export-parser/src/main/java/com/webank/blockchain/data/export/parser/handler/EventCrawlerHandler.java +++ b/WeBankBlockchain-Data-Export-sdk/WeBankBlockchain-Data-Export-parser/src/main/java/com/webank/blockchain/data/export/parser/handler/EventCrawlerHandler.java @@ -56,10 +56,10 @@ @Slf4j public class EventCrawlerHandler { - @SuppressWarnings({ "rawtypes", "unused" }) public static List crawl(Block block, Map txHashContractNameMapping) throws IOException { List transactionResults = block.getTransactions(); + List boList = new ArrayList<>(); for (TransactionResult result : transactionResults) { TransactionObject to = (TransactionObject) result; JsonTransactionResponse transaction = to.get(); @@ -81,10 +81,10 @@ public static List crawl(Block block, Map txHashContrac if (abi == null) { continue; } - return parserEvent(contractAbiMap, contractName.get(), abi, tr,block); + boList.addAll(parserEvent(contractAbiMap, contractName.get(), abi, tr,block)); } } - return ListUtil.empty(); + return boList; } private static List parserEvent(Map contractAbiMap, String contractName, String abi, diff --git a/WeBankBlockchain-Data-Export-sdk/WeBankBlockchain-Data-Export-parser/src/main/java/com/webank/blockchain/data/export/parser/handler/MethodCrawlerHandler.java b/WeBankBlockchain-Data-Export-sdk/WeBankBlockchain-Data-Export-parser/src/main/java/com/webank/blockchain/data/export/parser/handler/MethodCrawlerHandler.java index f5ef7a3a..0611d975 100644 --- a/WeBankBlockchain-Data-Export-sdk/WeBankBlockchain-Data-Export-parser/src/main/java/com/webank/blockchain/data/export/parser/handler/MethodCrawlerHandler.java +++ b/WeBankBlockchain-Data-Export-sdk/WeBankBlockchain-Data-Export-parser/src/main/java/com/webank/blockchain/data/export/parser/handler/MethodCrawlerHandler.java @@ -27,6 +27,7 @@ import com.webank.blockchain.data.export.common.entity.ExportConfig; import com.webank.blockchain.data.export.common.entity.ExportConstant; import com.webank.blockchain.data.export.common.entity.TableSQL; +import com.webank.blockchain.data.export.common.enums.IgnoreBasicDataParam; import com.webank.blockchain.data.export.common.tools.DateUtils; import com.webank.blockchain.data.export.common.tools.JacksonUtils; import com.webank.blockchain.data.export.common.tools.MethodUtils; @@ -48,6 +49,9 @@ import java.util.Map; import java.util.Optional; +import static com.webank.blockchain.data.export.common.entity.ExportConstant.TX_RAW_DATA_TABLE; +import static com.webank.blockchain.data.export.common.entity.ExportConstant.TX_RECEIPT_RAW_DATA_TABLE; + /** * MethodCrawlerHandler @@ -201,41 +205,112 @@ public static BlockTxDetailInfoBO getBlockTxDetailInfo(Block block, JsonTransact } public static TxRawDataBO getTxRawDataBO(Block block, JsonTransactionResponse transaction, TransactionReceipt receipt) { + ExportConfig config = ExportConstant.getCurrentContext().getConfig(); + Map> ignoreBasicDataTableParam = config.getIgnoreBasicDataTableParam(); + TxRawDataBO txRawDataBO = new TxRawDataBO(); txRawDataBO.setBlockHash(receipt.getBlockHash()) .setBlockHeight(Numeric.decodeQuantity((receipt.getBlockNumber())).longValue()) .setBlockTimeStamp(DateUtils.hexStrToDate(block.getTimestamp())) - .setTxHash(receipt.getTransactionHash()) - .setTxIndex(transaction.getTransactionIndex()) - .setFrom(transaction.getFrom()) - .setGas(transaction.getGas()) - .setGasPrice(transaction.getGasPrice()) - .setInput(transaction.getInput()) - .setNonce(transaction.getNonce()) - .setTo(transaction.getTo()) - .setValue(transaction.getValue()); + .setTxHash(receipt.getTransactionHash()); + if (!ignoreBasicDataTableParam.containsKey(IgnoreBasicDataParam.IgnoreBasicDataTable.TX_RAW_DATA_TABLE.name())) { + txRawDataBO.setTxIndex(transaction.getTransactionIndex()) + .setFrom(transaction.getFrom()) + .setGas(transaction.getGas()) + .setGasPrice(transaction.getGasPrice()) + .setInput(transaction.getInput()) + .setNonce(transaction.getNonce()) + .setTo(transaction.getTo()) + .setValue(transaction.getValue()); + } else { + List params = ignoreBasicDataTableParam.get(IgnoreBasicDataParam.IgnoreBasicDataTable.TX_RAW_DATA_TABLE.name()); + if (!params.contains(IgnoreBasicDataParam.TxRawDataParams.FROM.name())) { + txRawDataBO.setFrom(transaction.getFrom()); + } + if (!params.contains(IgnoreBasicDataParam.TxRawDataParams.GAS.name())) { + txRawDataBO.setGas(transaction.getGas()); + } + if (!params.contains(IgnoreBasicDataParam.TxRawDataParams.GAS_PRICE.name())) { + txRawDataBO.setGasPrice(transaction.getGasPrice()); + } + if (!params.contains(IgnoreBasicDataParam.TxRawDataParams.INPUT.name())) { + txRawDataBO.setInput(transaction.getInput()); + } + if (!params.contains(IgnoreBasicDataParam.TxRawDataParams.NONCE.name())) { + txRawDataBO.setNonce(transaction.getNonce()); + } + if (!params.contains(IgnoreBasicDataParam.TxRawDataParams.TO.name())) { + txRawDataBO.setTo(transaction.getTo()); + } + if (!params.contains(IgnoreBasicDataParam.TxRawDataParams.VALUE.name())) { + txRawDataBO.setValue(transaction.getValue()); + } + } return txRawDataBO; } public static TxReceiptRawDataBO getTxReceiptRawDataBO(Block block, TransactionReceipt receipt, String contractAddress) { + ExportConfig config = ExportConstant.getCurrentContext().getConfig(); + Map> ignoreBasicDataTableParam = config.getIgnoreBasicDataTableParam(); + TxReceiptRawDataBO txReceiptRawDataBO = new TxReceiptRawDataBO(); txReceiptRawDataBO.setBlockHash(receipt.getBlockHash()) .setBlockHeight(Numeric.decodeQuantity((receipt.getBlockNumber())).longValue()) .setBlockTimeStamp(DateUtils.hexStrToDate(block.getTimestamp())) .setTxHash(receipt.getTransactionHash()) - .setContractAddress(contractAddress) - .setFrom(receipt.getFrom()) - .setGasUsed(receipt.getGasUsed()) - .setInput(receipt.getInput()) - .setLogs(receipt.getLogsBloom()) - .setMessage(receipt.getMessage()) - .setOutput(receipt.getOutput()) - .setLogsBloom(JacksonUtils.toJson(receipt.getLogsBloom())) - .setRoot(receipt.getRoot()) - .setTo(receipt.getTo()) - .setTxIndex(receipt.getTransactionIndex()) - .setTxProof(JacksonUtils.toJson(receipt.getTxProof())) - .setReceiptProof(JacksonUtils.toJson(receipt.getReceiptProof())); + .setContractAddress(contractAddress); + if (!ignoreBasicDataTableParam.containsKey(IgnoreBasicDataParam.IgnoreBasicDataTable.TX_RECEIPT_RAW_DATA_TABLE.name())) { + txReceiptRawDataBO.setFrom(receipt.getFrom()) + .setGasUsed(receipt.getGasUsed()) + .setInput(receipt.getInput()) + .setLogs(JacksonUtils.toJson(receipt.getLogs())) + .setMessage(receipt.getMessage()) + .setOutput(receipt.getOutput()) + .setLogsBloom(JacksonUtils.toJson(receipt.getLogsBloom())) + .setRoot(receipt.getRoot()) + .setTo(receipt.getTo()) + .setTxIndex(receipt.getTransactionIndex()) + .setTxProof(JacksonUtils.toJson(receipt.getTxProof())) + .setReceiptProof(JacksonUtils.toJson(receipt.getReceiptProof())); + }else { + List params = ignoreBasicDataTableParam.get(IgnoreBasicDataParam.IgnoreBasicDataTable.TX_RECEIPT_RAW_DATA_TABLE.name()); + if (!params.contains(IgnoreBasicDataParam.TxReceiptRawDataParams.FROM.name())) { + txReceiptRawDataBO.setFrom(receipt.getFrom()); + } + if (!params.contains(IgnoreBasicDataParam.TxReceiptRawDataParams.GAS_USED.name())) { + txReceiptRawDataBO.setGasUsed(receipt.getGasUsed()); + } + if (!params.contains(IgnoreBasicDataParam.TxReceiptRawDataParams.LOGS.name())) { + txReceiptRawDataBO.setLogs(JacksonUtils.toJson(receipt.getLogs())); + } + if (!params.contains(IgnoreBasicDataParam.TxReceiptRawDataParams.INPUT.name())) { + txReceiptRawDataBO.setInput(receipt.getInput()); + } + if (!params.contains(IgnoreBasicDataParam.TxReceiptRawDataParams.MESSAGE.name())) { + txReceiptRawDataBO.setMessage(receipt.getMessage()); + } + if (!params.contains(IgnoreBasicDataParam.TxReceiptRawDataParams.OUTPUT.name())) { + txReceiptRawDataBO.setOutput(receipt.getOutput()); + } + if (!params.contains(IgnoreBasicDataParam.TxReceiptRawDataParams.LOGS_BLOOM.name())) { + txReceiptRawDataBO.setLogsBloom(JacksonUtils.toJson(receipt.getLogsBloom())); + } + if (!params.contains(IgnoreBasicDataParam.TxReceiptRawDataParams.ROOT.name())) { + txReceiptRawDataBO.setMessage(receipt.getRoot()); + } + if (!params.contains(IgnoreBasicDataParam.TxReceiptRawDataParams.TO.name())) { + txReceiptRawDataBO.setTo(receipt.getTo()); + } + if (!params.contains(IgnoreBasicDataParam.TxReceiptRawDataParams.TX_INDEX.name())) { + txReceiptRawDataBO.setTxIndex(receipt.getTransactionIndex()); + } + if (!params.contains(IgnoreBasicDataParam.TxReceiptRawDataParams.TX_PROOF.name())) { + txReceiptRawDataBO.setTxProof(JacksonUtils.toJson(receipt.getTxProof())); + } + if (!params.contains(IgnoreBasicDataParam.TxReceiptRawDataParams.RECEIPT_PROOF.name())) { + txReceiptRawDataBO.setReceiptProof(JacksonUtils.toJson(receipt.getReceiptProof())); + } + } return txReceiptRawDataBO; } diff --git a/WeBankBlockchain-Data-Export-sdk/build.gradle b/WeBankBlockchain-Data-Export-sdk/build.gradle index 919ad31e..292bdc68 100644 --- a/WeBankBlockchain-Data-Export-sdk/build.gradle +++ b/WeBankBlockchain-Data-Export-sdk/build.gradle @@ -9,7 +9,7 @@ apply plugin: 'signing' [compileJava, compileTestJava, javadoc]*.options*.encoding = 'UTF-8' group = 'com.webank' -version = "1.7.3" +version = "1.7.4" allprojects { repositories { diff --git a/WeBankBlockchain-Data-Export-service/src/main/java/com/webank/blockchain/data/export/config/ServiceConfig.java b/WeBankBlockchain-Data-Export-service/src/main/java/com/webank/blockchain/data/export/config/ServiceConfig.java index fdafe90d..d61d3ff5 100644 --- a/WeBankBlockchain-Data-Export-service/src/main/java/com/webank/blockchain/data/export/config/ServiceConfig.java +++ b/WeBankBlockchain-Data-Export-service/src/main/java/com/webank/blockchain/data/export/config/ServiceConfig.java @@ -1,9 +1,9 @@ package com.webank.blockchain.data.export.config; -import cn.hutool.core.io.FileUtil; import com.webank.blockchain.data.export.common.entity.ContractInfo; import com.webank.blockchain.data.export.common.entity.ESDataSource; import com.webank.blockchain.data.export.common.entity.MysqlDataSource; +import com.webank.blockchain.data.export.common.enums.DataType; import com.webank.solc.plugin.compiler.CompileSolToJava; import lombok.Data; import lombok.extern.slf4j.Slf4j; @@ -113,6 +113,10 @@ public class ServiceConfig { */ private Map>> paramSQLType_SDK; + private Map> ignoreBasicDataTableParam; + + private List dataTypeBlackList; + private ESDataSource esDataSource; private boolean grafanaEnable; @@ -155,6 +159,9 @@ private void init() { generatedOff_SDK = PropertiesUtils.getGeneratedOff(); ignoreParam_SDK = PropertiesUtils.getIgnoreParam(); paramSQLType_SDK = PropertiesUtils.getparamSQLType(); + ignoreBasicDataTableParam = PropertiesUtils.getIgnoreBasicDataTableParam(); + dataTypeBlackList = PropertiesUtils.getDataTypeBlackList(); + if (esEnabled) { esDataSource = new ESDataSource(); esDataSource.setClusterName(clusterName); diff --git a/WeBankBlockchain-Data-Export-service/src/main/java/com/webank/blockchain/data/export/service/ExportService.java b/WeBankBlockchain-Data-Export-service/src/main/java/com/webank/blockchain/data/export/service/ExportService.java index 0a54c410..d6ea6c2b 100644 --- a/WeBankBlockchain-Data-Export-service/src/main/java/com/webank/blockchain/data/export/service/ExportService.java +++ b/WeBankBlockchain-Data-Export-service/src/main/java/com/webank/blockchain/data/export/service/ExportService.java @@ -97,6 +97,8 @@ private ExportConfig buildExportConfig(){ config.setGeneratedOff(serviceConfig.getGeneratedOff_SDK()); config.setIgnoreParam(serviceConfig.getIgnoreParam_SDK()); config.setMultiLiving(serviceConfig.isMultiLiving()); + config.setDataTypeBlackList(serviceConfig.getDataTypeBlackList()); + config.setIgnoreBasicDataTableParam(serviceConfig.getIgnoreBasicDataTableParam()); return config; } diff --git a/WeBankBlockchain-Data-Export-service/src/main/java/com/webank/blockchain/data/export/utils/PropertiesUtils.java b/WeBankBlockchain-Data-Export-service/src/main/java/com/webank/blockchain/data/export/utils/PropertiesUtils.java index cb8361cc..89d547da 100644 --- a/WeBankBlockchain-Data-Export-service/src/main/java/com/webank/blockchain/data/export/utils/PropertiesUtils.java +++ b/WeBankBlockchain-Data-Export-service/src/main/java/com/webank/blockchain/data/export/utils/PropertiesUtils.java @@ -16,6 +16,8 @@ import cn.hutool.core.collection.CollectionUtil; import com.webank.blockchain.data.export.common.entity.ContractInfo; import com.webank.blockchain.data.export.common.entity.MysqlDataSource; +import com.webank.blockchain.data.export.common.enums.DataType; +import com.webank.blockchain.data.export.common.enums.IgnoreBasicDataParam; import com.webank.blockchain.data.export.config.ServiceConfig; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -241,5 +243,166 @@ public Map>> getparamSQLType() { return map; } + public List getDataTypeBlackList() { + String dataTypeBlackList = getProperty("system", "dataTypeBlackLists"); + if (dataTypeBlackList == null) { + return null; + } + String[] strings = dataTypeBlackList.split(","); + List dataTypes = new ArrayList<>(); + for (String dataType : strings) { + DataType type = DataType.getDataType(dataType); + if (!type.equals(DataType.NULL)){ + dataTypes.add(type); + } + } + return dataTypes; + } + + + public Map> getIgnoreBasicDataTableParam() { + String ignoreBasicDataTableParams = getProperty("system", "ignoreBasicDataTableParams"); + Map> map = new HashMap<>(); + if (ignoreBasicDataTableParams == null) { + return map; + } + String[] strings = ignoreBasicDataTableParams.split("\\|"); + for (String str : strings) { + String[] attributes = str.split("\\."); + if (attributes.length != 2){ + log.error("Incorrect system.ignoreBasicDataTableParams size ! Wrong field : " + str); + System.exit(1); + } + String tableName = attributes[0]; + String[] paramStrs = attributes[1].split(","); + List params = getParams(tableName,paramStrs); + if (tableName.equals("block_raw_data")){ + map.put(IgnoreBasicDataParam.IgnoreBasicDataTable.BLOCK_RAW_DATA_TABLE.name(),params); + } + if (tableName.equals("tx_raw_data")){ + map.put(IgnoreBasicDataParam.IgnoreBasicDataTable.TX_RAW_DATA_TABLE.name(),params); + } + if (tableName.equals("tx_receipt_raw_data")){ + map.put(IgnoreBasicDataParam.IgnoreBasicDataTable.TX_RECEIPT_RAW_DATA_TABLE.name(),params); + } + } + return map; + } + + + private List getParams(String tableName, String[] paramStrs) { + List params = new ArrayList<>(); + if (tableName.equals("block_raw_data")) { + for (String param : paramStrs) { + if (param.equals("db_hash")) { + params.add(IgnoreBasicDataParam.BlockRawDataParams.DB_HASH.name()); + } + if (param.equals("extra_data")) { + params.add(IgnoreBasicDataParam.BlockRawDataParams.EXTRA_DATA.name()); + } + if (param.equals("gas_limit")) { + params.add(IgnoreBasicDataParam.BlockRawDataParams.GAS_LIMIT.name()); + } + if (param.equals("gas_used")) { + params.add(IgnoreBasicDataParam.BlockRawDataParams.GAS_USED.name()); + } + if (param.equals("logs_bloom")) { + params.add(IgnoreBasicDataParam.BlockRawDataParams.LOGS_BLOOM.name()); + } + if (param.equals("parent_hash")) { + params.add(IgnoreBasicDataParam.BlockRawDataParams.PARENT_HASH.name()); + } + if (param.equals("receipts_root")) { + params.add(IgnoreBasicDataParam.BlockRawDataParams.RECEIPTS_ROOT.name()); + } + if (param.equals("sealer")) { + params.add(IgnoreBasicDataParam.BlockRawDataParams.SEALER.name()); + } + if (param.equals("sealer_list")) { + params.add(IgnoreBasicDataParam.BlockRawDataParams.SEALER_LIST.name()); + } + if (param.equals("signature_list")) { + params.add(IgnoreBasicDataParam.BlockRawDataParams.SIGNATURE_LIST.name()); + } + if (param.equals("state_root")) { + params.add(IgnoreBasicDataParam.BlockRawDataParams.STATE_ROOT.name()); + } + if (param.equals("transaction_list")) { + params.add(IgnoreBasicDataParam.BlockRawDataParams.TRANSACTION_LIST.name()); + } + if (param.equals("transactions_root")) { + params.add(IgnoreBasicDataParam.BlockRawDataParams.TRANSACTIONS_ROOT.name()); + } + if (param.equals("sealer")) { + params.add(IgnoreBasicDataParam.BlockRawDataParams.SEALER.name()); + } + } + } + + if (tableName.equals("tx_raw_data")) { + for (String param : paramStrs) { + if (param.equals("from")) { + params.add(IgnoreBasicDataParam.TxRawDataParams.FROM.name()); + } + if (param.equals("gas")) { + params.add(IgnoreBasicDataParam.TxRawDataParams.GAS.name()); + } + if (param.equals("gas_price")) { + params.add(IgnoreBasicDataParam.TxRawDataParams.GAS_PRICE.name()); + } + if (param.equals("input")) { + params.add(IgnoreBasicDataParam.TxRawDataParams.INPUT.name()); + } + if (param.equals("nonce")) { + params.add(IgnoreBasicDataParam.TxRawDataParams.NONCE.name()); + } + if (param.equals("to")) { + params.add(IgnoreBasicDataParam.TxRawDataParams.TO.name()); + } + if (param.equals("value")) { + params.add(IgnoreBasicDataParam.TxRawDataParams.VALUE.name()); + } + } + } + + if (tableName.equals("tx_receipt_raw_data")) { + for (String param : paramStrs) { + if (param.equals("gas_used")) { + params.add(IgnoreBasicDataParam.TxReceiptRawDataParams.GAS_USED.name()); + } + if (param.equals("input")) { + params.add(IgnoreBasicDataParam.TxReceiptRawDataParams.INPUT.name()); + } + if (param.equals("logs")) { + params.add(IgnoreBasicDataParam.TxReceiptRawDataParams.LOGS.name()); + } + if (param.equals("logs_bloom")) { + params.add(IgnoreBasicDataParam.TxReceiptRawDataParams.LOGS_BLOOM.name()); + } + if (param.equals("message")) { + params.add(IgnoreBasicDataParam.TxReceiptRawDataParams.MESSAGE.name()); + } + if (param.equals("output")) { + params.add(IgnoreBasicDataParam.TxReceiptRawDataParams.OUTPUT.name()); + } + if (param.equals("receipt_proof")) { + params.add(IgnoreBasicDataParam.TxReceiptRawDataParams.RECEIPT_PROOF.name()); + } + if (param.equals("root")) { + params.add(IgnoreBasicDataParam.TxReceiptRawDataParams.ROOT.name()); + } + if (param.equals("tx_proof")) { + params.add(IgnoreBasicDataParam.TxReceiptRawDataParams.TX_PROOF.name()); + } + if (param.equals("from")) { + params.add(IgnoreBasicDataParam.TxReceiptRawDataParams.FROM.name()); + } + if (param.equals("to")) { + params.add(IgnoreBasicDataParam.TxReceiptRawDataParams.TO.name()); + } + } + } + return params; + } } diff --git a/build.gradle b/build.gradle index c37c3894..94c7c679 100644 --- a/build.gradle +++ b/build.gradle @@ -20,7 +20,7 @@ allprojects { } subprojects{ - version = "1.7.3" + version = "1.7.4" group = 'com.webank' } diff --git a/tools/config/application.properties b/tools/config/application.properties index 4e059c25..e2ee2ff9 100644 --- a/tools/config/application.properties +++ b/tools/config/application.properties @@ -73,6 +73,10 @@ system.solcVersion=0.4.25.1 ### With '|' between multiple options; With ',' between multiple javaNameParamName; ### ex: HelloWorld.set.nameA,nameB #system.ignoreParam=[contractName.methodName or eventName.paramName,paramName]|[contractName.methodName or eventName.paramName,paramName] +### ex: block_detail_info,block_raw_data,block_tx_detail_info,tx_raw_data,tx_receipt_raw_data,deployed_account_info,contract_info +#system.dataTypeBlackList= +### ex: tx_raw_data.from,to|block_raw_data.db_hash,gas_limit +#system.ignoreBasicDataTableParams= ### Specifies the parameter database type in a method or event,With '|' between multiple options; ### ex: HelloWorld.set.name.text #system.paramSQLType=[contractName.methodName or eventName.paramName.sqlType]|......