From 5bf73a7e11a1546dcdd3442061591e0371770b2e Mon Sep 17 00:00:00 2001 From: ssd04 Date: Tue, 2 Jul 2024 11:41:54 +0300 Subject: [PATCH 01/55] accounts adapter api for interceptors --- factory/processing/processComponents.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/factory/processing/processComponents.go b/factory/processing/processComponents.go index 72d75c69dc3..00f4391d163 100644 --- a/factory/processing/processComponents.go +++ b/factory/processing/processComponents.go @@ -1665,7 +1665,7 @@ func (pcf *processComponentsFactory) newShardInterceptorContainerFactory( shardInterceptorsContainerFactoryArgs := interceptorscontainer.CommonInterceptorsContainerFactoryArgs{ CoreComponents: pcf.coreData, CryptoComponents: pcf.crypto, - Accounts: pcf.state.AccountsAdapter(), + Accounts: pcf.state.AccountsAdapterAPI(), ShardCoordinator: pcf.bootstrapComponents.ShardCoordinator(), NodesCoordinator: pcf.nodesCoordinator, MainMessenger: pcf.network.NetworkMessenger(), @@ -1724,7 +1724,7 @@ func (pcf *processComponentsFactory) newMetaInterceptorContainerFactory( FullArchiveMessenger: pcf.network.FullArchiveNetworkMessenger(), Store: pcf.data.StorageService(), DataPool: pcf.data.Datapool(), - Accounts: pcf.state.AccountsAdapter(), + Accounts: pcf.state.AccountsAdapterAPI(), MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, TxFeeHandler: pcf.coreData.EconomicsData(), BlockBlackList: headerBlackList, From 56f4020819daae17e7d0ac24dc52f483ab81c4e9 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Thu, 4 Jul 2024 12:47:51 +0300 Subject: [PATCH 02/55] added dump to json file --- .../disabled/disabledAccountsAdapter.go | 3 +- process/transaction/metaProcess.go | 5 +- process/transaction/shardProcess.go | 5 +- .../simulationAccountsDB.go | 5 +- state/accountsDB.go | 44 ++++++++++-- state/accountsDBApi.go | 5 +- state/accountsDBApiWithHistory.go | 3 +- .../disabled/disabledStateChangesCollector.go | 8 ++- state/interface.go | 6 +- state/stateChangesCollector.go | 67 ++++++++++++++++--- 10 files changed, 126 insertions(+), 25 deletions(-) diff --git a/epochStart/bootstrap/disabled/disabledAccountsAdapter.go b/epochStart/bootstrap/disabled/disabledAccountsAdapter.go index f2ac368babf..1310d59ce6e 100644 --- a/epochStart/bootstrap/disabled/disabledAccountsAdapter.go +++ b/epochStart/bootstrap/disabled/disabledAccountsAdapter.go @@ -3,6 +3,7 @@ package disabled import ( "context" + "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/state" vmcommon "github.com/multiversx/mx-chain-vm-common-go" @@ -134,7 +135,7 @@ func (a *accountsAdapter) GetStackDebugFirstEntry() []byte { } // SetTxHashForLatestStateChanges - -func (a *accountsAdapter) SetTxHashForLatestStateChanges(_ []byte) { +func (a *accountsAdapter) SetTxHashForLatestStateChanges(_ []byte, _ *transaction.Transaction) { } // ResetStateChangesCollector - diff --git a/process/transaction/metaProcess.go b/process/transaction/metaProcess.go index c9a2b868dab..a84bf305a64 100644 --- a/process/transaction/metaProcess.go +++ b/process/transaction/metaProcess.go @@ -123,7 +123,10 @@ func (txProc *metaTxProcessor) ProcessTransaction(tx *transaction.Transaction) ( txProc.pubkeyConv, ) - defer txProc.accounts.SetTxHashForLatestStateChanges(txHash) + defer func() { + txProc.accounts.SetTxHashForLatestStateChanges(txHash, tx) + log.Debug("SetTxHashForLatestStateChanges", "txHash", txHash) + }() err = txProc.checkTxValues(tx, acntSnd, acntDst, false) if err != nil { diff --git a/process/transaction/shardProcess.go b/process/transaction/shardProcess.go index 43c5b4878bf..c52044287cc 100644 --- a/process/transaction/shardProcess.go +++ b/process/transaction/shardProcess.go @@ -195,7 +195,10 @@ func (txProc *txProcessor) ProcessTransaction(tx *transaction.Transaction) (vmco txProc.pubkeyConv, ) - defer txProc.accounts.SetTxHashForLatestStateChanges(txHash) + defer func() { + txProc.accounts.SetTxHashForLatestStateChanges(txHash, tx) + log.Debug("SetTxHashForLatestStateChanges", "txHash", txHash) + }() txType, dstShardTxType := txProc.txTypeHandler.ComputeTransactionType(tx) err = txProc.checkTxValues(tx, acntSnd, acntDst, false) diff --git a/process/transactionEvaluator/simulationAccountsDB.go b/process/transactionEvaluator/simulationAccountsDB.go index 72299570455..21c045a2841 100644 --- a/process/transactionEvaluator/simulationAccountsDB.go +++ b/process/transactionEvaluator/simulationAccountsDB.go @@ -5,6 +5,7 @@ import ( "sync" "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/state" vmcommon "github.com/multiversx/mx-chain-vm-common-go" @@ -173,8 +174,8 @@ func (r *simulationAccountsDB) GetStackDebugFirstEntry() []byte { } // SetTxHashForLatestStateChanges - -func (r *simulationAccountsDB) SetTxHashForLatestStateChanges(txHash []byte) { - r.originalAccounts.SetTxHashForLatestStateChanges(txHash) +func (r *simulationAccountsDB) SetTxHashForLatestStateChanges(txHash []byte, tx *transaction.Transaction) { + r.originalAccounts.SetTxHashForLatestStateChanges(txHash, tx) } // ResetStateChangesCollector - diff --git a/state/accountsDB.go b/state/accountsDB.go index ce450bdcb95..0697cadaba9 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -13,6 +13,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" @@ -207,6 +208,13 @@ func (adb *AccountsDB) GetCode(codeHash []byte) []byte { return nil } + stateChange := StateChangeDTO{ + MainTrieKey: codeHash, + MainTrieVal: val, + DataTrieChanges: nil, + } + adb.stateChangesCollector.AddStateChange(stateChange) + err = adb.marshaller.Unmarshal(&codeEntry, val) if err != nil { return nil @@ -357,6 +365,13 @@ func (adb *AccountsDB) updateOldCodeEntry(oldCodeHash []byte) (*CodeEntry, error return nil, nil } + stateChange := StateChangeDTO{ + MainTrieKey: oldCodeHash, + MainTrieVal: nil, + DataTrieChanges: nil, + } + adb.stateChangesCollector.AddStateChange(stateChange) + unmodifiedOldCodeEntry := &CodeEntry{ Code: oldCodeEntry.Code, NumReferences: oldCodeEntry.NumReferences, @@ -384,7 +399,7 @@ func (adb *AccountsDB) updateOldCodeEntry(oldCodeHash []byte) (*CodeEntry, error return nil, err } - stateChange := StateChangeDTO{ + stateChange = StateChangeDTO{ MainTrieKey: oldCodeHash, MainTrieVal: codeEntryBytes, DataTrieChanges: nil, @@ -862,6 +877,19 @@ func (adb *AccountsDB) Commit() ([]byte, error) { return adb.commit() } +func printStateChanges(stateChanges []StateChangesForTx) { + for _, stateChange := range stateChanges { + if stateChange.TxHash != nil { + fmt.Println(hex.EncodeToString(stateChange.TxHash)) + } + + for _, st := range stateChange.StateChanges { + fmt.Println(string(st.MainTrieKey)) + fmt.Println(hex.EncodeToString(st.MainTrieVal)) + } + } +} + func (adb *AccountsDB) commit() ([]byte, error) { log.Trace("accountsDB.Commit started") adb.entries = make([]JournalEntry, 0) @@ -869,7 +897,13 @@ func (adb *AccountsDB) commit() ([]byte, error) { stateChanges := adb.stateChangesCollector.GetStateChanges() if len(stateChanges) != 0 { log.Warn("state changes collector is not empty", "state changes", stateChanges) - adb.stateChangesCollector.Reset() + // adb.stateChangesCollector.Reset() + } + + printStateChanges(stateChanges) + err := adb.stateChangesCollector.DumpToJSONFile() + if err != nil { + log.Warn("failed to dump state changes to json file", "error", err) } oldHashes := make(common.ModifiedHashes) @@ -887,7 +921,7 @@ func (adb *AccountsDB) commit() ([]byte, error) { oldRoot := adb.mainTrie.GetOldRoot() // Step 2. commit main trie - err := adb.commitTrie(adb.mainTrie, oldHashes, newHashes) + err = adb.commitTrie(adb.mainTrie, oldHashes, newHashes) if err != nil { return nil, err } @@ -1250,8 +1284,8 @@ func collectStats( } // SetTxHashForLatestStateChanges will return the state changes since the last call of this method -func (adb *AccountsDB) SetTxHashForLatestStateChanges(txHash []byte) { - adb.stateChangesCollector.AddTxHashToCollectedStateChanges(txHash) +func (adb *AccountsDB) SetTxHashForLatestStateChanges(txHash []byte, tx *transaction.Transaction) { + adb.stateChangesCollector.AddTxHashToCollectedStateChanges(txHash, tx) } func (adb *AccountsDB) ResetStateChangesCollector() []StateChangesForTx { diff --git a/state/accountsDBApi.go b/state/accountsDBApi.go index 2d07d99e818..207504384aa 100644 --- a/state/accountsDBApi.go +++ b/state/accountsDBApi.go @@ -6,6 +6,7 @@ import ( "sync" "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/holders" vmcommon "github.com/multiversx/mx-chain-vm-common-go" @@ -238,8 +239,8 @@ func (accountsDB *accountsDBApi) GetStackDebugFirstEntry() []byte { } // SetTxHashForLatestStateChanges will call the inner accountsAdapter method -func (accountsDB *accountsDBApi) SetTxHashForLatestStateChanges(txHash []byte) { - accountsDB.innerAccountsAdapter.SetTxHashForLatestStateChanges(txHash) +func (accountsDB *accountsDBApi) SetTxHashForLatestStateChanges(txHash []byte, tx *transaction.Transaction) { + accountsDB.innerAccountsAdapter.SetTxHashForLatestStateChanges(txHash, tx) } // ResetStateChangesCollector returns nil diff --git a/state/accountsDBApiWithHistory.go b/state/accountsDBApiWithHistory.go index 0377a3b07bb..82a19887f3b 100644 --- a/state/accountsDBApiWithHistory.go +++ b/state/accountsDBApiWithHistory.go @@ -6,6 +6,7 @@ import ( "sync" "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/holders" vmcommon "github.com/multiversx/mx-chain-vm-common-go" @@ -141,7 +142,7 @@ func (accountsDB *accountsDBApiWithHistory) GetStackDebugFirstEntry() []byte { } // SetTxHashForLatestStateChanges returns nil -func (accountsDB *accountsDBApiWithHistory) SetTxHashForLatestStateChanges(_ []byte) { +func (accountsDB *accountsDBApiWithHistory) SetTxHashForLatestStateChanges(_ []byte, _ *transaction.Transaction) { } // ResetStateChangesCollector returns nil diff --git a/state/disabled/disabledStateChangesCollector.go b/state/disabled/disabledStateChangesCollector.go index a4f5c15c069..33749946560 100644 --- a/state/disabled/disabledStateChangesCollector.go +++ b/state/disabled/disabledStateChangesCollector.go @@ -1,6 +1,7 @@ package disabled import ( + "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-go/state" ) @@ -27,7 +28,12 @@ func (d *disabledStateChangesCollector) Reset() { } // AddTxHashToCollectedStateChanges does nothing -func (d *disabledStateChangesCollector) AddTxHashToCollectedStateChanges(_ []byte) { +func (d *disabledStateChangesCollector) AddTxHashToCollectedStateChanges(_ []byte, _ *transaction.Transaction) { +} + +// DumpToJSONFile returns nil +func (d *disabledStateChangesCollector) DumpToJSONFile() error { + return nil } // IsInterfaceNil returns true if there is no value under the interface diff --git a/state/interface.go b/state/interface.go index 018590feb42..e386d62c791 100644 --- a/state/interface.go +++ b/state/interface.go @@ -6,6 +6,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data/api" + "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-go/common" vmcommon "github.com/multiversx/mx-chain-vm-common-go" ) @@ -90,7 +91,7 @@ type AccountsAdapter interface { GetStackDebugFirstEntry() []byte SetSyncer(syncer AccountsDBSyncer) error StartSnapshotIfNeeded() error - SetTxHashForLatestStateChanges(txHash []byte) + SetTxHashForLatestStateChanges(txHash []byte, tx *transaction.Transaction) ResetStateChangesCollector() []StateChangesForTx Close() error IsInterfaceNil() bool @@ -358,6 +359,7 @@ type StateChangesCollector interface { AddStateChange(stateChange StateChangeDTO) GetStateChanges() []StateChangesForTx Reset() - AddTxHashToCollectedStateChanges(txHash []byte) + AddTxHashToCollectedStateChanges(txHash []byte, tx *transaction.Transaction) + DumpToJSONFile() error IsInterfaceNil() bool } diff --git a/state/stateChangesCollector.go b/state/stateChangesCollector.go index 9e3f8fabf64..24fa33c1499 100644 --- a/state/stateChangesCollector.go +++ b/state/stateChangesCollector.go @@ -1,22 +1,38 @@ package state +import ( + "encoding/json" + "io/ioutil" + "os" + "path/filepath" + + "github.com/multiversx/mx-chain-core-go/core" + "github.com/multiversx/mx-chain-core-go/data/transaction" +) + +const ( + workingDir = "." + defaultStateChangesPath = "stateChanges" +) + // DataTrieChange represents a change in the data trie type DataTrieChange struct { - Key []byte - Val []byte + Key []byte `json:"key"` + Val []byte `json:"-"` } // StateChangeDTO is used to collect state changes type StateChangeDTO struct { - MainTrieKey []byte - MainTrieVal []byte - DataTrieChanges []DataTrieChange + MainTrieKey []byte `json:"mainTrieKey"` + MainTrieVal []byte `json:"-"` + DataTrieChanges []DataTrieChange `json:"dataTrieChanges"` } // StateChangesForTx is used to collect state changes for a transaction hash type StateChangesForTx struct { - TxHash []byte - StateChanges []StateChangeDTO + TxHash []byte `json:"txHash"` + Tx *transaction.Transaction `json:"tx"` + StateChanges []StateChangeDTO `json:"stateChanges"` } type stateChangesCollector struct { @@ -39,7 +55,7 @@ func (scc *stateChangesCollector) AddStateChange(stateChange StateChangeDTO) { // GetStateChanges returns the accumulated state changes func (scc *stateChangesCollector) GetStateChanges() []StateChangesForTx { if len(scc.stateChanges) > 0 { - scc.AddTxHashToCollectedStateChanges([]byte{}) + scc.AddTxHashToCollectedStateChanges([]byte{}, nil) } return scc.stateChangesForTx @@ -51,9 +67,10 @@ func (scc *stateChangesCollector) Reset() { scc.stateChangesForTx = make([]StateChangesForTx, 0) } -func (scc *stateChangesCollector) AddTxHashToCollectedStateChanges(txHash []byte) { +func (scc *stateChangesCollector) AddTxHashToCollectedStateChanges(txHash []byte, tx *transaction.Transaction) { stateChangesForTx := StateChangesForTx{ TxHash: txHash, + Tx: tx, StateChanges: scc.stateChanges, } @@ -61,6 +78,38 @@ func (scc *stateChangesCollector) AddTxHashToCollectedStateChanges(txHash []byte scc.stateChangesForTx = append(scc.stateChangesForTx, stateChangesForTx) } +func (scc *stateChangesCollector) DumpToJSONFile() error { + directory := filepath.Join(workingDir, defaultStateChangesPath) + args := core.ArgCreateFileArgument{ + Directory: directory, + Prefix: "stateChanges", + FileExtension: "json", + } + jsonFile, err := core.CreateFile(args) + if err != nil { + return err + } + + marshalledData, err := json.Marshal(scc.stateChangesForTx) + if err != nil { + return err + } + + // encoder := json.NewEncoder(jsonFile) + + // err = encoder.Encode(marshalledData) + // if err != nil { + // return err + // } + + err = ioutil.WriteFile(jsonFile.Name(), marshalledData, os.ModePerm) + if err != nil { + return err + } + + return nil +} + // IsInterfaceNil returns true if there is no value under the interface func (scc *stateChangesCollector) IsInterfaceNil() bool { return scc == nil From a7efc5654b5d72d798482f0794501b3ab4d74883 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Thu, 4 Jul 2024 15:11:34 +0300 Subject: [PATCH 03/55] added reset and collect state in accounts db --- state/accountsDB.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/state/accountsDB.go b/state/accountsDB.go index 0697cadaba9..a18afcede53 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -894,18 +894,18 @@ func (adb *AccountsDB) commit() ([]byte, error) { log.Trace("accountsDB.Commit started") adb.entries = make([]JournalEntry, 0) - stateChanges := adb.stateChangesCollector.GetStateChanges() - if len(stateChanges) != 0 { - log.Warn("state changes collector is not empty", "state changes", stateChanges) - // adb.stateChangesCollector.Reset() - } - + stateChanges := adb.ResetStateChangesCollector() printStateChanges(stateChanges) err := adb.stateChangesCollector.DumpToJSONFile() if err != nil { log.Warn("failed to dump state changes to json file", "error", err) } + if len(stateChanges) != 0 { + log.Warn("state changes collector is not empty", "state changes", stateChanges) + adb.stateChangesCollector.Reset() + } + oldHashes := make(common.ModifiedHashes) newHashes := make(common.ModifiedHashes) // Step 1. commit all data tries From 3ae6652bbe48ab2bfa011d548db26c2f9e89a7a0 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Fri, 5 Jul 2024 15:54:14 +0300 Subject: [PATCH 04/55] read state changes in trackable data trie --- factory/api/apiResolverFactory.go | 7 ++- factory/interface.go | 1 + factory/state/stateComponents.go | 56 ++++++++++--------- factory/state/stateComponentsHandler.go | 11 ++++ genesis/process/genesisBlockCreator.go | 8 ++- .../components/stateComponents.go | 5 ++ process/block/baseProcess.go | 8 ++- process/smartContract/hooks/blockChainHook.go | 1 + state/accountsDB.go | 23 ++++++-- state/factory/accountCreator.go | 23 ++++---- state/stateChangesCollector.go | 6 +- state/trackableDataTrie/trackableDataTrie.go | 52 +++++++++++------ update/genesis/import.go | 7 ++- 13 files changed, 137 insertions(+), 71 deletions(-) diff --git a/factory/api/apiResolverFactory.go b/factory/api/apiResolverFactory.go index 852cfd9ce09..b7f97c406ac 100644 --- a/factory/api/apiResolverFactory.go +++ b/factory/api/apiResolverFactory.go @@ -545,9 +545,10 @@ func createShardVmContainerFactory(args scQueryElementArgs, argsHook hooks.ArgBl func createNewAccountsAdapterApi(args scQueryElementArgs, chainHandler data.ChainHandler) (state.AccountsAdapterAPI, common.StorageManager, error) { argsAccCreator := factoryState.ArgsAccountCreator{ - Hasher: args.coreComponents.Hasher(), - Marshaller: args.coreComponents.InternalMarshalizer(), - EnableEpochsHandler: args.coreComponents.EnableEpochsHandler(), + Hasher: args.coreComponents.Hasher(), + Marshaller: args.coreComponents.InternalMarshalizer(), + EnableEpochsHandler: args.coreComponents.EnableEpochsHandler(), + StateChangesCollector: args.stateComponents.StateChangesCollector(), } accountFactory, err := factoryState.NewAccountCreator(argsAccCreator) if err != nil { diff --git a/factory/interface.go b/factory/interface.go index ede9f39089b..6035b62e161 100644 --- a/factory/interface.go +++ b/factory/interface.go @@ -334,6 +334,7 @@ type StateComponentsHolder interface { TriesContainer() common.TriesHolder TrieStorageManagers() map[string]common.StorageManager MissingTrieNodesNotifier() common.MissingTrieNodesNotifier + StateChangesCollector() state.StateChangesCollector Close() error IsInterfaceNil() bool } diff --git a/factory/state/stateComponents.go b/factory/state/stateComponents.go index 33ca9488dd2..1a1bdc41a6f 100644 --- a/factory/state/stateComponents.go +++ b/factory/state/stateComponents.go @@ -2,6 +2,7 @@ package state import ( "fmt" + "github.com/multiversx/mx-chain-core-go/core/check" chainData "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-go/common" @@ -53,6 +54,7 @@ type stateComponents struct { triesContainer common.TriesHolder trieStorageManagers map[string]common.StorageManager missingTrieNodesNotifier common.MissingTrieNodesNotifier + stateChangesCollector state.StateChangesCollector } // NewStateComponentsFactory will return a new instance of stateComponentsFactory @@ -90,7 +92,12 @@ func (scf *stateComponentsFactory) Create() (*stateComponents, error) { return nil, err } - accountsAdapter, accountsAdapterAPI, accountsRepository, err := scf.createAccountsAdapters(triesContainer) + stateChangesCollector := disabled.NewDisabledStateChangesCollector() + if scf.config.StateTriesConfig.CollectStateChangesEnabled { + stateChangesCollector = state.NewStateChangesCollector() + } + + accountsAdapter, accountsAdapterAPI, accountsRepository, err := scf.createAccountsAdapters(triesContainer, stateChangesCollector) if err != nil { return nil, err } @@ -108,6 +115,7 @@ func (scf *stateComponentsFactory) Create() (*stateComponents, error) { triesContainer: triesContainer, trieStorageManagers: trieStorageManagers, missingTrieNodesNotifier: syncer.NewMissingTrieNodesNotifier(), + stateChangesCollector: stateChangesCollector, }, nil } @@ -135,11 +143,12 @@ func (scf *stateComponentsFactory) createSnapshotManager( return state.NewSnapshotsManager(argsSnapshotsManager) } -func (scf *stateComponentsFactory) createAccountsAdapters(triesContainer common.TriesHolder) (state.AccountsAdapter, state.AccountsAdapter, state.AccountsRepository, error) { +func (scf *stateComponentsFactory) createAccountsAdapters(triesContainer common.TriesHolder, stateChangesCollector state.StateChangesCollector) (state.AccountsAdapter, state.AccountsAdapter, state.AccountsRepository, error) { argsAccCreator := factoryState.ArgsAccountCreator{ - Hasher: scf.core.Hasher(), - Marshaller: scf.core.InternalMarshalizer(), - EnableEpochsHandler: scf.core.EnableEpochsHandler(), + Hasher: scf.core.Hasher(), + Marshaller: scf.core.InternalMarshalizer(), + EnableEpochsHandler: scf.core.EnableEpochsHandler(), + StateChangesCollector: stateChangesCollector, } accountFactory, err := factoryState.NewAccountCreator(argsAccCreator) if err != nil { @@ -152,11 +161,6 @@ func (scf *stateComponentsFactory) createAccountsAdapters(triesContainer common. return nil, nil, nil, err } - stateChangesCollector := disabled.NewDisabledStateChangesCollector() - if scf.config.StateTriesConfig.CollectStateChangesEnabled { - stateChangesCollector = state.NewStateChangesCollector() - } - argStateMetrics := stateMetrics.ArgsStateMetrics{ SnapshotInProgressKey: common.MetricAccountsSnapshotInProgress, LastSnapshotDurationKey: common.MetricLastAccountsSnapshotDurationSec, @@ -173,14 +177,14 @@ func (scf *stateComponentsFactory) createAccountsAdapters(triesContainer common. } argsProcessingAccountsDB := state.ArgsAccountsDB{ - Trie: merkleTrie, - Hasher: scf.core.Hasher(), - Marshaller: scf.core.InternalMarshalizer(), - AccountFactory: accountFactory, - StoragePruningManager: storagePruning, - AddressConverter: scf.core.AddressPubKeyConverter(), - SnapshotsManager: snapshotsManager, - StateChangesCollector: stateChangesCollector, + Trie: merkleTrie, + Hasher: scf.core.Hasher(), + Marshaller: scf.core.InternalMarshalizer(), + AccountFactory: accountFactory, + StoragePruningManager: storagePruning, + AddressConverter: scf.core.AddressPubKeyConverter(), + SnapshotsManager: snapshotsManager, + StateChangesCollector: stateChangesCollector, } accountsAdapter, err := state.NewAccountsDB(argsProcessingAccountsDB) if err != nil { @@ -256,14 +260,14 @@ func (scf *stateComponentsFactory) createPeerAdapter(triesContainer common.Tries } argsProcessingPeerAccountsDB := state.ArgsAccountsDB{ - Trie: merkleTrie, - Hasher: scf.core.Hasher(), - Marshaller: scf.core.InternalMarshalizer(), - AccountFactory: accountFactory, - StoragePruningManager: storagePruning, - AddressConverter: scf.core.AddressPubKeyConverter(), - SnapshotsManager: snapshotManager, - StateChangesCollector: stateChangesCollector, + Trie: merkleTrie, + Hasher: scf.core.Hasher(), + Marshaller: scf.core.InternalMarshalizer(), + AccountFactory: accountFactory, + StoragePruningManager: storagePruning, + AddressConverter: scf.core.AddressPubKeyConverter(), + SnapshotsManager: snapshotManager, + StateChangesCollector: stateChangesCollector, } peerAdapter, err := state.NewPeerAccountsDB(argsProcessingPeerAccountsDB) if err != nil { diff --git a/factory/state/stateComponentsHandler.go b/factory/state/stateComponentsHandler.go index 78271a28ffe..b9ba0250d7d 100644 --- a/factory/state/stateComponentsHandler.go +++ b/factory/state/stateComponentsHandler.go @@ -214,6 +214,17 @@ func (msc *managedStateComponents) MissingTrieNodesNotifier() common.MissingTrie return msc.stateComponents.missingTrieNodesNotifier } +func (msc *managedStateComponents) StateChangesCollector() state.StateChangesCollector { + msc.mutStateComponents.RLock() + defer msc.mutStateComponents.RUnlock() + + if msc.stateComponents == nil { + return nil + } + + return msc.stateComponents.stateChangesCollector +} + // IsInterfaceNil returns true if the interface is nil func (msc *managedStateComponents) IsInterfaceNil() bool { return msc == nil diff --git a/genesis/process/genesisBlockCreator.go b/genesis/process/genesisBlockCreator.go index 7c37922ae28..e03af6a10e3 100644 --- a/genesis/process/genesisBlockCreator.go +++ b/genesis/process/genesisBlockCreator.go @@ -24,6 +24,7 @@ import ( "github.com/multiversx/mx-chain-go/process/smartContract/hooks" "github.com/multiversx/mx-chain-go/process/smartContract/hooks/counters" "github.com/multiversx/mx-chain-go/sharding" + disabledState "github.com/multiversx/mx-chain-go/state/disabled" factoryState "github.com/multiversx/mx-chain-go/state/factory" "github.com/multiversx/mx-chain-go/state/syncer" "github.com/multiversx/mx-chain-go/statusHandler" @@ -494,9 +495,10 @@ func (gbc *genesisBlockCreator) getNewArgForShard(shardID uint32) (ArgsGenesisBl } argsAccCreator := factoryState.ArgsAccountCreator{ - Hasher: newArgument.Core.Hasher(), - Marshaller: newArgument.Core.InternalMarshalizer(), - EnableEpochsHandler: newArgument.Core.EnableEpochsHandler(), + Hasher: newArgument.Core.Hasher(), + Marshaller: newArgument.Core.InternalMarshalizer(), + EnableEpochsHandler: newArgument.Core.EnableEpochsHandler(), + StateChangesCollector: disabledState.NewDisabledStateChangesCollector(), } accCreator, err := factoryState.NewAccountCreator(argsAccCreator) if err != nil { diff --git a/node/chainSimulator/components/stateComponents.go b/node/chainSimulator/components/stateComponents.go index b3fddf55f40..dddac1d31a1 100644 --- a/node/chainSimulator/components/stateComponents.go +++ b/node/chainSimulator/components/stateComponents.go @@ -10,6 +10,7 @@ import ( "github.com/multiversx/mx-chain-go/factory" factoryState "github.com/multiversx/mx-chain-go/factory/state" "github.com/multiversx/mx-chain-go/state" + "github.com/multiversx/mx-chain-go/state/disabled" ) // ArgsStateComponents will hold the components needed for state components @@ -109,6 +110,10 @@ func (s *stateComponentsHolder) MissingTrieNodesNotifier() common.MissingTrieNod return s.missingTrieNodesNotifier } +func (s *stateComponentsHolder) StateChangesCollector() state.StateChangesCollector { + return disabled.NewDisabledStateChangesCollector() +} + // Close will close the state components func (s *stateComponentsHolder) Close() error { return s.stateComponentsCloser.Close() diff --git a/process/block/baseProcess.go b/process/block/baseProcess.go index b12aa6b2783..9e1cf4f02f9 100644 --- a/process/block/baseProcess.go +++ b/process/block/baseProcess.go @@ -39,6 +39,7 @@ import ( "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" "github.com/multiversx/mx-chain-go/state" + "github.com/multiversx/mx-chain-go/state/disabled" "github.com/multiversx/mx-chain-go/state/factory" "github.com/multiversx/mx-chain-go/state/parsers" "github.com/multiversx/mx-chain-go/storage/storageunit" @@ -1772,9 +1773,10 @@ func (bp *baseProcessor) commitTrieEpochRootHashIfNeeded(metaBlock *block.MetaBl totalSizeCodeLeaves := 0 argsAccCreator := factory.ArgsAccountCreator{ - Hasher: bp.hasher, - Marshaller: bp.marshalizer, - EnableEpochsHandler: bp.enableEpochsHandler, + Hasher: bp.hasher, + Marshaller: bp.marshalizer, + EnableEpochsHandler: bp.enableEpochsHandler, + StateChangesCollector: disabled.NewDisabledStateChangesCollector(), } accountCreator, err := factory.NewAccountCreator(argsAccCreator) if err != nil { diff --git a/process/smartContract/hooks/blockChainHook.go b/process/smartContract/hooks/blockChainHook.go index 827d08da435..c1a389f2ed8 100644 --- a/process/smartContract/hooks/blockChainHook.go +++ b/process/smartContract/hooks/blockChainHook.go @@ -273,6 +273,7 @@ func (bh *BlockChainHookImpl) GetStorageData(accountAddress []byte, index []byte } value, trieDepth, err := userAcc.AccountDataHandler().RetrieveValue(index) + messages := []interface{}{ "address", accountAddress, "rootHash", userAcc.GetRootHash(), diff --git a/state/accountsDB.go b/state/accountsDB.go index a18afcede53..910dc30290a 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -209,6 +209,7 @@ func (adb *AccountsDB) GetCode(codeHash []byte) []byte { } stateChange := StateChangeDTO{ + Type: "read", MainTrieKey: codeHash, MainTrieVal: val, DataTrieChanges: nil, @@ -282,6 +283,7 @@ func (adb *AccountsDB) SaveAccount(account vmcommon.AccountHandler) error { } stateChange := StateChangeDTO{ + Type: "write", MainTrieKey: account.AddressBytes(), MainTrieVal: marshalledAccount, DataTrieChanges: newDataTrieValues, @@ -366,6 +368,7 @@ func (adb *AccountsDB) updateOldCodeEntry(oldCodeHash []byte) (*CodeEntry, error } stateChange := StateChangeDTO{ + Type: "read", MainTrieKey: oldCodeHash, MainTrieVal: nil, DataTrieChanges: nil, @@ -384,6 +387,7 @@ func (adb *AccountsDB) updateOldCodeEntry(oldCodeHash []byte) (*CodeEntry, error } stateChange := StateChangeDTO{ + Type: "write", MainTrieKey: oldCodeHash, MainTrieVal: nil, DataTrieChanges: nil, @@ -400,6 +404,7 @@ func (adb *AccountsDB) updateOldCodeEntry(oldCodeHash []byte) (*CodeEntry, error } stateChange = StateChangeDTO{ + Type: "write", MainTrieKey: oldCodeHash, MainTrieVal: codeEntryBytes, DataTrieChanges: nil, @@ -432,6 +437,7 @@ func (adb *AccountsDB) updateNewCodeEntry(newCodeHash []byte, newCode []byte) er } stateChange := StateChangeDTO{ + Type: "write", MainTrieKey: newCodeHash, MainTrieVal: codeEntryBytes, DataTrieChanges: nil, @@ -894,17 +900,23 @@ func (adb *AccountsDB) commit() ([]byte, error) { log.Trace("accountsDB.Commit started") adb.entries = make([]JournalEntry, 0) - stateChanges := adb.ResetStateChangesCollector() + stateChanges := adb.stateChangesCollector.GetStateChanges() printStateChanges(stateChanges) err := adb.stateChangesCollector.DumpToJSONFile() if err != nil { log.Warn("failed to dump state changes to json file", "error", err) } - if len(stateChanges) != 0 { - log.Warn("state changes collector is not empty", "state changes", stateChanges) - adb.stateChangesCollector.Reset() - } + log.Debug("commit: dump to json file") + + adb.stateChangesCollector.Reset() + + log.Debug("commit: reset") + + // if len(stateChanges) != 0 { + // log.Warn("state changes collector is not empty", "state changes", stateChanges) + // adb.stateChangesCollector.Reset() + // } oldHashes := make(common.ModifiedHashes) newHashes := make(common.ModifiedHashes) @@ -1286,6 +1298,7 @@ func collectStats( // SetTxHashForLatestStateChanges will return the state changes since the last call of this method func (adb *AccountsDB) SetTxHashForLatestStateChanges(txHash []byte, tx *transaction.Transaction) { adb.stateChangesCollector.AddTxHashToCollectedStateChanges(txHash, tx) + } func (adb *AccountsDB) ResetStateChangesCollector() []StateChangesForTx { diff --git a/state/factory/accountCreator.go b/state/factory/accountCreator.go index c117a3dfe25..b8d97415b76 100644 --- a/state/factory/accountCreator.go +++ b/state/factory/accountCreator.go @@ -15,16 +15,18 @@ import ( // ArgsAccountCreator holds the arguments needed to create a new account creator type ArgsAccountCreator struct { - Hasher hashing.Hasher - Marshaller marshal.Marshalizer - EnableEpochsHandler common.EnableEpochsHandler + Hasher hashing.Hasher + Marshaller marshal.Marshalizer + EnableEpochsHandler common.EnableEpochsHandler + StateChangesCollector state.StateChangesCollector } // AccountCreator has method to create a new account type accountCreator struct { - hasher hashing.Hasher - marshaller marshal.Marshalizer - enableEpochsHandler common.EnableEpochsHandler + hasher hashing.Hasher + marshaller marshal.Marshalizer + enableEpochsHandler common.EnableEpochsHandler + stateChangesCollector state.StateChangesCollector } // NewAccountCreator creates a new instance of AccountCreator @@ -40,15 +42,16 @@ func NewAccountCreator(args ArgsAccountCreator) (state.AccountFactory, error) { } return &accountCreator{ - hasher: args.Hasher, - marshaller: args.Marshaller, - enableEpochsHandler: args.EnableEpochsHandler, + hasher: args.Hasher, + marshaller: args.Marshaller, + enableEpochsHandler: args.EnableEpochsHandler, + stateChangesCollector: args.StateChangesCollector, }, nil } // CreateAccount calls the new Account creator and returns the result func (ac *accountCreator) CreateAccount(address []byte) (vmcommon.AccountHandler, error) { - tdt, err := trackableDataTrie.NewTrackableDataTrie(address, ac.hasher, ac.marshaller, ac.enableEpochsHandler) + tdt, err := trackableDataTrie.NewTrackableDataTrie(address, ac.hasher, ac.marshaller, ac.enableEpochsHandler, ac.stateChangesCollector) if err != nil { return nil, err } diff --git a/state/stateChangesCollector.go b/state/stateChangesCollector.go index 24fa33c1499..1291df7481e 100644 --- a/state/stateChangesCollector.go +++ b/state/stateChangesCollector.go @@ -17,12 +17,14 @@ const ( // DataTrieChange represents a change in the data trie type DataTrieChange struct { - Key []byte `json:"key"` - Val []byte `json:"-"` + Type string `json:"type"` + Key []byte `json:"key"` + Val []byte `json:"-"` } // StateChangeDTO is used to collect state changes type StateChangeDTO struct { + Type string `json:"type"` MainTrieKey []byte `json:"mainTrieKey"` MainTrieVal []byte `json:"-"` DataTrieChanges []DataTrieChange `json:"dataTrieChanges"` diff --git a/state/trackableDataTrie/trackableDataTrie.go b/state/trackableDataTrie/trackableDataTrie.go index 4226974189a..6a4a7103d13 100644 --- a/state/trackableDataTrie/trackableDataTrie.go +++ b/state/trackableDataTrie/trackableDataTrie.go @@ -28,12 +28,13 @@ type dirtyData struct { // TrackableDataTrie wraps a PatriciaMerkelTrie adding modifying data capabilities type trackableDataTrie struct { - dirtyData map[string]dirtyData - tr common.Trie - hasher hashing.Hasher - marshaller marshal.Marshalizer - enableEpochsHandler common.EnableEpochsHandler - identifier []byte + dirtyData map[string]dirtyData + tr common.Trie + hasher hashing.Hasher + marshaller marshal.Marshalizer + enableEpochsHandler common.EnableEpochsHandler + identifier []byte + stateChangesCollector state.StateChangesCollector } // NewTrackableDataTrie returns an instance of trackableDataTrie @@ -42,6 +43,7 @@ func NewTrackableDataTrie( hasher hashing.Hasher, marshaller marshal.Marshalizer, enableEpochsHandler common.EnableEpochsHandler, + stateChangesCollector state.StateChangesCollector, ) (*trackableDataTrie, error) { if check.IfNil(hasher) { return nil, state.ErrNilHasher @@ -60,12 +62,13 @@ func NewTrackableDataTrie( } return &trackableDataTrie{ - tr: nil, - hasher: hasher, - marshaller: marshaller, - dirtyData: make(map[string]dirtyData), - identifier: identifier, - enableEpochsHandler: enableEpochsHandler, + tr: nil, + hasher: hasher, + marshaller: marshaller, + dirtyData: make(map[string]dirtyData), + identifier: identifier, + enableEpochsHandler: enableEpochsHandler, + stateChangesCollector: stateChangesCollector, }, nil } @@ -76,6 +79,7 @@ func (tdt *trackableDataTrie) RetrieveValue(key []byte) ([]byte, uint32, error) // search in dirty data cache if dataEntry, found := tdt.dirtyData[string(key)]; found { log.Trace("retrieve value from dirty data", "key", key, "value", dataEntry.value, "account", tdt.identifier) + return dataEntry.value, 0, nil } @@ -95,6 +99,20 @@ func (tdt *trackableDataTrie) RetrieveValue(key []byte) ([]byte, uint32, error) log.Trace("retrieve value from trie", "key", key, "value", val, "account", tdt.identifier) + stateChange := state.StateChangeDTO{ + Type: "read", + MainTrieKey: tdt.identifier, + MainTrieVal: nil, + DataTrieChanges: []state.DataTrieChange{ + { + Type: "read", + Key: key, + Val: val, + }, + }, + } + tdt.stateChangesCollector.AddStateChange(stateChange) + return val, depth, nil } @@ -265,8 +283,9 @@ func (tdt *trackableDataTrie) updateTrie(dtr state.DataTrie) ([]state.DataTrieCh if wasDeleted { deletedKeys = append(deletedKeys, state.DataTrieChange{ - Key: []byte(key), - Val: nil, + Type: "write", + Key: []byte(key), + Val: nil, }, ) } @@ -295,8 +314,9 @@ func (tdt *trackableDataTrie) updateTrie(dtr state.DataTrie) ([]state.DataTrieCh } newData[dataEntry.index] = state.DataTrieChange{ - Key: dataTrieKey, - Val: dataTrieVal, + Type: "write", + Key: dataTrieKey, + Val: dataTrieVal, } } diff --git a/update/genesis/import.go b/update/genesis/import.go index 0568b58cb25..817e1ee57e1 100644 --- a/update/genesis/import.go +++ b/update/genesis/import.go @@ -287,9 +287,10 @@ func newAccountCreator( switch accType { case UserAccount: args := factory.ArgsAccountCreator{ - Hasher: hasher, - Marshaller: marshaller, - EnableEpochsHandler: handler, + Hasher: hasher, + Marshaller: marshaller, + EnableEpochsHandler: handler, + StateChangesCollector: disabledState.NewDisabledStateChangesCollector(), } return factory.NewAccountCreator(args) case ValidatorAccount: From 33d471c8ba958ee6cc091839e78ac9a7a5b3e85d Mon Sep 17 00:00:00 2001 From: ssd04 Date: Tue, 9 Jul 2024 16:22:44 +0300 Subject: [PATCH 05/55] added json encoder --- state/stateChangesCollector.go | 54 +++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/state/stateChangesCollector.go b/state/stateChangesCollector.go index 1291df7481e..b4798973adb 100644 --- a/state/stateChangesCollector.go +++ b/state/stateChangesCollector.go @@ -2,7 +2,6 @@ package state import ( "encoding/json" - "io/ioutil" "os" "path/filepath" @@ -27,6 +26,7 @@ type StateChangeDTO struct { Type string `json:"type"` MainTrieKey []byte `json:"mainTrieKey"` MainTrieVal []byte `json:"-"` + Operation string `json:"operation,omitempty"` DataTrieChanges []DataTrieChange `json:"dataTrieChanges"` } @@ -40,12 +40,36 @@ type StateChangesForTx struct { type stateChangesCollector struct { stateChanges []StateChangeDTO stateChangesForTx []StateChangesForTx + + jsonFile *os.File + jsonEncoder *json.Encoder } // NewStateChangesCollector creates a new StateChangesCollector func NewStateChangesCollector() *stateChangesCollector { + directory := filepath.Join(workingDir, defaultStateChangesPath) + args := core.ArgCreateFileArgument{ + Directory: directory, + Prefix: "stateChanges", + FileExtension: "json", + } + + jsonFile, err := core.CreateFile(args) + if err != nil { + log.Error("NewStateChangesCollector: failed to create json file") + } + + // _, err = jsonFile.Write([]byte("[")) + // if err != nil { + // log.Error("NewStateChangesCollector: failed to write to json file") + // } + + encoder := json.NewEncoder(jsonFile) + return &stateChangesCollector{ stateChanges: []StateChangeDTO{}, + jsonFile: jsonFile, + jsonEncoder: encoder, } } @@ -81,34 +105,18 @@ func (scc *stateChangesCollector) AddTxHashToCollectedStateChanges(txHash []byte } func (scc *stateChangesCollector) DumpToJSONFile() error { - directory := filepath.Join(workingDir, defaultStateChangesPath) - args := core.ArgCreateFileArgument{ - Directory: directory, - Prefix: "stateChanges", - FileExtension: "json", - } - jsonFile, err := core.CreateFile(args) - if err != nil { - return err + for _, stateChange := range scc.stateChangesForTx { + err := scc.jsonEncoder.Encode(stateChange) + if err != nil { + return err + } } - marshalledData, err := json.Marshal(scc.stateChangesForTx) - if err != nil { - return err - } - - // encoder := json.NewEncoder(jsonFile) - - // err = encoder.Encode(marshalledData) + // _, err := scc.jsonFile.Write([]byte(",")) // if err != nil { // return err // } - err = ioutil.WriteFile(jsonFile.Name(), marshalledData, os.ModePerm) - if err != nil { - return err - } - return nil } From 5b04c8f370ec662a0d5dbeece6594351106be716 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Tue, 9 Jul 2024 21:34:02 +0300 Subject: [PATCH 06/55] change to save to level db --- state/accountsDB.go | 33 ++++++++++++++++------------ state/stateChangesCollector.go | 40 +++++++++++----------------------- 2 files changed, 32 insertions(+), 41 deletions(-) diff --git a/state/accountsDB.go b/state/accountsDB.go index 910dc30290a..f6c1048615d 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -212,6 +212,7 @@ func (adb *AccountsDB) GetCode(codeHash []byte) []byte { Type: "read", MainTrieKey: codeHash, MainTrieVal: val, + Operation: "getCode", DataTrieChanges: nil, } adb.stateChangesCollector.AddStateChange(stateChange) @@ -371,6 +372,7 @@ func (adb *AccountsDB) updateOldCodeEntry(oldCodeHash []byte) (*CodeEntry, error Type: "read", MainTrieKey: oldCodeHash, MainTrieVal: nil, + Operation: "getCode", DataTrieChanges: nil, } adb.stateChangesCollector.AddStateChange(stateChange) @@ -390,6 +392,7 @@ func (adb *AccountsDB) updateOldCodeEntry(oldCodeHash []byte) (*CodeEntry, error Type: "write", MainTrieKey: oldCodeHash, MainTrieVal: nil, + Operation: "writeCode", DataTrieChanges: nil, } adb.stateChangesCollector.AddStateChange(stateChange) @@ -407,6 +410,7 @@ func (adb *AccountsDB) updateOldCodeEntry(oldCodeHash []byte) (*CodeEntry, error Type: "write", MainTrieKey: oldCodeHash, MainTrieVal: codeEntryBytes, + Operation: "writeCode", DataTrieChanges: nil, } adb.stateChangesCollector.AddStateChange(stateChange) @@ -440,6 +444,7 @@ func (adb *AccountsDB) updateNewCodeEntry(newCodeHash []byte, newCode []byte) er Type: "write", MainTrieKey: newCodeHash, MainTrieVal: codeEntryBytes, + Operation: "writeCode", DataTrieChanges: nil, } adb.stateChangesCollector.AddStateChange(stateChange) @@ -900,19 +905,6 @@ func (adb *AccountsDB) commit() ([]byte, error) { log.Trace("accountsDB.Commit started") adb.entries = make([]JournalEntry, 0) - stateChanges := adb.stateChangesCollector.GetStateChanges() - printStateChanges(stateChanges) - err := adb.stateChangesCollector.DumpToJSONFile() - if err != nil { - log.Warn("failed to dump state changes to json file", "error", err) - } - - log.Debug("commit: dump to json file") - - adb.stateChangesCollector.Reset() - - log.Debug("commit: reset") - // if len(stateChanges) != 0 { // log.Warn("state changes collector is not empty", "state changes", stateChanges) // adb.stateChangesCollector.Reset() @@ -933,7 +925,7 @@ func (adb *AccountsDB) commit() ([]byte, error) { oldRoot := adb.mainTrie.GetOldRoot() // Step 2. commit main trie - err = adb.commitTrie(adb.mainTrie, oldHashes, newHashes) + err := adb.commitTrie(adb.mainTrie, oldHashes, newHashes) if err != nil { return nil, err } @@ -1299,6 +1291,19 @@ func collectStats( func (adb *AccountsDB) SetTxHashForLatestStateChanges(txHash []byte, tx *transaction.Transaction) { adb.stateChangesCollector.AddTxHashToCollectedStateChanges(txHash, tx) + stateChanges := adb.stateChangesCollector.GetStateChanges() + printStateChanges(stateChanges) + err := adb.stateChangesCollector.DumpToJSONFile() + if err != nil { + log.Warn("failed to dump state changes to json file", "error", err) + } + + log.Debug("SetTxHashForLatestStateChanges: dump to json file") + + adb.stateChangesCollector.Reset() + + log.Debug("SetTxHashForLatestStateChanges: reset") + } func (adb *AccountsDB) ResetStateChangesCollector() []StateChangesForTx { diff --git a/state/stateChangesCollector.go b/state/stateChangesCollector.go index b4798973adb..78e63e589a3 100644 --- a/state/stateChangesCollector.go +++ b/state/stateChangesCollector.go @@ -2,11 +2,11 @@ package state import ( "encoding/json" - "os" "path/filepath" - "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/multiversx/mx-chain-storage-go/leveldb" + "github.com/multiversx/mx-chain-storage-go/types" ) const ( @@ -41,35 +41,21 @@ type stateChangesCollector struct { stateChanges []StateChangeDTO stateChangesForTx []StateChangesForTx - jsonFile *os.File - jsonEncoder *json.Encoder + storer types.Persister } // NewStateChangesCollector creates a new StateChangesCollector func NewStateChangesCollector() *stateChangesCollector { - directory := filepath.Join(workingDir, defaultStateChangesPath) - args := core.ArgCreateFileArgument{ - Directory: directory, - Prefix: "stateChanges", - FileExtension: "json", - } + dbPath := filepath.Join(workingDir, "stateChangesDB", "StateChanges") - jsonFile, err := core.CreateFile(args) + db, err := leveldb.NewSerialDB(dbPath, 2, 100, 10) if err != nil { - log.Error("NewStateChangesCollector: failed to create json file") + log.Error("NewStateChangesCollector: failed to create level db") } - // _, err = jsonFile.Write([]byte("[")) - // if err != nil { - // log.Error("NewStateChangesCollector: failed to write to json file") - // } - - encoder := json.NewEncoder(jsonFile) - return &stateChangesCollector{ stateChanges: []StateChangeDTO{}, - jsonFile: jsonFile, - jsonEncoder: encoder, + storer: db, } } @@ -106,16 +92,16 @@ func (scc *stateChangesCollector) AddTxHashToCollectedStateChanges(txHash []byte func (scc *stateChangesCollector) DumpToJSONFile() error { for _, stateChange := range scc.stateChangesForTx { - err := scc.jsonEncoder.Encode(stateChange) + marshalledData, err := json.Marshal(stateChange) if err != nil { return err } - } - // _, err := scc.jsonFile.Write([]byte(",")) - // if err != nil { - // return err - // } + err = scc.storer.Put(stateChange.TxHash, marshalledData) + if err != nil { + return err + } + } return nil } From 00f7ba41573afb0cdaebb1f912aa87bd24128d81 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Tue, 9 Jul 2024 21:34:38 +0300 Subject: [PATCH 07/55] use diabled components for peer accounts --- factory/state/stateComponents.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/factory/state/stateComponents.go b/factory/state/stateComponents.go index 1a1bdc41a6f..af9e2466507 100644 --- a/factory/state/stateComponents.go +++ b/factory/state/stateComponents.go @@ -240,9 +240,9 @@ func (scf *stateComponentsFactory) createPeerAdapter(triesContainer common.Tries } stateChangesCollector := disabled.NewDisabledStateChangesCollector() - if scf.config.StateTriesConfig.CollectStateChangesEnabled { - stateChangesCollector = state.NewStateChangesCollector() - } + // if scf.config.StateTriesConfig.CollectStateChangesEnabled { + // stateChangesCollector = state.NewStateChangesCollector() + // } argStateMetrics := stateMetrics.ArgsStateMetrics{ SnapshotInProgressKey: common.MetricPeersSnapshotInProgress, From d31042c6a0fff163b8fadcd4b83c1f6291febf0f Mon Sep 17 00:00:00 2001 From: ssd04 Date: Fri, 12 Jul 2024 17:07:14 +0300 Subject: [PATCH 08/55] add save account operation type --- state/accountsDB.go | 1 + 1 file changed, 1 insertion(+) diff --git a/state/accountsDB.go b/state/accountsDB.go index f6c1048615d..674b24972be 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -288,6 +288,7 @@ func (adb *AccountsDB) SaveAccount(account vmcommon.AccountHandler) error { MainTrieKey: account.AddressBytes(), MainTrieVal: marshalledAccount, DataTrieChanges: newDataTrieValues, + Operation: "saveAccount", } adb.stateChangesCollector.AddStateChange(stateChange) From a232b4b99d4f121c58ab1ddb88675677355fe289 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Tue, 16 Jul 2024 11:41:34 +0300 Subject: [PATCH 09/55] added changed fields on save account --- state/accountsDB.go | 35 ++++++++++++++++++++++++++++++++-- state/stateChangesCollector.go | 5 ++++- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/state/accountsDB.go b/state/accountsDB.go index 674b24972be..06dba47fea8 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -283,18 +283,49 @@ func (adb *AccountsDB) SaveAccount(account vmcommon.AccountHandler) error { return err } - stateChange := StateChangeDTO{ + stateChange := &StateChangeDTO{ Type: "write", MainTrieKey: account.AddressBytes(), MainTrieVal: marshalledAccount, DataTrieChanges: newDataTrieValues, Operation: "saveAccount", } - adb.stateChangesCollector.AddStateChange(stateChange) + + checkAccountChanges(oldAccount, account, stateChange) + + adb.stateChangesCollector.AddStateChange(*stateChange) return err } +func checkAccountChanges(oldAcc, newAcc vmcommon.AccountHandler, stateChange *StateChangeDTO) { + baseNewAcc, newAccOk := newAcc.(UserAccountHandler) + if !newAccOk { + return + } + baseOldAccount, oldAccOk := oldAcc.(UserAccountHandler) + if !oldAccOk { + return + } + + if baseNewAcc.GetNonce() != baseOldAccount.GetNonce() { + log.Trace("CheckAcc: diff nonce") + stateChange.Nonce = true + } + + if baseNewAcc.GetBalance().Uint64() != baseOldAccount.GetBalance().Uint64() { + log.Trace("CheckAcc: diff balance") + stateChange.Balance = true + } + + if bytes.Equal(baseNewAcc.GetRootHash(), baseOldAccount.GetRootHash()) { + log.Trace("CheckAcc: diff rootHash") + stateChange.RootHash = true + } + + log.Trace("CheckAcc: checked all") +} + func (adb *AccountsDB) saveCodeAndDataTrie(oldAcc, newAcc vmcommon.AccountHandler) ([]DataTrieChange, error) { baseNewAcc, newAccOk := newAcc.(baseAccountHandler) baseOldAccount, _ := oldAcc.(baseAccountHandler) diff --git a/state/stateChangesCollector.go b/state/stateChangesCollector.go index 78e63e589a3..93f5c749257 100644 --- a/state/stateChangesCollector.go +++ b/state/stateChangesCollector.go @@ -26,7 +26,10 @@ type StateChangeDTO struct { Type string `json:"type"` MainTrieKey []byte `json:"mainTrieKey"` MainTrieVal []byte `json:"-"` - Operation string `json:"operation,omitempty"` + Operation string `json:"operation"` + Balance bool `json:"balanceChanged"` + Nonce bool `json:"nonceChanged"` + RootHash bool `json:"rootHashChanged"` DataTrieChanges []DataTrieChange `json:"dataTrieChanges"` } From c501ee990ef42c63b38fb391231cdb080ced6717 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Fri, 19 Jul 2024 11:56:50 +0300 Subject: [PATCH 10/55] added revert state changes --- state/accountsDB.go | 54 +++++++++++--- state/journalEntries.go | 130 ++++++++++++++++++++++++++------- state/stateChangesCollector.go | 7 +- 3 files changed, 152 insertions(+), 39 deletions(-) diff --git a/state/accountsDB.go b/state/accountsDB.go index 06dba47fea8..9eb282afe41 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -260,13 +260,13 @@ func (adb *AccountsDB) SaveAccount(account vmcommon.AccountHandler) error { var entry JournalEntry if check.IfNil(oldAccount) { - entry, err = NewJournalEntryAccountCreation(account.AddressBytes(), adb.mainTrie) + entry, err = NewJournalEntryAccountCreation(account.AddressBytes(), adb.mainTrie, adb.stateChangesCollector) if err != nil { return err } adb.journalize(entry) } else { - entry, err = NewJournalEntryAccount(oldAccount) + entry, err = NewJournalEntryAccount(oldAccount, adb.stateChangesCollector) if err != nil { return err } @@ -301,29 +301,50 @@ func (adb *AccountsDB) SaveAccount(account vmcommon.AccountHandler) error { func checkAccountChanges(oldAcc, newAcc vmcommon.AccountHandler, stateChange *StateChangeDTO) { baseNewAcc, newAccOk := newAcc.(UserAccountHandler) if !newAccOk { + log.Trace("checkAccountChanges: Not able to cast new acc") return } baseOldAccount, oldAccOk := oldAcc.(UserAccountHandler) if !oldAccOk { + log.Trace("checkAccountChanges: Not able to cast old acc") return } if baseNewAcc.GetNonce() != baseOldAccount.GetNonce() { - log.Trace("CheckAcc: diff nonce") stateChange.Nonce = true } if baseNewAcc.GetBalance().Uint64() != baseOldAccount.GetBalance().Uint64() { - log.Trace("CheckAcc: diff balance") stateChange.Balance = true } - if bytes.Equal(baseNewAcc.GetRootHash(), baseOldAccount.GetRootHash()) { - log.Trace("CheckAcc: diff rootHash") + if !bytes.Equal(baseNewAcc.GetCodeHash(), baseOldAccount.GetCodeHash()) { + stateChange.CodeHash = true + } + + if !bytes.Equal(baseNewAcc.GetRootHash(), baseOldAccount.GetRootHash()) { stateChange.RootHash = true + log.Trace("checkAccountChanges: rootHash changed", "address", baseNewAcc.AddressBytes()) + } + log.Trace("checkAccountChanges: rootHash NOT changed", "address", baseNewAcc.AddressBytes()) + + if !bytes.Equal(baseNewAcc.GetDeveloperReward().Bytes(), baseOldAccount.GetDeveloperReward().Bytes()) { + stateChange.DeveloperReward = true + } + + if !bytes.Equal(baseNewAcc.GetOwnerAddress(), baseOldAccount.GetOwnerAddress()) { + stateChange.OwnerAddress = true + } + + if !bytes.Equal(baseNewAcc.GetUserName(), baseOldAccount.GetUserName()) { + stateChange.UserName = true } - log.Trace("CheckAcc: checked all") + if !bytes.Equal(baseNewAcc.GetCodeMetadata(), baseOldAccount.GetCodeMetadata()) { + stateChange.CodeMetadata = true + } + + log.Trace("checkAccountChanges: checked all", "address", baseNewAcc.AddressBytes()) } func (adb *AccountsDB) saveCodeAndDataTrie(oldAcc, newAcc vmcommon.AccountHandler) ([]DataTrieChange, error) { @@ -380,7 +401,7 @@ func (adb *AccountsDB) saveCode(newAcc, oldAcc baseAccountHandler) error { return err } - entry, err := NewJournalEntryCode(unmodifiedOldCodeEntry, oldCodeHash, newCodeHash, adb.mainTrie, adb.marshaller) + entry, err := NewJournalEntryCode(unmodifiedOldCodeEntry, oldCodeHash, newCodeHash, adb.mainTrie, adb.marshaller, adb.stateChangesCollector) if err != nil { return err } @@ -554,7 +575,7 @@ func (adb *AccountsDB) saveDataTrie(accountHandler baseAccountHandler) ([]DataTr return nil, nil } - entry, err := NewJournalEntryDataTrieUpdates(oldValues, accountHandler) + entry, err := NewJournalEntryDataTrieUpdates(oldValues, accountHandler, adb.stateChangesCollector) if err != nil { return nil, err } @@ -565,6 +586,7 @@ func (adb *AccountsDB) saveDataTrie(accountHandler baseAccountHandler) ([]DataTr return nil, err } accountHandler.SetRootHash(rootHash) + log.Trace("saveDataTrie: rootHash changed", "address", accountHandler.AddressBytes()) if check.IfNil(adb.dataTries.Get(accountHandler.AddressBytes())) { trie, ok := accountHandler.DataTrie().(common.Trie) @@ -617,7 +639,7 @@ func (adb *AccountsDB) RemoveAccount(address []byte) error { return fmt.Errorf("%w in RemoveAccount for address %s", ErrAccNotFound, address) } - entry, err := NewJournalEntryAccount(acnt) + entry, err := NewJournalEntryAccount(acnt, adb.stateChangesCollector) if err != nil { return err } @@ -672,12 +694,20 @@ func (adb *AccountsDB) removeDataTrie(baseAcc baseAccountHandler) error { adb.obsoleteDataTrieHashes[string(rootHash)] = hashes - entry, err := NewJournalEntryDataTrieRemove(rootHash, adb.obsoleteDataTrieHashes) + entry, err := NewJournalEntryDataTrieRemove(rootHash, adb.obsoleteDataTrieHashes, adb.stateChangesCollector) if err != nil { return err } adb.journalize(entry) + stateChange := &StateChangeDTO{ + Type: "write", + MainTrieKey: baseAcc.AddressBytes(), + Operation: "removeDataTrie", + DataTrieChanges: nil, + } + adb.stateChangesCollector.AddStateChange(*stateChange) + return nil } @@ -688,7 +718,7 @@ func (adb *AccountsDB) removeCode(baseAcc baseAccountHandler) error { return err } - codeChangeEntry, err := NewJournalEntryCode(unmodifiedOldCodeEntry, oldCodeHash, nil, adb.mainTrie, adb.marshaller) + codeChangeEntry, err := NewJournalEntryCode(unmodifiedOldCodeEntry, oldCodeHash, nil, adb.mainTrie, adb.marshaller, adb.stateChangesCollector) if err != nil { return err } diff --git a/state/journalEntries.go b/state/journalEntries.go index a4fa75dcf5a..99bf3cfd39a 100644 --- a/state/journalEntries.go +++ b/state/journalEntries.go @@ -11,11 +11,12 @@ import ( ) type journalEntryCode struct { - oldCodeEntry *CodeEntry - oldCodeHash []byte - newCodeHash []byte - trie Updater - marshalizer marshal.Marshalizer + oldCodeEntry *CodeEntry + oldCodeHash []byte + newCodeHash []byte + trie Updater + marshalizer marshal.Marshalizer + stateChangesCollector StateChangesCollector } // NewJournalEntryCode creates a new instance of JournalEntryCode @@ -25,6 +26,7 @@ func NewJournalEntryCode( newCodeHash []byte, trie Updater, marshalizer marshal.Marshalizer, + stateChangesCollector StateChangesCollector, ) (*journalEntryCode, error) { if check.IfNil(trie) { return nil, ErrNilUpdater @@ -34,11 +36,12 @@ func NewJournalEntryCode( } return &journalEntryCode{ - oldCodeEntry: oldCodeEntry, - oldCodeHash: oldCodeHash, - newCodeHash: newCodeHash, - trie: trie, - marshalizer: marshalizer, + oldCodeEntry: oldCodeEntry, + oldCodeHash: oldCodeHash, + newCodeHash: newCodeHash, + trie: trie, + marshalizer: marshalizer, + stateChangesCollector: stateChangesCollector, }, nil } @@ -66,11 +69,20 @@ func (jea *journalEntryCode) revertOldCodeEntry() error { return nil } - _, err := saveCodeEntry(jea.oldCodeHash, jea.oldCodeEntry, jea.trie, jea.marshalizer) + codeEntryBytes, err := saveCodeEntry(jea.oldCodeHash, jea.oldCodeEntry, jea.trie, jea.marshalizer) if err != nil { return err } + stateChange := StateChangeDTO{ + Type: "revert", + MainTrieKey: jea.oldCodeHash, + MainTrieVal: codeEntryBytes, + Operation: "writeCode", + DataTrieChanges: nil, + } + jea.stateChangesCollector.AddStateChange(stateChange) + return nil } @@ -90,15 +102,33 @@ func (jea *journalEntryCode) revertNewCodeEntry() error { return err } + stateChange := StateChangeDTO{ + Type: "revert", + MainTrieKey: jea.newCodeHash, + MainTrieVal: nil, + Operation: "writeCode", + DataTrieChanges: nil, + } + jea.stateChangesCollector.AddStateChange(stateChange) + return nil } newCodeEntry.NumReferences-- - _, err = saveCodeEntry(jea.newCodeHash, newCodeEntry, jea.trie, jea.marshalizer) + codeEntryBytes, err := saveCodeEntry(jea.newCodeHash, newCodeEntry, jea.trie, jea.marshalizer) if err != nil { return err } + stateChange := StateChangeDTO{ + Type: "revert", + MainTrieKey: jea.newCodeHash, + MainTrieVal: codeEntryBytes, + Operation: "writeCode", + DataTrieChanges: nil, + } + jea.stateChangesCollector.AddStateChange(stateChange) + return nil } @@ -109,22 +139,32 @@ func (jea *journalEntryCode) IsInterfaceNil() bool { // JournalEntryAccount represents a journal entry for account fields change type journalEntryAccount struct { - account vmcommon.AccountHandler + account vmcommon.AccountHandler + stateChangesCollector StateChangesCollector } // NewJournalEntryAccount creates a new instance of JournalEntryAccount -func NewJournalEntryAccount(account vmcommon.AccountHandler) (*journalEntryAccount, error) { +func NewJournalEntryAccount(account vmcommon.AccountHandler, stateChangesCollector StateChangesCollector) (*journalEntryAccount, error) { if check.IfNil(account) { return nil, fmt.Errorf("%w in NewJournalEntryAccount", ErrNilAccountHandler) } return &journalEntryAccount{ - account: account, + account: account, + stateChangesCollector: stateChangesCollector, }, nil } // Revert applies undo operation func (jea *journalEntryAccount) Revert() (vmcommon.AccountHandler, error) { + stateChange := StateChangeDTO{ + Type: "revert", + MainTrieKey: jea.account.AddressBytes(), + Operation: "revertAccount", + DataTrieChanges: nil, + } + jea.stateChangesCollector.AddStateChange(stateChange) + return jea.account, nil } @@ -135,12 +175,13 @@ func (jea *journalEntryAccount) IsInterfaceNil() bool { // JournalEntryAccountCreation represents a journal entry for account creation type journalEntryAccountCreation struct { - address []byte - updater Updater + address []byte + updater Updater + stateChangesCollector StateChangesCollector } // NewJournalEntryAccountCreation creates a new instance of JournalEntryAccountCreation -func NewJournalEntryAccountCreation(address []byte, updater Updater) (*journalEntryAccountCreation, error) { +func NewJournalEntryAccountCreation(address []byte, updater Updater, stateChangesCollector StateChangesCollector) (*journalEntryAccountCreation, error) { if check.IfNil(updater) { return nil, ErrNilUpdater } @@ -149,13 +190,22 @@ func NewJournalEntryAccountCreation(address []byte, updater Updater) (*journalEn } return &journalEntryAccountCreation{ - address: address, - updater: updater, + address: address, + updater: updater, + stateChangesCollector: stateChangesCollector, }, nil } // Revert applies undo operation func (jea *journalEntryAccountCreation) Revert() (vmcommon.AccountHandler, error) { + stateChange := StateChangeDTO{ + Type: "revert", + MainTrieKey: jea.address, + Operation: "revertNewAccount", + DataTrieChanges: nil, + } + jea.stateChangesCollector.AddStateChange(stateChange) + return nil, jea.updater.Update(jea.address, nil) } @@ -167,12 +217,13 @@ func (jea *journalEntryAccountCreation) IsInterfaceNil() bool { // JournalEntryDataTrieUpdates stores all the updates done to the account's data trie, // so it can be reverted in case of rollback type journalEntryDataTrieUpdates struct { - trieUpdates []core.TrieData - account baseAccountHandler + trieUpdates []core.TrieData + account baseAccountHandler + stateChangesCollector StateChangesCollector } // NewJournalEntryDataTrieUpdates outputs a new JournalEntryDataTrieUpdates implementation used to revert an account's data trie -func NewJournalEntryDataTrieUpdates(trieUpdates []core.TrieData, account baseAccountHandler) (*journalEntryDataTrieUpdates, error) { +func NewJournalEntryDataTrieUpdates(trieUpdates []core.TrieData, account baseAccountHandler, stateChangesCollector StateChangesCollector) (*journalEntryDataTrieUpdates, error) { if check.IfNil(account) { return nil, fmt.Errorf("%w in NewJournalEntryDataTrieUpdates", ErrNilAccountHandler) } @@ -181,8 +232,9 @@ func NewJournalEntryDataTrieUpdates(trieUpdates []core.TrieData, account baseAcc } return &journalEntryDataTrieUpdates{ - trieUpdates: trieUpdates, - account: account, + trieUpdates: trieUpdates, + account: account, + stateChangesCollector: stateChangesCollector, }, nil } @@ -193,6 +245,8 @@ func (jedtu *journalEntryDataTrieUpdates) Revert() (vmcommon.AccountHandler, err return nil, fmt.Errorf("invalid trie, type is %T", jedtu.account.DataTrie()) } + dataTrieChanges := make([]DataTrieChange, len(jedtu.trieUpdates)) + for _, trieUpdate := range jedtu.trieUpdates { err := trie.UpdateWithVersion(trieUpdate.Key, trieUpdate.Value, trieUpdate.Version) if err != nil { @@ -204,6 +258,12 @@ func (jedtu *journalEntryDataTrieUpdates) Revert() (vmcommon.AccountHandler, err "val", trieUpdate.Value, "version", trieUpdate.Version, ) + + dataTrieChanges = append(dataTrieChanges, DataTrieChange{ + Type: "write", + Key: trieUpdate.Key, + Val: trieUpdate.Value, + }) } rootHash, err := trie.RootHash() @@ -213,6 +273,14 @@ func (jedtu *journalEntryDataTrieUpdates) Revert() (vmcommon.AccountHandler, err jedtu.account.SetRootHash(rootHash) + stateChange := StateChangeDTO{ + Type: "revert", + MainTrieKey: jedtu.account.AddressBytes(), + Operation: "dataTrieUpdates", + DataTrieChanges: dataTrieChanges, + } + jedtu.stateChangesCollector.AddStateChange(stateChange) + return jedtu.account, nil } @@ -225,11 +293,12 @@ func (jedtu *journalEntryDataTrieUpdates) IsInterfaceNil() bool { type journalEntryDataTrieRemove struct { rootHash []byte obsoleteDataTrieHashes map[string][][]byte + stateChangesCollector StateChangesCollector } // NewJournalEntryDataTrieRemove outputs a new journalEntryDataTrieRemove implementation used to cancel // the eviction of the hashes from the data trie with the given root hash -func NewJournalEntryDataTrieRemove(rootHash []byte, obsoleteDataTrieHashes map[string][][]byte) (*journalEntryDataTrieRemove, error) { +func NewJournalEntryDataTrieRemove(rootHash []byte, obsoleteDataTrieHashes map[string][][]byte, stateChangesCollector StateChangesCollector) (*journalEntryDataTrieRemove, error) { if obsoleteDataTrieHashes == nil { return nil, fmt.Errorf("%w in NewJournalEntryDataTrieRemove", ErrNilMapOfHashes) } @@ -240,6 +309,7 @@ func NewJournalEntryDataTrieRemove(rootHash []byte, obsoleteDataTrieHashes map[s return &journalEntryDataTrieRemove{ rootHash: rootHash, obsoleteDataTrieHashes: obsoleteDataTrieHashes, + stateChangesCollector: stateChangesCollector, }, nil } @@ -247,6 +317,14 @@ func NewJournalEntryDataTrieRemove(rootHash []byte, obsoleteDataTrieHashes map[s func (jedtr *journalEntryDataTrieRemove) Revert() (vmcommon.AccountHandler, error) { delete(jedtr.obsoleteDataTrieHashes, string(jedtr.rootHash)) + stateChange := StateChangeDTO{ + Type: "revert", + MainTrieKey: jedtr.rootHash, + Operation: "removeDataTrie", + DataTrieChanges: nil, + } + jedtr.stateChangesCollector.AddStateChange(stateChange) + return nil, nil } diff --git a/state/stateChangesCollector.go b/state/stateChangesCollector.go index 93f5c749257..0710d7f6afa 100644 --- a/state/stateChangesCollector.go +++ b/state/stateChangesCollector.go @@ -27,9 +27,14 @@ type StateChangeDTO struct { MainTrieKey []byte `json:"mainTrieKey"` MainTrieVal []byte `json:"-"` Operation string `json:"operation"` - Balance bool `json:"balanceChanged"` Nonce bool `json:"nonceChanged"` + Balance bool `json:"balanceChanged"` + CodeHash bool `json:"codeHashChanged"` RootHash bool `json:"rootHashChanged"` + DeveloperReward bool `json:"developerRewardChanged"` + OwnerAddress bool `json:"ownerAddressChanged"` + UserName bool `json:"userNameChanged"` + CodeMetadata bool `json:"codeMetadataChanged"` DataTrieChanges []DataTrieChange `json:"dataTrieChanges"` } From 4f579cc5850d18becd4ad3b1f945045d0d9068dd Mon Sep 17 00:00:00 2001 From: ssd04 Date: Fri, 2 Aug 2024 16:04:48 +0300 Subject: [PATCH 11/55] remove state changes collector from journal entries --- state/accountsDB.go | 38 ++++++------ state/journalEntries.go | 130 ++++++++-------------------------------- 2 files changed, 44 insertions(+), 124 deletions(-) diff --git a/state/accountsDB.go b/state/accountsDB.go index 9eb282afe41..c9f34f24417 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -260,13 +260,13 @@ func (adb *AccountsDB) SaveAccount(account vmcommon.AccountHandler) error { var entry JournalEntry if check.IfNil(oldAccount) { - entry, err = NewJournalEntryAccountCreation(account.AddressBytes(), adb.mainTrie, adb.stateChangesCollector) + entry, err = NewJournalEntryAccountCreation(account.AddressBytes(), adb.mainTrie) if err != nil { return err } adb.journalize(entry) } else { - entry, err = NewJournalEntryAccount(oldAccount, adb.stateChangesCollector) + entry, err = NewJournalEntryAccount(oldAccount) if err != nil { return err } @@ -401,7 +401,7 @@ func (adb *AccountsDB) saveCode(newAcc, oldAcc baseAccountHandler) error { return err } - entry, err := NewJournalEntryCode(unmodifiedOldCodeEntry, oldCodeHash, newCodeHash, adb.mainTrie, adb.marshaller, adb.stateChangesCollector) + entry, err := NewJournalEntryCode(unmodifiedOldCodeEntry, oldCodeHash, newCodeHash, adb.mainTrie, adb.marshaller) if err != nil { return err } @@ -575,7 +575,7 @@ func (adb *AccountsDB) saveDataTrie(accountHandler baseAccountHandler) ([]DataTr return nil, nil } - entry, err := NewJournalEntryDataTrieUpdates(oldValues, accountHandler, adb.stateChangesCollector) + entry, err := NewJournalEntryDataTrieUpdates(oldValues, accountHandler) if err != nil { return nil, err } @@ -639,7 +639,7 @@ func (adb *AccountsDB) RemoveAccount(address []byte) error { return fmt.Errorf("%w in RemoveAccount for address %s", ErrAccNotFound, address) } - entry, err := NewJournalEntryAccount(acnt, adb.stateChangesCollector) + entry, err := NewJournalEntryAccount(acnt) if err != nil { return err } @@ -694,7 +694,7 @@ func (adb *AccountsDB) removeDataTrie(baseAcc baseAccountHandler) error { adb.obsoleteDataTrieHashes[string(rootHash)] = hashes - entry, err := NewJournalEntryDataTrieRemove(rootHash, adb.obsoleteDataTrieHashes, adb.stateChangesCollector) + entry, err := NewJournalEntryDataTrieRemove(rootHash, adb.obsoleteDataTrieHashes) if err != nil { return err } @@ -718,7 +718,7 @@ func (adb *AccountsDB) removeCode(baseAcc baseAccountHandler) error { return err } - codeChangeEntry, err := NewJournalEntryCode(unmodifiedOldCodeEntry, oldCodeHash, nil, adb.mainTrie, adb.marshaller, adb.stateChangesCollector) + codeChangeEntry, err := NewJournalEntryCode(unmodifiedOldCodeEntry, oldCodeHash, nil, adb.mainTrie, adb.marshaller) if err != nil { return err } @@ -972,6 +972,17 @@ func (adb *AccountsDB) commit() ([]byte, error) { // adb.stateChangesCollector.Reset() // } + stateChanges := adb.stateChangesCollector.GetStateChanges() + printStateChanges(stateChanges) + err := adb.stateChangesCollector.DumpToJSONFile() + if err != nil { + log.Warn("failed to dump state changes to json file", "error", err) + } + log.Debug("SetTxHashForLatestStateChanges: dump to json file") + + adb.stateChangesCollector.Reset() + log.Debug("SetTxHashForLatestStateChanges: reset") + oldHashes := make(common.ModifiedHashes) newHashes := make(common.ModifiedHashes) // Step 1. commit all data tries @@ -1353,19 +1364,6 @@ func collectStats( func (adb *AccountsDB) SetTxHashForLatestStateChanges(txHash []byte, tx *transaction.Transaction) { adb.stateChangesCollector.AddTxHashToCollectedStateChanges(txHash, tx) - stateChanges := adb.stateChangesCollector.GetStateChanges() - printStateChanges(stateChanges) - err := adb.stateChangesCollector.DumpToJSONFile() - if err != nil { - log.Warn("failed to dump state changes to json file", "error", err) - } - - log.Debug("SetTxHashForLatestStateChanges: dump to json file") - - adb.stateChangesCollector.Reset() - - log.Debug("SetTxHashForLatestStateChanges: reset") - } func (adb *AccountsDB) ResetStateChangesCollector() []StateChangesForTx { diff --git a/state/journalEntries.go b/state/journalEntries.go index 99bf3cfd39a..a4fa75dcf5a 100644 --- a/state/journalEntries.go +++ b/state/journalEntries.go @@ -11,12 +11,11 @@ import ( ) type journalEntryCode struct { - oldCodeEntry *CodeEntry - oldCodeHash []byte - newCodeHash []byte - trie Updater - marshalizer marshal.Marshalizer - stateChangesCollector StateChangesCollector + oldCodeEntry *CodeEntry + oldCodeHash []byte + newCodeHash []byte + trie Updater + marshalizer marshal.Marshalizer } // NewJournalEntryCode creates a new instance of JournalEntryCode @@ -26,7 +25,6 @@ func NewJournalEntryCode( newCodeHash []byte, trie Updater, marshalizer marshal.Marshalizer, - stateChangesCollector StateChangesCollector, ) (*journalEntryCode, error) { if check.IfNil(trie) { return nil, ErrNilUpdater @@ -36,12 +34,11 @@ func NewJournalEntryCode( } return &journalEntryCode{ - oldCodeEntry: oldCodeEntry, - oldCodeHash: oldCodeHash, - newCodeHash: newCodeHash, - trie: trie, - marshalizer: marshalizer, - stateChangesCollector: stateChangesCollector, + oldCodeEntry: oldCodeEntry, + oldCodeHash: oldCodeHash, + newCodeHash: newCodeHash, + trie: trie, + marshalizer: marshalizer, }, nil } @@ -69,20 +66,11 @@ func (jea *journalEntryCode) revertOldCodeEntry() error { return nil } - codeEntryBytes, err := saveCodeEntry(jea.oldCodeHash, jea.oldCodeEntry, jea.trie, jea.marshalizer) + _, err := saveCodeEntry(jea.oldCodeHash, jea.oldCodeEntry, jea.trie, jea.marshalizer) if err != nil { return err } - stateChange := StateChangeDTO{ - Type: "revert", - MainTrieKey: jea.oldCodeHash, - MainTrieVal: codeEntryBytes, - Operation: "writeCode", - DataTrieChanges: nil, - } - jea.stateChangesCollector.AddStateChange(stateChange) - return nil } @@ -102,33 +90,15 @@ func (jea *journalEntryCode) revertNewCodeEntry() error { return err } - stateChange := StateChangeDTO{ - Type: "revert", - MainTrieKey: jea.newCodeHash, - MainTrieVal: nil, - Operation: "writeCode", - DataTrieChanges: nil, - } - jea.stateChangesCollector.AddStateChange(stateChange) - return nil } newCodeEntry.NumReferences-- - codeEntryBytes, err := saveCodeEntry(jea.newCodeHash, newCodeEntry, jea.trie, jea.marshalizer) + _, err = saveCodeEntry(jea.newCodeHash, newCodeEntry, jea.trie, jea.marshalizer) if err != nil { return err } - stateChange := StateChangeDTO{ - Type: "revert", - MainTrieKey: jea.newCodeHash, - MainTrieVal: codeEntryBytes, - Operation: "writeCode", - DataTrieChanges: nil, - } - jea.stateChangesCollector.AddStateChange(stateChange) - return nil } @@ -139,32 +109,22 @@ func (jea *journalEntryCode) IsInterfaceNil() bool { // JournalEntryAccount represents a journal entry for account fields change type journalEntryAccount struct { - account vmcommon.AccountHandler - stateChangesCollector StateChangesCollector + account vmcommon.AccountHandler } // NewJournalEntryAccount creates a new instance of JournalEntryAccount -func NewJournalEntryAccount(account vmcommon.AccountHandler, stateChangesCollector StateChangesCollector) (*journalEntryAccount, error) { +func NewJournalEntryAccount(account vmcommon.AccountHandler) (*journalEntryAccount, error) { if check.IfNil(account) { return nil, fmt.Errorf("%w in NewJournalEntryAccount", ErrNilAccountHandler) } return &journalEntryAccount{ - account: account, - stateChangesCollector: stateChangesCollector, + account: account, }, nil } // Revert applies undo operation func (jea *journalEntryAccount) Revert() (vmcommon.AccountHandler, error) { - stateChange := StateChangeDTO{ - Type: "revert", - MainTrieKey: jea.account.AddressBytes(), - Operation: "revertAccount", - DataTrieChanges: nil, - } - jea.stateChangesCollector.AddStateChange(stateChange) - return jea.account, nil } @@ -175,13 +135,12 @@ func (jea *journalEntryAccount) IsInterfaceNil() bool { // JournalEntryAccountCreation represents a journal entry for account creation type journalEntryAccountCreation struct { - address []byte - updater Updater - stateChangesCollector StateChangesCollector + address []byte + updater Updater } // NewJournalEntryAccountCreation creates a new instance of JournalEntryAccountCreation -func NewJournalEntryAccountCreation(address []byte, updater Updater, stateChangesCollector StateChangesCollector) (*journalEntryAccountCreation, error) { +func NewJournalEntryAccountCreation(address []byte, updater Updater) (*journalEntryAccountCreation, error) { if check.IfNil(updater) { return nil, ErrNilUpdater } @@ -190,22 +149,13 @@ func NewJournalEntryAccountCreation(address []byte, updater Updater, stateChange } return &journalEntryAccountCreation{ - address: address, - updater: updater, - stateChangesCollector: stateChangesCollector, + address: address, + updater: updater, }, nil } // Revert applies undo operation func (jea *journalEntryAccountCreation) Revert() (vmcommon.AccountHandler, error) { - stateChange := StateChangeDTO{ - Type: "revert", - MainTrieKey: jea.address, - Operation: "revertNewAccount", - DataTrieChanges: nil, - } - jea.stateChangesCollector.AddStateChange(stateChange) - return nil, jea.updater.Update(jea.address, nil) } @@ -217,13 +167,12 @@ func (jea *journalEntryAccountCreation) IsInterfaceNil() bool { // JournalEntryDataTrieUpdates stores all the updates done to the account's data trie, // so it can be reverted in case of rollback type journalEntryDataTrieUpdates struct { - trieUpdates []core.TrieData - account baseAccountHandler - stateChangesCollector StateChangesCollector + trieUpdates []core.TrieData + account baseAccountHandler } // NewJournalEntryDataTrieUpdates outputs a new JournalEntryDataTrieUpdates implementation used to revert an account's data trie -func NewJournalEntryDataTrieUpdates(trieUpdates []core.TrieData, account baseAccountHandler, stateChangesCollector StateChangesCollector) (*journalEntryDataTrieUpdates, error) { +func NewJournalEntryDataTrieUpdates(trieUpdates []core.TrieData, account baseAccountHandler) (*journalEntryDataTrieUpdates, error) { if check.IfNil(account) { return nil, fmt.Errorf("%w in NewJournalEntryDataTrieUpdates", ErrNilAccountHandler) } @@ -232,9 +181,8 @@ func NewJournalEntryDataTrieUpdates(trieUpdates []core.TrieData, account baseAcc } return &journalEntryDataTrieUpdates{ - trieUpdates: trieUpdates, - account: account, - stateChangesCollector: stateChangesCollector, + trieUpdates: trieUpdates, + account: account, }, nil } @@ -245,8 +193,6 @@ func (jedtu *journalEntryDataTrieUpdates) Revert() (vmcommon.AccountHandler, err return nil, fmt.Errorf("invalid trie, type is %T", jedtu.account.DataTrie()) } - dataTrieChanges := make([]DataTrieChange, len(jedtu.trieUpdates)) - for _, trieUpdate := range jedtu.trieUpdates { err := trie.UpdateWithVersion(trieUpdate.Key, trieUpdate.Value, trieUpdate.Version) if err != nil { @@ -258,12 +204,6 @@ func (jedtu *journalEntryDataTrieUpdates) Revert() (vmcommon.AccountHandler, err "val", trieUpdate.Value, "version", trieUpdate.Version, ) - - dataTrieChanges = append(dataTrieChanges, DataTrieChange{ - Type: "write", - Key: trieUpdate.Key, - Val: trieUpdate.Value, - }) } rootHash, err := trie.RootHash() @@ -273,14 +213,6 @@ func (jedtu *journalEntryDataTrieUpdates) Revert() (vmcommon.AccountHandler, err jedtu.account.SetRootHash(rootHash) - stateChange := StateChangeDTO{ - Type: "revert", - MainTrieKey: jedtu.account.AddressBytes(), - Operation: "dataTrieUpdates", - DataTrieChanges: dataTrieChanges, - } - jedtu.stateChangesCollector.AddStateChange(stateChange) - return jedtu.account, nil } @@ -293,12 +225,11 @@ func (jedtu *journalEntryDataTrieUpdates) IsInterfaceNil() bool { type journalEntryDataTrieRemove struct { rootHash []byte obsoleteDataTrieHashes map[string][][]byte - stateChangesCollector StateChangesCollector } // NewJournalEntryDataTrieRemove outputs a new journalEntryDataTrieRemove implementation used to cancel // the eviction of the hashes from the data trie with the given root hash -func NewJournalEntryDataTrieRemove(rootHash []byte, obsoleteDataTrieHashes map[string][][]byte, stateChangesCollector StateChangesCollector) (*journalEntryDataTrieRemove, error) { +func NewJournalEntryDataTrieRemove(rootHash []byte, obsoleteDataTrieHashes map[string][][]byte) (*journalEntryDataTrieRemove, error) { if obsoleteDataTrieHashes == nil { return nil, fmt.Errorf("%w in NewJournalEntryDataTrieRemove", ErrNilMapOfHashes) } @@ -309,7 +240,6 @@ func NewJournalEntryDataTrieRemove(rootHash []byte, obsoleteDataTrieHashes map[s return &journalEntryDataTrieRemove{ rootHash: rootHash, obsoleteDataTrieHashes: obsoleteDataTrieHashes, - stateChangesCollector: stateChangesCollector, }, nil } @@ -317,14 +247,6 @@ func NewJournalEntryDataTrieRemove(rootHash []byte, obsoleteDataTrieHashes map[s func (jedtr *journalEntryDataTrieRemove) Revert() (vmcommon.AccountHandler, error) { delete(jedtr.obsoleteDataTrieHashes, string(jedtr.rootHash)) - stateChange := StateChangeDTO{ - Type: "revert", - MainTrieKey: jedtr.rootHash, - Operation: "removeDataTrie", - DataTrieChanges: nil, - } - jedtr.stateChangesCollector.AddStateChange(stateChange) - return nil, nil } From 7d950ef9afe7ea6824af05a5e90b83f6ef857f2f Mon Sep 17 00:00:00 2001 From: ssd04 Date: Sat, 3 Aug 2024 19:46:57 +0300 Subject: [PATCH 12/55] added meta data for stateChanges dto --- state/accountsDB.go | 5 +- state/accountsDB_test.go | 5 +- .../disabled/disabledStateChangesCollector.go | 9 ++ state/interface.go | 1 + state/stateChangesCollector.go | 145 +++++++++++++++--- state/stateChangesCollector_test.go | 52 ++++--- testscommon/state/accountWrapperMock.go | 2 + testscommon/state/accountsAdapterStub.go | 7 +- 8 files changed, 173 insertions(+), 53 deletions(-) diff --git a/state/accountsDB.go b/state/accountsDB.go index c9f34f24417..5530aa7e428 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -998,7 +998,7 @@ func (adb *AccountsDB) commit() ([]byte, error) { oldRoot := adb.mainTrie.GetOldRoot() // Step 2. commit main trie - err := adb.commitTrie(adb.mainTrie, oldHashes, newHashes) + err = adb.commitTrie(adb.mainTrie, oldHashes, newHashes) if err != nil { return nil, err } @@ -1214,6 +1214,8 @@ func (adb *AccountsDB) journalize(entry JournalEntry) { "entry type", fmt.Sprintf("%T", entry), ) + adb.stateChangesCollector.SetIndexToLastStateChange(len(adb.entries)) + if len(adb.entries) == 1 { adb.stackDebug = debug.Stack() } @@ -1363,7 +1365,6 @@ func collectStats( // SetTxHashForLatestStateChanges will return the state changes since the last call of this method func (adb *AccountsDB) SetTxHashForLatestStateChanges(txHash []byte, tx *transaction.Transaction) { adb.stateChangesCollector.AddTxHashToCollectedStateChanges(txHash, tx) - } func (adb *AccountsDB) ResetStateChangesCollector() []StateChangesForTx { diff --git a/state/accountsDB_test.go b/state/accountsDB_test.go index 5625bea069a..2751290cf5a 100644 --- a/state/accountsDB_test.go +++ b/state/accountsDB_test.go @@ -16,6 +16,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core/atomic" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/core/keyValStorage" + "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/errChan" @@ -456,7 +457,7 @@ func stepCreateAccountWithDataTrieAndCode( _ = userAcc.SaveKeyValue(key1, []byte("value")) _ = userAcc.SaveKeyValue(key2, []byte("value")) _ = adb.SaveAccount(userAcc) - adb.SetTxHashForLatestStateChanges([]byte("accountCreationTxHash")) + adb.SetTxHashForLatestStateChanges([]byte("accountCreationTxHash"), &transaction.Transaction{}) serializedAcc, _ := marshaller.Marshal(userAcc) codeHash := userAcc.GetCodeHash() @@ -505,7 +506,7 @@ func stepMigrateDataTrieValAndChangeCode( userAcc.SetCode(code) _ = userAcc.SaveKeyValue([]byte("key1"), []byte("value1")) _ = adb.SaveAccount(userAcc) - adb.SetTxHashForLatestStateChanges([]byte("accountUpdateTxHash")) + adb.SetTxHashForLatestStateChanges([]byte("accountUpdateTxHash"), &transaction.Transaction{}) stateChangesForTx := adb.ResetStateChangesCollector() assert.Equal(t, 1, len(stateChangesForTx)) diff --git a/state/disabled/disabledStateChangesCollector.go b/state/disabled/disabledStateChangesCollector.go index 33749946560..1d2a34a384e 100644 --- a/state/disabled/disabledStateChangesCollector.go +++ b/state/disabled/disabledStateChangesCollector.go @@ -31,6 +31,15 @@ func (d *disabledStateChangesCollector) Reset() { func (d *disabledStateChangesCollector) AddTxHashToCollectedStateChanges(_ []byte, _ *transaction.Transaction) { } +// SetIndexToLastStateChange - +func (d *disabledStateChangesCollector) SetIndexToLastStateChange(index int) { +} + +// RevertToIndex - +func (d *disabledStateChangesCollector) RevertToIndex(index int) error { + return nil +} + // DumpToJSONFile returns nil func (d *disabledStateChangesCollector) DumpToJSONFile() error { return nil diff --git a/state/interface.go b/state/interface.go index e386d62c791..41e97c1ba71 100644 --- a/state/interface.go +++ b/state/interface.go @@ -360,6 +360,7 @@ type StateChangesCollector interface { GetStateChanges() []StateChangesForTx Reset() AddTxHashToCollectedStateChanges(txHash []byte, tx *transaction.Transaction) + SetIndexToLastStateChange(index int) DumpToJSONFile() error IsInterfaceNil() bool } diff --git a/state/stateChangesCollector.go b/state/stateChangesCollector.go index 0710d7f6afa..54be974a9af 100644 --- a/state/stateChangesCollector.go +++ b/state/stateChangesCollector.go @@ -1,11 +1,13 @@ package state import ( + "bytes" "encoding/json" - "path/filepath" + "errors" + "fmt" + "sync" "github.com/multiversx/mx-chain-core-go/data/transaction" - "github.com/multiversx/mx-chain-storage-go/leveldb" "github.com/multiversx/mx-chain-storage-go/types" ) @@ -21,8 +23,13 @@ type DataTrieChange struct { Val []byte `json:"-"` } +// ErrStateChangesIndexOutOfBounds signals that the state changes index is out of bounds +var ErrStateChangesIndexOutOfBounds = errors.New("state changes index out of bounds") + // StateChangeDTO is used to collect state changes type StateChangeDTO struct { + Index int `json:"-"` + TxHash []byte `json:"-"` Type string `json:"type"` MainTrieKey []byte `json:"mainTrieKey"` MainTrieVal []byte `json:"-"` @@ -45,61 +52,155 @@ type StateChangesForTx struct { StateChanges []StateChangeDTO `json:"stateChanges"` } +type StateChangesTx struct { + TxHash []byte `json:"txHash"` + Tx *transaction.Transaction `json:"tx"` +} + type stateChangesCollector struct { - stateChanges []StateChangeDTO - stateChangesForTx []StateChangesForTx + stateChanges []StateChangeDTO + stateChangesMut sync.RWMutex + + cachedTxs map[string]*transaction.Transaction storer types.Persister } // NewStateChangesCollector creates a new StateChangesCollector func NewStateChangesCollector() *stateChangesCollector { - dbPath := filepath.Join(workingDir, "stateChangesDB", "StateChanges") + // dbPath := filepath.Join(workingDir, "stateChangesDB", "StateChanges") - db, err := leveldb.NewSerialDB(dbPath, 2, 100, 10) - if err != nil { - log.Error("NewStateChangesCollector: failed to create level db") - } + // db, err := leveldb.NewSerialDB(dbPath, 2, 100, 10) + // if err != nil { + // log.Error("NewStateChangesCollector: failed to create level db") + // } return &stateChangesCollector{ - stateChanges: []StateChangeDTO{}, - storer: db, + stateChanges: make([]StateChangeDTO, 0), + cachedTxs: make(map[string]*transaction.Transaction), + storer: nil, } } // AddStateChange adds a new state change to the collector func (scc *stateChangesCollector) AddStateChange(stateChange StateChangeDTO) { + scc.stateChangesMut.Lock() scc.stateChanges = append(scc.stateChanges, stateChange) + scc.stateChangesMut.Unlock() } // GetStateChanges returns the accumulated state changes func (scc *stateChangesCollector) GetStateChanges() []StateChangesForTx { - if len(scc.stateChanges) > 0 { - scc.AddTxHashToCollectedStateChanges([]byte{}, nil) + stateChangesForTx, err := scc.getStateChangesForTxs() + if err != nil { + log.Warn("failed to get state changes for tx", "error", err) + return make([]StateChangesForTx, 0) } - return scc.stateChangesForTx + return stateChangesForTx +} + +func (scc *stateChangesCollector) getStateChangesForTxs() ([]StateChangesForTx, error) { + scc.stateChangesMut.Lock() + defer scc.stateChangesMut.Unlock() + + stateChangesForTxs := make([]StateChangesForTx, 0) + + for i := 0; i < len(scc.stateChanges); i++ { + txHash := scc.stateChanges[i].TxHash + + if len(txHash) == 0 { + log.Warn("empty tx hash, state change event not associated to a transaction") + break + } + + log.Warn("txHash", "txHash", string(txHash)) + + cachedTx, txOk := scc.cachedTxs[string(txHash)] + if !txOk { + return nil, fmt.Errorf("did not find tx in cache") + } + + innerStateChangesForTx := make([]StateChangeDTO, 0) + for j := i; j < len(scc.stateChanges); j++ { + txHash2 := scc.stateChanges[j].TxHash + if !bytes.Equal(txHash, txHash2) { + i = j + break + } + + innerStateChangesForTx = append(innerStateChangesForTx, scc.stateChanges[j]) + i = j + } + + stateChangesForTx := StateChangesForTx{ + TxHash: txHash, + Tx: cachedTx, + StateChanges: innerStateChangesForTx, + } + stateChangesForTxs = append(stateChangesForTxs, stateChangesForTx) + } + + return stateChangesForTxs, nil } // Reset resets the state changes collector func (scc *stateChangesCollector) Reset() { + scc.stateChangesMut.Lock() + defer scc.stateChangesMut.Unlock() + scc.stateChanges = make([]StateChangeDTO, 0) - scc.stateChangesForTx = make([]StateChangesForTx, 0) + scc.cachedTxs = make(map[string]*transaction.Transaction) } func (scc *stateChangesCollector) AddTxHashToCollectedStateChanges(txHash []byte, tx *transaction.Transaction) { - stateChangesForTx := StateChangesForTx{ - TxHash: txHash, - Tx: tx, - StateChanges: scc.stateChanges, + scc.stateChangesMut.Lock() + defer scc.stateChangesMut.Unlock() + + scc.cachedTxs[string(txHash)] = tx + + for i := len(scc.stateChanges) - 1; i >= 0; i-- { + if len(scc.stateChanges[i].TxHash) > 0 { + break + } + + scc.stateChanges[i].TxHash = txHash } +} - scc.stateChanges = make([]StateChangeDTO, 0) - scc.stateChangesForTx = append(scc.stateChangesForTx, stateChangesForTx) +func (scc *stateChangesCollector) SetIndexToLastStateChange(index int) { + scc.stateChangesMut.Lock() + defer scc.stateChangesMut.Unlock() + + scc.stateChanges[len(scc.stateChanges)-1].Index = index +} + +func (scc *stateChangesCollector) RevertToIndex(index int) error { + scc.stateChangesMut.Lock() + defer scc.stateChangesMut.Unlock() + + if index > len(scc.stateChanges) || index < 0 { + return ErrStateChangesIndexOutOfBounds + } + + if index == 0 { + scc.Reset() + return nil + } + + // this should be improved since not all state changes indexer are set + scc.stateChanges = scc.stateChanges[:index] + + return nil } func (scc *stateChangesCollector) DumpToJSONFile() error { - for _, stateChange := range scc.stateChangesForTx { + stateChangesForTx, err := scc.getStateChangesForTxs() + if err != nil { + return err + } + + for _, stateChange := range stateChangesForTx { marshalledData, err := json.Marshal(stateChange) if err != nil { return err diff --git a/state/stateChangesCollector_test.go b/state/stateChangesCollector_test.go index 6a192601f2e..e8469f8a479 100644 --- a/state/stateChangesCollector_test.go +++ b/state/stateChangesCollector_test.go @@ -1,10 +1,12 @@ package state import ( - "github.com/stretchr/testify/assert" "strconv" "testing" + "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) @@ -36,7 +38,7 @@ func TestStateChangesCollector_GetStateChanges(t *testing.T) { scc := NewStateChangesCollector() assert.Equal(t, 0, len(scc.stateChanges)) - assert.Equal(t, 0, len(scc.stateChangesForTx)) + assert.Equal(t, 0, len(scc.GetStateChanges())) numStateChanges := 10 for i := 0; i < numStateChanges; i++ { @@ -45,12 +47,12 @@ func TestStateChangesCollector_GetStateChanges(t *testing.T) { }) } assert.Equal(t, numStateChanges, len(scc.stateChanges)) - assert.Equal(t, 0, len(scc.stateChangesForTx)) - scc.AddTxHashToCollectedStateChanges([]byte("txHash")) - assert.Equal(t, 0, len(scc.stateChanges)) - assert.Equal(t, 1, len(scc.stateChangesForTx)) - assert.Equal(t, []byte("txHash"), scc.stateChangesForTx[0].TxHash) - assert.Equal(t, numStateChanges, len(scc.stateChangesForTx[0].StateChanges)) + assert.Equal(t, 0, len(scc.GetStateChanges())) + scc.AddTxHashToCollectedStateChanges([]byte("txHash"), &transaction.Transaction{}) + assert.Equal(t, numStateChanges, len(scc.stateChanges)) + assert.Equal(t, 1, len(scc.GetStateChanges())) + assert.Equal(t, []byte("txHash"), scc.GetStateChanges()[0].TxHash) + assert.Equal(t, numStateChanges, len(scc.GetStateChanges()[0].StateChanges)) stateChangesForTx := scc.GetStateChanges() assert.Equal(t, 1, len(stateChangesForTx)) @@ -65,7 +67,7 @@ func TestStateChangesCollector_GetStateChanges(t *testing.T) { scc := NewStateChangesCollector() assert.Equal(t, 0, len(scc.stateChanges)) - assert.Equal(t, 0, len(scc.stateChangesForTx)) + assert.Equal(t, 0, len(scc.GetStateChanges())) numStateChanges := 10 for i := 0; i < numStateChanges; i++ { @@ -74,14 +76,14 @@ func TestStateChangesCollector_GetStateChanges(t *testing.T) { }) } assert.Equal(t, numStateChanges, len(scc.stateChanges)) - assert.Equal(t, 0, len(scc.stateChangesForTx)) + assert.Equal(t, 0, len(scc.GetStateChanges())) stateChangesForTx := scc.GetStateChanges() - assert.Equal(t, 1, len(stateChangesForTx)) - assert.Equal(t, []byte{}, stateChangesForTx[0].TxHash) - for i := 0; i < len(stateChangesForTx[0].StateChanges); i++ { - assert.Equal(t, []byte(strconv.Itoa(i)), stateChangesForTx[0].StateChanges[i].MainTrieKey) - } + assert.Equal(t, 0, len(stateChangesForTx)) + // assert.Equal(t, []byte{}, stateChangesForTx[0].TxHash) + // for i := 0; i < len(stateChangesForTx[0].StateChanges); i++ { + // assert.Equal(t, []byte(strconv.Itoa(i)), stateChangesForTx[0].StateChanges[i].MainTrieKey) + // } }) } @@ -90,7 +92,7 @@ func TestStateChangesCollector_AddTxHashToCollectedStateChanges(t *testing.T) { scc := NewStateChangesCollector() assert.Equal(t, 0, len(scc.stateChanges)) - assert.Equal(t, 0, len(scc.stateChangesForTx)) + assert.Equal(t, 0, len(scc.GetStateChanges())) stateChange := StateChangeDTO{ MainTrieKey: []byte("mainTrieKey"), @@ -100,10 +102,10 @@ func TestStateChangesCollector_AddTxHashToCollectedStateChanges(t *testing.T) { scc.AddStateChange(stateChange) assert.Equal(t, 1, len(scc.stateChanges)) - assert.Equal(t, 0, len(scc.stateChangesForTx)) - scc.AddTxHashToCollectedStateChanges([]byte("txHash")) - assert.Equal(t, 0, len(scc.stateChanges)) - assert.Equal(t, 1, len(scc.stateChangesForTx)) + assert.Equal(t, 0, len(scc.GetStateChanges())) + scc.AddTxHashToCollectedStateChanges([]byte("txHash"), &transaction.Transaction{}) + assert.Equal(t, 1, len(scc.stateChanges)) + assert.Equal(t, 1, len(scc.GetStateChanges())) stateChangesForTx := scc.GetStateChanges() assert.Equal(t, 1, len(stateChangesForTx)) @@ -124,14 +126,16 @@ func TestStateChangesCollector_Reset(t *testing.T) { for i := 0; i < numStateChanges; i++ { scc.AddStateChange(StateChangeDTO{}) } - scc.AddTxHashToCollectedStateChanges([]byte("txHash")) + scc.AddTxHashToCollectedStateChanges([]byte("txHash"), &transaction.Transaction{}) for i := numStateChanges; i < numStateChanges*2; i++ { scc.AddStateChange(StateChangeDTO{}) } - assert.Equal(t, numStateChanges, len(scc.stateChanges)) - assert.Equal(t, 1, len(scc.stateChangesForTx)) + assert.Equal(t, numStateChanges*2, len(scc.stateChanges)) + + assert.Equal(t, 1, len(scc.GetStateChanges())) scc.Reset() assert.Equal(t, 0, len(scc.stateChanges)) - assert.Equal(t, 0, len(scc.stateChangesForTx)) + + assert.Equal(t, 0, len(scc.GetStateChanges())) } diff --git a/testscommon/state/accountWrapperMock.go b/testscommon/state/accountWrapperMock.go index 75c7ab97517..7dcabfec30d 100644 --- a/testscommon/state/accountWrapperMock.go +++ b/testscommon/state/accountWrapperMock.go @@ -9,6 +9,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/state" + "github.com/multiversx/mx-chain-go/state/disabled" "github.com/multiversx/mx-chain-go/state/trackableDataTrie" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" @@ -47,6 +48,7 @@ func NewAccountWrapMock(adr []byte) *AccountWrapMock { &hashingMocks.HasherMock{}, &marshallerMock.MarshalizerMock{}, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + disabled.NewDisabledStateChangesCollector(), ) return &AccountWrapMock{ diff --git a/testscommon/state/accountsAdapterStub.go b/testscommon/state/accountsAdapterStub.go index b7dcbbb484a..85683ad7fe4 100644 --- a/testscommon/state/accountsAdapterStub.go +++ b/testscommon/state/accountsAdapterStub.go @@ -4,6 +4,7 @@ import ( "context" "errors" + "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/state" vmcommon "github.com/multiversx/mx-chain-vm-common-go" @@ -39,7 +40,7 @@ type AccountsStub struct { CloseCalled func() error SetSyncerCalled func(syncer state.AccountsDBSyncer) error StartSnapshotIfNeededCalled func() error - SetTxHashForLatestStateChangesCalled func(txHash []byte) + SetTxHashForLatestStateChangesCalled func(txHash []byte, tx *transaction.Transaction) ResetStateChangesCollectorCalled func() []state.StateChangesForTx } @@ -260,9 +261,9 @@ func (as *AccountsStub) GetCodeWithBlockInfo(codeHash []byte, options common.Roo } // SetTxHashForLatestStateChanges - -func (as *AccountsStub) SetTxHashForLatestStateChanges(txHash []byte) { +func (as *AccountsStub) SetTxHashForLatestStateChanges(txHash []byte, tx *transaction.Transaction) { if as.SetTxHashForLatestStateChangesCalled != nil { - as.SetTxHashForLatestStateChangesCalled(txHash) + as.SetTxHashForLatestStateChangesCalled(txHash, tx) } } From 485e094a9d2127fcb768e8d53d4f822654cb3997 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Mon, 5 Aug 2024 13:18:33 +0300 Subject: [PATCH 13/55] added more unit tests --- state/accountsDB.go | 2 + state/stateChangesCollector.go | 14 ++++--- state/stateChangesCollector_test.go | 59 +++++++++++++++++++++++++++-- 3 files changed, 66 insertions(+), 9 deletions(-) diff --git a/state/accountsDB.go b/state/accountsDB.go index 5530aa7e428..86bfd0578fd 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -1102,6 +1102,8 @@ func (adb *AccountsDB) recreateTrie(options common.RootHashHolder) error { adb.obsoleteDataTrieHashes = make(map[string][][]byte) adb.dataTries.Reset() adb.entries = make([]JournalEntry, 0) + adb.stateChangesCollector.Reset() + newTrie, err := adb.mainTrie.RecreateFromEpoch(options) if err != nil { return err diff --git a/state/stateChangesCollector.go b/state/stateChangesCollector.go index 54be974a9af..8b73dcdf622 100644 --- a/state/stateChangesCollector.go +++ b/state/stateChangesCollector.go @@ -176,9 +176,6 @@ func (scc *stateChangesCollector) SetIndexToLastStateChange(index int) { } func (scc *stateChangesCollector) RevertToIndex(index int) error { - scc.stateChangesMut.Lock() - defer scc.stateChangesMut.Unlock() - if index > len(scc.stateChanges) || index < 0 { return ErrStateChangesIndexOutOfBounds } @@ -188,8 +185,15 @@ func (scc *stateChangesCollector) RevertToIndex(index int) error { return nil } - // this should be improved since not all state changes indexer are set - scc.stateChanges = scc.stateChanges[:index] + scc.stateChangesMut.Lock() + defer scc.stateChangesMut.Unlock() + + for i := len(scc.stateChanges) - 1; i >= 0; i-- { + if scc.stateChanges[i].Index == index { + scc.stateChanges = scc.stateChanges[:i] + break + } + } return nil } diff --git a/state/stateChangesCollector_test.go b/state/stateChangesCollector_test.go index e8469f8a479..eaadbaec056 100644 --- a/state/stateChangesCollector_test.go +++ b/state/stateChangesCollector_test.go @@ -1,6 +1,7 @@ package state import ( + "fmt" "strconv" "testing" @@ -80,10 +81,6 @@ func TestStateChangesCollector_GetStateChanges(t *testing.T) { stateChangesForTx := scc.GetStateChanges() assert.Equal(t, 0, len(stateChangesForTx)) - // assert.Equal(t, []byte{}, stateChangesForTx[0].TxHash) - // for i := 0; i < len(stateChangesForTx[0].StateChanges); i++ { - // assert.Equal(t, []byte(strconv.Itoa(i)), stateChangesForTx[0].StateChanges[i].MainTrieKey) - // } }) } @@ -116,6 +113,60 @@ func TestStateChangesCollector_AddTxHashToCollectedStateChanges(t *testing.T) { assert.Equal(t, 1, len(stateChangesForTx[0].StateChanges[0].DataTrieChanges)) } +func TestStateChangesCollector_RevertToIndex_FailIfWrongIndex(t *testing.T) { + t.Parallel() + + scc := NewStateChangesCollector() + numStateChanges := len(scc.stateChanges) + + err := scc.RevertToIndex(-1) + require.Equal(t, ErrStateChangesIndexOutOfBounds, err) + + err = scc.RevertToIndex(numStateChanges + 1) + require.Equal(t, ErrStateChangesIndexOutOfBounds, err) +} + +func TestStateChangesCollector_RevertToIndex(t *testing.T) { + t.Parallel() + + scc := NewStateChangesCollector() + + numStateChanges := 10 + for i := 0; i < numStateChanges; i++ { + scc.AddStateChange(StateChangeDTO{}) + scc.SetIndexToLastStateChange(i) + } + scc.AddTxHashToCollectedStateChanges([]byte("txHash1"), &transaction.Transaction{}) + + for i := numStateChanges; i < numStateChanges*2; i++ { + scc.AddStateChange(StateChangeDTO{}) + scc.AddTxHashToCollectedStateChanges([]byte("txHash"+fmt.Sprintf("%d", i)), &transaction.Transaction{}) + } + scc.SetIndexToLastStateChange(numStateChanges) + + assert.Equal(t, numStateChanges*2, len(scc.stateChanges)) + + err := scc.RevertToIndex(numStateChanges) + require.Nil(t, err) + assert.Equal(t, numStateChanges*2-1, len(scc.stateChanges)) + + err = scc.RevertToIndex(numStateChanges - 1) + require.Nil(t, err) + assert.Equal(t, numStateChanges-1, len(scc.stateChanges)) + + err = scc.RevertToIndex(numStateChanges / 2) + require.Nil(t, err) + assert.Equal(t, numStateChanges/2, len(scc.stateChanges)) + + err = scc.RevertToIndex(1) + require.Nil(t, err) + assert.Equal(t, 1, len(scc.stateChanges)) + + err = scc.RevertToIndex(0) + require.Nil(t, err) + assert.Equal(t, 0, len(scc.stateChanges)) +} + func TestStateChangesCollector_Reset(t *testing.T) { t.Parallel() From 867bffe19a16587ae61e7a8dcf97d3ea49d67924 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Tue, 6 Aug 2024 14:45:53 +0300 Subject: [PATCH 14/55] fix index out of bounds --- state/accountsDB.go | 5 +++ .../disabled/disabledStateChangesCollector.go | 3 +- state/interface.go | 3 +- state/stateChangesCollector.go | 8 +++- state/stateChangesCollector_test.go | 42 +++++++++++++++++++ 5 files changed, 58 insertions(+), 3 deletions(-) diff --git a/state/accountsDB.go b/state/accountsDB.go index 86bfd0578fd..db02c2aed4d 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -898,6 +898,11 @@ func (adb *AccountsDB) RevertToSnapshot(snapshot int) error { adb.entries = adb.entries[:snapshot] + err := adb.stateChangesCollector.RevertToIndex(snapshot) + if err != nil { + return err + } + return nil } diff --git a/state/disabled/disabledStateChangesCollector.go b/state/disabled/disabledStateChangesCollector.go index 1d2a34a384e..77a08e73151 100644 --- a/state/disabled/disabledStateChangesCollector.go +++ b/state/disabled/disabledStateChangesCollector.go @@ -32,7 +32,8 @@ func (d *disabledStateChangesCollector) AddTxHashToCollectedStateChanges(_ []byt } // SetIndexToLastStateChange - -func (d *disabledStateChangesCollector) SetIndexToLastStateChange(index int) { +func (d *disabledStateChangesCollector) SetIndexToLastStateChange(index int) error { + return nil } // RevertToIndex - diff --git a/state/interface.go b/state/interface.go index 41e97c1ba71..e9785ab5794 100644 --- a/state/interface.go +++ b/state/interface.go @@ -360,7 +360,8 @@ type StateChangesCollector interface { GetStateChanges() []StateChangesForTx Reset() AddTxHashToCollectedStateChanges(txHash []byte, tx *transaction.Transaction) - SetIndexToLastStateChange(index int) + SetIndexToLastStateChange(index int) error + RevertToIndex(index int) error DumpToJSONFile() error IsInterfaceNil() bool } diff --git a/state/stateChangesCollector.go b/state/stateChangesCollector.go index 8b73dcdf622..f42e37bd280 100644 --- a/state/stateChangesCollector.go +++ b/state/stateChangesCollector.go @@ -168,11 +168,17 @@ func (scc *stateChangesCollector) AddTxHashToCollectedStateChanges(txHash []byte } } -func (scc *stateChangesCollector) SetIndexToLastStateChange(index int) { +func (scc *stateChangesCollector) SetIndexToLastStateChange(index int) error { + if index > len(scc.stateChanges) || index < 0 { + return ErrStateChangesIndexOutOfBounds + } + scc.stateChangesMut.Lock() defer scc.stateChangesMut.Unlock() scc.stateChanges[len(scc.stateChanges)-1].Index = index + + return nil } func (scc *stateChangesCollector) RevertToIndex(index int) error { diff --git a/state/stateChangesCollector_test.go b/state/stateChangesCollector_test.go index eaadbaec056..6853973bfcf 100644 --- a/state/stateChangesCollector_test.go +++ b/state/stateChangesCollector_test.go @@ -91,6 +91,8 @@ func TestStateChangesCollector_AddTxHashToCollectedStateChanges(t *testing.T) { assert.Equal(t, 0, len(scc.stateChanges)) assert.Equal(t, 0, len(scc.GetStateChanges())) + scc.AddTxHashToCollectedStateChanges([]byte("txHash0"), &transaction.Transaction{}) + stateChange := StateChangeDTO{ MainTrieKey: []byte("mainTrieKey"), MainTrieVal: []byte("mainTrieVal"), @@ -167,6 +169,46 @@ func TestStateChangesCollector_RevertToIndex(t *testing.T) { assert.Equal(t, 0, len(scc.stateChanges)) } +func TestStateChangesCollector_SetIndexToLastStateChange(t *testing.T) { + t.Parallel() + + t.Run("should fail if valid index", func(t *testing.T) { + t.Parallel() + + scc := NewStateChangesCollector() + + err := scc.SetIndexToLastStateChange(-1) + require.Equal(t, ErrStateChangesIndexOutOfBounds, err) + + numStateChanges := len(scc.stateChanges) + err = scc.SetIndexToLastStateChange(numStateChanges + 1) + require.Equal(t, ErrStateChangesIndexOutOfBounds, err) + }) + + t.Run("should work", func(t *testing.T) { + t.Parallel() + + scc := NewStateChangesCollector() + + numStateChanges := 10 + for i := 0; i < numStateChanges; i++ { + scc.AddStateChange(StateChangeDTO{}) + err := scc.SetIndexToLastStateChange(i) + require.Nil(t, err) + } + scc.AddTxHashToCollectedStateChanges([]byte("txHash1"), &transaction.Transaction{}) + + for i := numStateChanges; i < numStateChanges*2; i++ { + scc.AddStateChange(StateChangeDTO{}) + scc.AddTxHashToCollectedStateChanges([]byte("txHash"+fmt.Sprintf("%d", i)), &transaction.Transaction{}) + } + err := scc.SetIndexToLastStateChange(numStateChanges) + require.Nil(t, err) + + assert.Equal(t, numStateChanges*2, len(scc.stateChanges)) + }) +} + func TestStateChangesCollector_Reset(t *testing.T) { t.Parallel() From b87e3f0ade339a2953b19ff5540ee4ab7ef49161 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Mon, 26 Aug 2024 15:07:25 +0300 Subject: [PATCH 15/55] update separate components for state changes data analysis --- config/config.go | 15 +- .../disabled/disabledAccountsAdapter.go | 5 - factory/state/stateComponents.go | 40 ++- state/accountsDB.go | 102 +------- state/accountsDBApi.go | 5 - state/accountsDBApiWithHistory.go | 5 - .../disabled/disabledStateChangesCollector.go | 15 +- state/interface.go | 12 +- state/stateChanges/dataAnalysisCollector.go | 211 +++++++++++++++ .../dataAnalysisCollector_test.go | 1 + state/stateChanges/export_test.go | 7 + .../writeCollector.go} | 138 +++++----- state/stateChanges/writeCollector_test.go | 241 ++++++++++++++++++ state/trackableDataTrie/trackableDataTrie.go | 21 +- testscommon/trie/dataTrieTrackerStub.go | 8 +- 15 files changed, 609 insertions(+), 217 deletions(-) create mode 100644 state/stateChanges/dataAnalysisCollector.go create mode 100644 state/stateChanges/dataAnalysisCollector_test.go create mode 100644 state/stateChanges/export_test.go rename state/{stateChangesCollector.go => stateChanges/writeCollector.go} (58%) create mode 100644 state/stateChanges/writeCollector_test.go diff --git a/config/config.go b/config/config.go index 3b36baba8b8..e4b4ef1e6c0 100644 --- a/config/config.go +++ b/config/config.go @@ -306,13 +306,14 @@ type FacadeConfig struct { // StateTriesConfig will hold information about state tries type StateTriesConfig struct { - SnapshotsEnabled bool - AccountsStatePruningEnabled bool - PeerStatePruningEnabled bool - CollectStateChangesEnabled bool - MaxStateTrieLevelInMemory uint - MaxPeerTrieLevelInMemory uint - StateStatisticsEnabled bool + SnapshotsEnabled bool + AccountsStatePruningEnabled bool + PeerStatePruningEnabled bool + CollectStateChangesEnabled bool + CollectStateChangesWithReadEnabled bool + MaxStateTrieLevelInMemory uint + MaxPeerTrieLevelInMemory uint + StateStatisticsEnabled bool } // TrieStorageManagerConfig will hold config information about trie storage manager diff --git a/epochStart/bootstrap/disabled/disabledAccountsAdapter.go b/epochStart/bootstrap/disabled/disabledAccountsAdapter.go index 1310d59ce6e..197691ce097 100644 --- a/epochStart/bootstrap/disabled/disabledAccountsAdapter.go +++ b/epochStart/bootstrap/disabled/disabledAccountsAdapter.go @@ -138,11 +138,6 @@ func (a *accountsAdapter) GetStackDebugFirstEntry() []byte { func (a *accountsAdapter) SetTxHashForLatestStateChanges(_ []byte, _ *transaction.Transaction) { } -// ResetStateChangesCollector - -func (a *accountsAdapter) ResetStateChangesCollector() []state.StateChangesForTx { - return nil -} - // Close - func (a *accountsAdapter) Close() error { return nil diff --git a/factory/state/stateComponents.go b/factory/state/stateComponents.go index af9e2466507..20ea905e0f1 100644 --- a/factory/state/stateComponents.go +++ b/factory/state/stateComponents.go @@ -15,10 +15,13 @@ import ( factoryState "github.com/multiversx/mx-chain-go/state/factory" "github.com/multiversx/mx-chain-go/state/iteratorChannelsProvider" "github.com/multiversx/mx-chain-go/state/lastSnapshotMarker" + "github.com/multiversx/mx-chain-go/state/stateChanges" "github.com/multiversx/mx-chain-go/state/stateMetrics" "github.com/multiversx/mx-chain-go/state/storagePruningManager" "github.com/multiversx/mx-chain-go/state/storagePruningManager/evictionWaitingList" "github.com/multiversx/mx-chain-go/state/syncer" + storageFactory "github.com/multiversx/mx-chain-go/storage/factory" + "github.com/multiversx/mx-chain-go/storage/storageunit" trieFactory "github.com/multiversx/mx-chain-go/trie/factory" ) @@ -92,9 +95,9 @@ func (scf *stateComponentsFactory) Create() (*stateComponents, error) { return nil, err } - stateChangesCollector := disabled.NewDisabledStateChangesCollector() - if scf.config.StateTriesConfig.CollectStateChangesEnabled { - stateChangesCollector = state.NewStateChangesCollector() + stateChangesCollector, err := scf.createStateChangesCollector() + if err != nil { + return nil, err } accountsAdapter, accountsAdapterAPI, accountsRepository, err := scf.createAccountsAdapters(triesContainer, stateChangesCollector) @@ -119,6 +122,37 @@ func (scf *stateComponentsFactory) Create() (*stateComponents, error) { }, nil } +func (scf *stateComponentsFactory) createStateChangesCollector() (state.StateChangesCollector, error) { + if !scf.config.StateTriesConfig.CollectStateChangesEnabled { + return disabled.NewDisabledStateChangesCollector(), nil + } + + if !scf.config.StateTriesConfig.CollectStateChangesWithReadEnabled { + return stateChanges.NewStateChangesCollector(), nil + } + + dbConfig := config.DBConfig{ + FilePath: "stateChanges", + Type: "LvlDBSerial", + BatchDelaySeconds: 2, + MaxBatchSize: 100, + MaxOpenFiles: 10, + } + + dbConfigHandler := storageFactory.NewDBConfigHandler(dbConfig) + persisterFactory, err := storageFactory.NewPersisterFactory(dbConfigHandler) + if err != nil { + return nil, err + } + + db, err := storageunit.NewDB(persisterFactory, dbConfig.FilePath) + if err != nil { + return nil, fmt.Errorf("%w while creating the db for the trie nodes", err) + } + + return stateChanges.NewDataAnalysisStateChangesCollector(db) +} + func (scf *stateComponentsFactory) createSnapshotManager( accountFactory state.AccountFactory, stateMetrics state.StateMetrics, diff --git a/state/accountsDB.go b/state/accountsDB.go index db02c2aed4d..d3fe05323f1 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -20,6 +20,7 @@ import ( "github.com/multiversx/mx-chain-go/common/errChan" "github.com/multiversx/mx-chain-go/common/holders" "github.com/multiversx/mx-chain-go/state/parsers" + "github.com/multiversx/mx-chain-go/state/stateChanges" "github.com/multiversx/mx-chain-go/trie/keyBuilder" "github.com/multiversx/mx-chain-go/trie/statistics" logger "github.com/multiversx/mx-chain-logger-go" @@ -208,7 +209,7 @@ func (adb *AccountsDB) GetCode(codeHash []byte) []byte { return nil } - stateChange := StateChangeDTO{ + stateChange := &stateChanges.StateChangeDTO{ Type: "read", MainTrieKey: codeHash, MainTrieVal: val, @@ -283,7 +284,7 @@ func (adb *AccountsDB) SaveAccount(account vmcommon.AccountHandler) error { return err } - stateChange := &StateChangeDTO{ + stateChange := &stateChanges.StateChangeDTO{ Type: "write", MainTrieKey: account.AddressBytes(), MainTrieVal: marshalledAccount, @@ -291,68 +292,17 @@ func (adb *AccountsDB) SaveAccount(account vmcommon.AccountHandler) error { Operation: "saveAccount", } - checkAccountChanges(oldAccount, account, stateChange) - - adb.stateChangesCollector.AddStateChange(*stateChange) + adb.stateChangesCollector.AddSaveAccountStateChange(oldAccount, account, stateChange) return err } -func checkAccountChanges(oldAcc, newAcc vmcommon.AccountHandler, stateChange *StateChangeDTO) { - baseNewAcc, newAccOk := newAcc.(UserAccountHandler) - if !newAccOk { - log.Trace("checkAccountChanges: Not able to cast new acc") - return - } - baseOldAccount, oldAccOk := oldAcc.(UserAccountHandler) - if !oldAccOk { - log.Trace("checkAccountChanges: Not able to cast old acc") - return - } - - if baseNewAcc.GetNonce() != baseOldAccount.GetNonce() { - stateChange.Nonce = true - } - - if baseNewAcc.GetBalance().Uint64() != baseOldAccount.GetBalance().Uint64() { - stateChange.Balance = true - } - - if !bytes.Equal(baseNewAcc.GetCodeHash(), baseOldAccount.GetCodeHash()) { - stateChange.CodeHash = true - } - - if !bytes.Equal(baseNewAcc.GetRootHash(), baseOldAccount.GetRootHash()) { - stateChange.RootHash = true - log.Trace("checkAccountChanges: rootHash changed", "address", baseNewAcc.AddressBytes()) - } - log.Trace("checkAccountChanges: rootHash NOT changed", "address", baseNewAcc.AddressBytes()) - - if !bytes.Equal(baseNewAcc.GetDeveloperReward().Bytes(), baseOldAccount.GetDeveloperReward().Bytes()) { - stateChange.DeveloperReward = true - } - - if !bytes.Equal(baseNewAcc.GetOwnerAddress(), baseOldAccount.GetOwnerAddress()) { - stateChange.OwnerAddress = true - } - - if !bytes.Equal(baseNewAcc.GetUserName(), baseOldAccount.GetUserName()) { - stateChange.UserName = true - } - - if !bytes.Equal(baseNewAcc.GetCodeMetadata(), baseOldAccount.GetCodeMetadata()) { - stateChange.CodeMetadata = true - } - - log.Trace("checkAccountChanges: checked all", "address", baseNewAcc.AddressBytes()) -} - -func (adb *AccountsDB) saveCodeAndDataTrie(oldAcc, newAcc vmcommon.AccountHandler) ([]DataTrieChange, error) { +func (adb *AccountsDB) saveCodeAndDataTrie(oldAcc, newAcc vmcommon.AccountHandler) ([]stateChanges.DataTrieChange, error) { baseNewAcc, newAccOk := newAcc.(baseAccountHandler) baseOldAccount, _ := oldAcc.(baseAccountHandler) if !newAccOk { - return make([]DataTrieChange, 0), nil + return make([]stateChanges.DataTrieChange, 0), nil } newValues, err := adb.saveDataTrie(baseNewAcc) @@ -421,7 +371,7 @@ func (adb *AccountsDB) updateOldCodeEntry(oldCodeHash []byte) (*CodeEntry, error return nil, nil } - stateChange := StateChangeDTO{ + stateChange := &stateChanges.StateChangeDTO{ Type: "read", MainTrieKey: oldCodeHash, MainTrieVal: nil, @@ -441,7 +391,7 @@ func (adb *AccountsDB) updateOldCodeEntry(oldCodeHash []byte) (*CodeEntry, error return nil, err } - stateChange := StateChangeDTO{ + stateChange := &stateChanges.StateChangeDTO{ Type: "write", MainTrieKey: oldCodeHash, MainTrieVal: nil, @@ -459,7 +409,7 @@ func (adb *AccountsDB) updateOldCodeEntry(oldCodeHash []byte) (*CodeEntry, error return nil, err } - stateChange = StateChangeDTO{ + stateChange = &stateChanges.StateChangeDTO{ Type: "write", MainTrieKey: oldCodeHash, MainTrieVal: codeEntryBytes, @@ -493,7 +443,7 @@ func (adb *AccountsDB) updateNewCodeEntry(newCodeHash []byte, newCode []byte) er return err } - stateChange := StateChangeDTO{ + stateChange := &stateChanges.StateChangeDTO{ Type: "write", MainTrieKey: newCodeHash, MainTrieVal: codeEntryBytes, @@ -566,7 +516,7 @@ func (adb *AccountsDB) loadDataTrieConcurrentSafe(accountHandler baseAccountHand // SaveDataTrie is used to save the data trie (not committing it) and to recompute the new Root value // If data is not dirtied, method will not create its JournalEntries to keep track of data modification -func (adb *AccountsDB) saveDataTrie(accountHandler baseAccountHandler) ([]DataTrieChange, error) { +func (adb *AccountsDB) saveDataTrie(accountHandler baseAccountHandler) ([]stateChanges.DataTrieChange, error) { newValues, oldValues, err := accountHandler.SaveDirtyData(adb.mainTrie) if err != nil { return nil, err @@ -700,13 +650,13 @@ func (adb *AccountsDB) removeDataTrie(baseAcc baseAccountHandler) error { } adb.journalize(entry) - stateChange := &StateChangeDTO{ + stateChange := &stateChanges.StateChangeDTO{ Type: "write", MainTrieKey: baseAcc.AddressBytes(), Operation: "removeDataTrie", DataTrieChanges: nil, } - adb.stateChangesCollector.AddStateChange(*stateChange) + adb.stateChangesCollector.AddStateChange(stateChange) return nil } @@ -955,19 +905,6 @@ func (adb *AccountsDB) Commit() ([]byte, error) { return adb.commit() } -func printStateChanges(stateChanges []StateChangesForTx) { - for _, stateChange := range stateChanges { - if stateChange.TxHash != nil { - fmt.Println(hex.EncodeToString(stateChange.TxHash)) - } - - for _, st := range stateChange.StateChanges { - fmt.Println(string(st.MainTrieKey)) - fmt.Println(hex.EncodeToString(st.MainTrieVal)) - } - } -} - func (adb *AccountsDB) commit() ([]byte, error) { log.Trace("accountsDB.Commit started") adb.entries = make([]JournalEntry, 0) @@ -977,16 +914,12 @@ func (adb *AccountsDB) commit() ([]byte, error) { // adb.stateChangesCollector.Reset() // } - stateChanges := adb.stateChangesCollector.GetStateChanges() - printStateChanges(stateChanges) - err := adb.stateChangesCollector.DumpToJSONFile() + err := adb.stateChangesCollector.Publish() if err != nil { log.Warn("failed to dump state changes to json file", "error", err) } - log.Debug("SetTxHashForLatestStateChanges: dump to json file") adb.stateChangesCollector.Reset() - log.Debug("SetTxHashForLatestStateChanges: reset") oldHashes := make(common.ModifiedHashes) newHashes := make(common.ModifiedHashes) @@ -1374,13 +1307,6 @@ func (adb *AccountsDB) SetTxHashForLatestStateChanges(txHash []byte, tx *transac adb.stateChangesCollector.AddTxHashToCollectedStateChanges(txHash, tx) } -func (adb *AccountsDB) ResetStateChangesCollector() []StateChangesForTx { - stateChanges := adb.stateChangesCollector.GetStateChanges() - adb.stateChangesCollector.Reset() - - return stateChanges -} - // IsSnapshotInProgress returns true if there is a snapshot in progress func (adb *AccountsDB) IsSnapshotInProgress() bool { return adb.snapshotsManger.IsSnapshotInProgress() diff --git a/state/accountsDBApi.go b/state/accountsDBApi.go index 207504384aa..a9d42146a64 100644 --- a/state/accountsDBApi.go +++ b/state/accountsDBApi.go @@ -243,11 +243,6 @@ func (accountsDB *accountsDBApi) SetTxHashForLatestStateChanges(txHash []byte, t accountsDB.innerAccountsAdapter.SetTxHashForLatestStateChanges(txHash, tx) } -// ResetStateChangesCollector returns nil -func (accountsDB *accountsDBApi) ResetStateChangesCollector() []StateChangesForTx { - return nil -} - // Close will handle the closing of the underlying components func (accountsDB *accountsDBApi) Close() error { return accountsDB.innerAccountsAdapter.Close() diff --git a/state/accountsDBApiWithHistory.go b/state/accountsDBApiWithHistory.go index 82a19887f3b..96d43a9cd31 100644 --- a/state/accountsDBApiWithHistory.go +++ b/state/accountsDBApiWithHistory.go @@ -145,11 +145,6 @@ func (accountsDB *accountsDBApiWithHistory) GetStackDebugFirstEntry() []byte { func (accountsDB *accountsDBApiWithHistory) SetTxHashForLatestStateChanges(_ []byte, _ *transaction.Transaction) { } -// ResetStateChangesCollector returns nil -func (accountsDB *accountsDBApiWithHistory) ResetStateChangesCollector() []StateChangesForTx { - return nil -} - // Close will handle the closing of the underlying components func (accountsDB *accountsDBApiWithHistory) Close() error { return accountsDB.innerAccountsAdapter.Close() diff --git a/state/disabled/disabledStateChangesCollector.go b/state/disabled/disabledStateChangesCollector.go index 77a08e73151..8405bd3821a 100644 --- a/state/disabled/disabledStateChangesCollector.go +++ b/state/disabled/disabledStateChangesCollector.go @@ -3,6 +3,8 @@ package disabled import ( "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-go/state" + "github.com/multiversx/mx-chain-go/state/stateChanges" + vmcommon "github.com/multiversx/mx-chain-vm-common-go" ) // disabledStateChangesCollector is a state changes collector that does nothing @@ -14,13 +16,12 @@ func NewDisabledStateChangesCollector() state.StateChangesCollector { return &disabledStateChangesCollector{} } -// AddStateChange does nothing -func (d *disabledStateChangesCollector) AddStateChange(_ state.StateChangeDTO) { +// AddSaveAccountStateChange - +func (d *disabledStateChangesCollector) AddSaveAccountStateChange(oldAccount, account vmcommon.AccountHandler, stateChange stateChanges.StateChange) { } -// GetStateChanges returns an empty slice -func (d *disabledStateChangesCollector) GetStateChanges() []state.StateChangesForTx { - return make([]state.StateChangesForTx, 0) +// AddStateChange does nothing +func (d *disabledStateChangesCollector) AddStateChange(_ stateChanges.StateChange) { } // Reset does nothing @@ -41,8 +42,8 @@ func (d *disabledStateChangesCollector) RevertToIndex(index int) error { return nil } -// DumpToJSONFile returns nil -func (d *disabledStateChangesCollector) DumpToJSONFile() error { +// Publish returns nil +func (d *disabledStateChangesCollector) Publish() error { return nil } diff --git a/state/interface.go b/state/interface.go index e9785ab5794..ae2100aa7a3 100644 --- a/state/interface.go +++ b/state/interface.go @@ -8,6 +8,7 @@ import ( "github.com/multiversx/mx-chain-core-go/data/api" "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-go/common" + "github.com/multiversx/mx-chain-go/state/stateChanges" vmcommon "github.com/multiversx/mx-chain-vm-common-go" ) @@ -92,7 +93,6 @@ type AccountsAdapter interface { SetSyncer(syncer AccountsDBSyncer) error StartSnapshotIfNeeded() error SetTxHashForLatestStateChanges(txHash []byte, tx *transaction.Transaction) - ResetStateChangesCollector() []StateChangesForTx Close() error IsInterfaceNil() bool } @@ -162,7 +162,7 @@ type baseAccountHandler interface { GetRootHash() []byte SetDataTrie(trie common.Trie) DataTrie() common.DataTrieHandler - SaveDirtyData(trie common.Trie) ([]DataTrieChange, []core.TrieData, error) + SaveDirtyData(trie common.Trie) ([]stateChanges.DataTrieChange, []core.TrieData, error) IsInterfaceNil() bool } @@ -260,7 +260,7 @@ type DataTrieTracker interface { SaveKeyValue(key []byte, value []byte) error SetDataTrie(tr common.Trie) DataTrie() common.DataTrieHandler - SaveDirtyData(common.Trie) ([]DataTrieChange, []core.TrieData, error) + SaveDirtyData(common.Trie) ([]stateChanges.DataTrieChange, []core.TrieData, error) MigrateDataTrieLeaves(args vmcommon.ArgsMigrateDataTrieLeaves) error IsInterfaceNil() bool } @@ -356,12 +356,12 @@ type ValidatorInfoHandler interface { // StateChangesCollector defines the methods needed for an StateChangesCollector implementation type StateChangesCollector interface { - AddStateChange(stateChange StateChangeDTO) - GetStateChanges() []StateChangesForTx + AddStateChange(stateChange stateChanges.StateChange) + AddSaveAccountStateChange(oldAccount, account vmcommon.AccountHandler, stateChange stateChanges.StateChange) Reset() AddTxHashToCollectedStateChanges(txHash []byte, tx *transaction.Transaction) SetIndexToLastStateChange(index int) error RevertToIndex(index int) error - DumpToJSONFile() error + Publish() error IsInterfaceNil() bool } diff --git a/state/stateChanges/dataAnalysisCollector.go b/state/stateChanges/dataAnalysisCollector.go new file mode 100644 index 00000000000..c9a004676c1 --- /dev/null +++ b/state/stateChanges/dataAnalysisCollector.go @@ -0,0 +1,211 @@ +package stateChanges + +import ( + "bytes" + "encoding/json" + "fmt" + "math/big" + + "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/multiversx/mx-chain-go/storage" + vmcommon "github.com/multiversx/mx-chain-vm-common-go" +) + +const ( + workingDir = "." + defaultStateChangesPath = "stateChanges" +) + +// StateChangeDTO is used to collect state changes +type DataAnalysisStateChangeDTO struct { + StateChange + Operation string `json:"operation"` + Nonce bool `json:"nonceChanged"` + Balance bool `json:"balanceChanged"` + CodeHash bool `json:"codeHashChanged"` + RootHash bool `json:"rootHashChanged"` + DeveloperReward bool `json:"developerRewardChanged"` + OwnerAddress bool `json:"ownerAddressChanged"` + UserName bool `json:"userNameChanged"` + CodeMetadata bool `json:"codeMetadataChanged"` +} + +// StateChangesForTx is used to collect state changes for a transaction hash +type DataAnalysisStateChangesForTx struct { + TxHash []byte `json:"txHash"` + Tx *transaction.Transaction `json:"tx"` + StateChanges []StateChange `json:"stateChanges"` +} + +type userAccountHandler interface { + GetCodeMetadata() []byte + GetCodeHash() []byte + GetRootHash() []byte + GetBalance() *big.Int + GetDeveloperReward() *big.Int + GetOwnerAddress() []byte + GetUserName() []byte + vmcommon.AccountHandler +} + +type dataAnalysisCollector struct { + *stateChangesCollector + + cachedTxs map[string]*transaction.Transaction + storer storage.Persister +} + +// NewStateChangesCollector creates a new StateChangesCollector +func NewDataAnalysisStateChangesCollector(storer storage.Persister) (*dataAnalysisCollector, error) { + if check.IfNil(storer) { + return nil, storage.ErrNilPersisterFactory + } + + return &dataAnalysisCollector{ + stateChangesCollector: &stateChangesCollector{ + stateChanges: make([]StateChange, 0), + }, + cachedTxs: make(map[string]*transaction.Transaction), + storer: storer, + }, nil +} + +func (scc *dataAnalysisCollector) AddSaveAccountStateChange(oldAccount, account vmcommon.AccountHandler, stateChange StateChange) { + dataAnalysisStateChange := &DataAnalysisStateChangeDTO{ + StateChange: stateChange, + } + + checkAccountChanges(oldAccount, account, dataAnalysisStateChange) + + scc.AddStateChange(stateChange) +} + +func checkAccountChanges(oldAcc, newAcc vmcommon.AccountHandler, stateChange *DataAnalysisStateChangeDTO) { + baseNewAcc, newAccOk := newAcc.(userAccountHandler) + if !newAccOk { + return + } + baseOldAccount, oldAccOk := oldAcc.(userAccountHandler) + if !oldAccOk { + return + } + + if baseNewAcc.GetNonce() != baseOldAccount.GetNonce() { + stateChange.Nonce = true + } + + if baseNewAcc.GetBalance().Uint64() != baseOldAccount.GetBalance().Uint64() { + stateChange.Balance = true + } + + if !bytes.Equal(baseNewAcc.GetCodeHash(), baseOldAccount.GetCodeHash()) { + stateChange.CodeHash = true + } + + if !bytes.Equal(baseNewAcc.GetRootHash(), baseOldAccount.GetRootHash()) { + stateChange.RootHash = true + } + + if !bytes.Equal(baseNewAcc.GetDeveloperReward().Bytes(), baseOldAccount.GetDeveloperReward().Bytes()) { + stateChange.DeveloperReward = true + } + + if !bytes.Equal(baseNewAcc.GetOwnerAddress(), baseOldAccount.GetOwnerAddress()) { + stateChange.OwnerAddress = true + } + + if !bytes.Equal(baseNewAcc.GetUserName(), baseOldAccount.GetUserName()) { + stateChange.UserName = true + } + + if !bytes.Equal(baseNewAcc.GetCodeMetadata(), baseOldAccount.GetCodeMetadata()) { + stateChange.CodeMetadata = true + } +} + +// AddStateChange adds a new state change to the collector +func (scc *dataAnalysisCollector) AddStateChange(stateChange StateChange) { + scc.stateChangesMut.Lock() + scc.stateChanges = append(scc.stateChanges, stateChange) + scc.stateChangesMut.Unlock() +} + +func (scc *dataAnalysisCollector) getStateChangesForTxs() ([]DataAnalysisStateChangesForTx, error) { + scc.stateChangesMut.Lock() + defer scc.stateChangesMut.Unlock() + + stateChangesForTxs := make([]DataAnalysisStateChangesForTx, 0) + + for i := 0; i < len(scc.stateChanges); i++ { + txHash := scc.stateChanges[i].GetTxHash() + + if len(txHash) == 0 { + log.Warn("empty tx hash, state change event not associated to a transaction") + break + } + + log.Warn("txHash", "txHash", string(txHash)) + + cachedTx, txOk := scc.cachedTxs[string(txHash)] + if !txOk { + return nil, fmt.Errorf("did not find tx in cache") + } + + innerStateChangesForTx := make([]StateChange, 0) + for j := i; j < len(scc.stateChanges); j++ { + txHash2 := scc.stateChanges[j].GetTxHash() + if !bytes.Equal(txHash, txHash2) { + i = j + break + } + + innerStateChangesForTx = append(innerStateChangesForTx, scc.stateChanges[j]) + i = j + } + + stateChangesForTx := DataAnalysisStateChangesForTx{ + TxHash: txHash, + Tx: cachedTx, + StateChanges: innerStateChangesForTx, + } + stateChangesForTxs = append(stateChangesForTxs, stateChangesForTx) + } + + return stateChangesForTxs, nil +} + +// Reset resets the state changes collector +func (scc *dataAnalysisCollector) Reset() { + scc.stateChangesMut.Lock() + defer scc.stateChangesMut.Unlock() + + scc.stateChanges = make([]StateChange, 0) + scc.cachedTxs = make(map[string]*transaction.Transaction) +} + +func (scc *dataAnalysisCollector) Publish() error { + stateChangesForTx, err := scc.getStateChangesForTxs() + if err != nil { + return err + } + + for _, stateChange := range stateChangesForTx { + marshalledData, err := json.Marshal(stateChange) + if err != nil { + return err + } + + err = scc.storer.Put(stateChange.TxHash, marshalledData) + if err != nil { + return err + } + } + + return nil +} + +// IsInterfaceNil returns true if there is no value under the interface +func (scc *dataAnalysisCollector) IsInterfaceNil() bool { + return scc == nil +} diff --git a/state/stateChanges/dataAnalysisCollector_test.go b/state/stateChanges/dataAnalysisCollector_test.go new file mode 100644 index 00000000000..e45e002a9c1 --- /dev/null +++ b/state/stateChanges/dataAnalysisCollector_test.go @@ -0,0 +1 @@ +package stateChanges diff --git a/state/stateChanges/export_test.go b/state/stateChanges/export_test.go new file mode 100644 index 00000000000..86350f45a5c --- /dev/null +++ b/state/stateChanges/export_test.go @@ -0,0 +1,7 @@ +package stateChanges + +// GetStateChanges - +func (scc *stateChangesCollector) GetStateChanges() []StateChangesForTx { + scs, _ := scc.getStateChangesForTxs() + return scs +} diff --git a/state/stateChangesCollector.go b/state/stateChanges/writeCollector.go similarity index 58% rename from state/stateChangesCollector.go rename to state/stateChanges/writeCollector.go index f42e37bd280..ad7d318e85a 100644 --- a/state/stateChangesCollector.go +++ b/state/stateChanges/writeCollector.go @@ -1,20 +1,18 @@ -package state +package stateChanges import ( "bytes" - "encoding/json" + "encoding/hex" "errors" "fmt" "sync" "github.com/multiversx/mx-chain-core-go/data/transaction" - "github.com/multiversx/mx-chain-storage-go/types" + logger "github.com/multiversx/mx-chain-logger-go" + vmcommon "github.com/multiversx/mx-chain-vm-common-go" ) -const ( - workingDir = "." - defaultStateChangesPath = "stateChanges" -) +var log = logger.GetOrCreate("state/stateChanges") // DataTrieChange represents a change in the data trie type DataTrieChange struct { @@ -26,80 +24,72 @@ type DataTrieChange struct { // ErrStateChangesIndexOutOfBounds signals that the state changes index is out of bounds var ErrStateChangesIndexOutOfBounds = errors.New("state changes index out of bounds") +type StateChange interface { + GetTxHash() []byte + SetTxHash(txHash []byte) + GetIndex() int + SetIndex(index int) +} + // StateChangeDTO is used to collect state changes +// TODO: change to proto structs type StateChangeDTO struct { + Type string `json:"type"` Index int `json:"-"` TxHash []byte `json:"-"` - Type string `json:"type"` MainTrieKey []byte `json:"mainTrieKey"` MainTrieVal []byte `json:"-"` Operation string `json:"operation"` - Nonce bool `json:"nonceChanged"` - Balance bool `json:"balanceChanged"` - CodeHash bool `json:"codeHashChanged"` - RootHash bool `json:"rootHashChanged"` - DeveloperReward bool `json:"developerRewardChanged"` - OwnerAddress bool `json:"ownerAddressChanged"` - UserName bool `json:"userNameChanged"` - CodeMetadata bool `json:"codeMetadataChanged"` DataTrieChanges []DataTrieChange `json:"dataTrieChanges"` } -// StateChangesForTx is used to collect state changes for a transaction hash -type StateChangesForTx struct { - TxHash []byte `json:"txHash"` - Tx *transaction.Transaction `json:"tx"` - StateChanges []StateChangeDTO `json:"stateChanges"` +func (sc *StateChangeDTO) GetIndex() int { + return sc.Index } -type StateChangesTx struct { - TxHash []byte `json:"txHash"` - Tx *transaction.Transaction `json:"tx"` +func (sc *StateChangeDTO) SetIndex(index int) { + sc.Index = index } -type stateChangesCollector struct { - stateChanges []StateChangeDTO - stateChangesMut sync.RWMutex +func (sc *StateChangeDTO) GetTxHash() []byte { + return sc.TxHash +} - cachedTxs map[string]*transaction.Transaction +func (sc *StateChangeDTO) SetTxHash(txHash []byte) { + sc.TxHash = txHash +} - storer types.Persister +// StateChangesForTx is used to collect state changes for a transaction hash +type StateChangesForTx struct { + TxHash []byte `json:"txHash"` + StateChanges []StateChange `json:"stateChanges"` +} + +type stateChangesCollector struct { + stateChanges []StateChange + stateChangesMut sync.RWMutex } // NewStateChangesCollector creates a new StateChangesCollector func NewStateChangesCollector() *stateChangesCollector { - // dbPath := filepath.Join(workingDir, "stateChangesDB", "StateChanges") - - // db, err := leveldb.NewSerialDB(dbPath, 2, 100, 10) - // if err != nil { - // log.Error("NewStateChangesCollector: failed to create level db") - // } + // TODO: add outport driver return &stateChangesCollector{ - stateChanges: make([]StateChangeDTO, 0), - cachedTxs: make(map[string]*transaction.Transaction), - storer: nil, + stateChanges: make([]StateChange, 0), } } +func (scc *stateChangesCollector) AddSaveAccountStateChange(_, _ vmcommon.AccountHandler, stateChange StateChange) { + scc.AddStateChange(stateChange) +} + // AddStateChange adds a new state change to the collector -func (scc *stateChangesCollector) AddStateChange(stateChange StateChangeDTO) { +func (scc *stateChangesCollector) AddStateChange(stateChange StateChange) { scc.stateChangesMut.Lock() scc.stateChanges = append(scc.stateChanges, stateChange) scc.stateChangesMut.Unlock() } -// GetStateChanges returns the accumulated state changes -func (scc *stateChangesCollector) GetStateChanges() []StateChangesForTx { - stateChangesForTx, err := scc.getStateChangesForTxs() - if err != nil { - log.Warn("failed to get state changes for tx", "error", err) - return make([]StateChangesForTx, 0) - } - - return stateChangesForTx -} - func (scc *stateChangesCollector) getStateChangesForTxs() ([]StateChangesForTx, error) { scc.stateChangesMut.Lock() defer scc.stateChangesMut.Unlock() @@ -107,7 +97,7 @@ func (scc *stateChangesCollector) getStateChangesForTxs() ([]StateChangesForTx, stateChangesForTxs := make([]StateChangesForTx, 0) for i := 0; i < len(scc.stateChanges); i++ { - txHash := scc.stateChanges[i].TxHash + txHash := scc.stateChanges[i].GetTxHash() if len(txHash) == 0 { log.Warn("empty tx hash, state change event not associated to a transaction") @@ -116,14 +106,9 @@ func (scc *stateChangesCollector) getStateChangesForTxs() ([]StateChangesForTx, log.Warn("txHash", "txHash", string(txHash)) - cachedTx, txOk := scc.cachedTxs[string(txHash)] - if !txOk { - return nil, fmt.Errorf("did not find tx in cache") - } - - innerStateChangesForTx := make([]StateChangeDTO, 0) + innerStateChangesForTx := make([]StateChange, 0) for j := i; j < len(scc.stateChanges); j++ { - txHash2 := scc.stateChanges[j].TxHash + txHash2 := scc.stateChanges[j].GetTxHash() if !bytes.Equal(txHash, txHash2) { i = j break @@ -135,7 +120,6 @@ func (scc *stateChangesCollector) getStateChangesForTxs() ([]StateChangesForTx, stateChangesForTx := StateChangesForTx{ TxHash: txHash, - Tx: cachedTx, StateChanges: innerStateChangesForTx, } stateChangesForTxs = append(stateChangesForTxs, stateChangesForTx) @@ -149,22 +133,19 @@ func (scc *stateChangesCollector) Reset() { scc.stateChangesMut.Lock() defer scc.stateChangesMut.Unlock() - scc.stateChanges = make([]StateChangeDTO, 0) - scc.cachedTxs = make(map[string]*transaction.Transaction) + scc.stateChanges = make([]StateChange, 0) } func (scc *stateChangesCollector) AddTxHashToCollectedStateChanges(txHash []byte, tx *transaction.Transaction) { scc.stateChangesMut.Lock() defer scc.stateChangesMut.Unlock() - scc.cachedTxs[string(txHash)] = tx - for i := len(scc.stateChanges) - 1; i >= 0; i-- { - if len(scc.stateChanges[i].TxHash) > 0 { + if len(scc.stateChanges[i].GetTxHash()) > 0 { break } - scc.stateChanges[i].TxHash = txHash + scc.stateChanges[i].SetTxHash(txHash) } } @@ -176,7 +157,7 @@ func (scc *stateChangesCollector) SetIndexToLastStateChange(index int) error { scc.stateChangesMut.Lock() defer scc.stateChangesMut.Unlock() - scc.stateChanges[len(scc.stateChanges)-1].Index = index + scc.stateChanges[len(scc.stateChanges)-1].SetIndex(index) return nil } @@ -195,7 +176,7 @@ func (scc *stateChangesCollector) RevertToIndex(index int) error { defer scc.stateChangesMut.Unlock() for i := len(scc.stateChanges) - 1; i >= 0; i-- { - if scc.stateChanges[i].Index == index { + if scc.stateChanges[i].GetIndex() == index { scc.stateChanges = scc.stateChanges[:i] break } @@ -204,25 +185,28 @@ func (scc *stateChangesCollector) RevertToIndex(index int) error { return nil } -func (scc *stateChangesCollector) DumpToJSONFile() error { +func (scc *stateChangesCollector) Publish() error { stateChangesForTx, err := scc.getStateChangesForTxs() if err != nil { return err } - for _, stateChange := range stateChangesForTx { - marshalledData, err := json.Marshal(stateChange) - if err != nil { - return err + printStateChanges(stateChangesForTx) + + return nil +} + +func printStateChanges(stateChanges []StateChangesForTx) { + for _, stateChange := range stateChanges { + + if stateChange.TxHash != nil { + fmt.Println(hex.EncodeToString(stateChange.TxHash)) } - err = scc.storer.Put(stateChange.TxHash, marshalledData) - if err != nil { - return err + for _, st := range stateChange.StateChanges { + fmt.Println(st) } } - - return nil } // IsInterfaceNil returns true if there is no value under the interface diff --git a/state/stateChanges/writeCollector_test.go b/state/stateChanges/writeCollector_test.go new file mode 100644 index 00000000000..91ae533f969 --- /dev/null +++ b/state/stateChanges/writeCollector_test.go @@ -0,0 +1,241 @@ +package stateChanges + +import ( + "fmt" + "strconv" + "testing" + + "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/stretchr/testify/assert" + + "github.com/stretchr/testify/require" +) + +func TestNewStateChangesCollector(t *testing.T) { + t.Parallel() + + stateChangesCollector := NewStateChangesCollector() + require.False(t, stateChangesCollector.IsInterfaceNil()) +} + +func TestStateChangesCollector_AddStateChange(t *testing.T) { + t.Parallel() + + scc := NewStateChangesCollector() + assert.Equal(t, 0, len(scc.stateChanges)) + + numStateChanges := 10 + for i := 0; i < numStateChanges; i++ { + scc.AddStateChange(&StateChangeDTO{}) + } + assert.Equal(t, numStateChanges, len(scc.stateChanges)) +} + +func TestStateChangesCollector_GetStateChanges(t *testing.T) { + t.Parallel() + + t.Run("getStateChanges with tx hash", func(t *testing.T) { + t.Parallel() + + scc := NewStateChangesCollector() + assert.Equal(t, 0, len(scc.stateChanges)) + assert.Equal(t, 0, len(scc.GetStateChanges())) + + numStateChanges := 10 + for i := 0; i < numStateChanges; i++ { + scc.AddStateChange(&StateChangeDTO{ + MainTrieKey: []byte(strconv.Itoa(i)), + }) + } + assert.Equal(t, numStateChanges, len(scc.stateChanges)) + assert.Equal(t, 0, len(scc.GetStateChanges())) + scc.AddTxHashToCollectedStateChanges([]byte("txHash"), &transaction.Transaction{}) + assert.Equal(t, numStateChanges, len(scc.stateChanges)) + assert.Equal(t, 1, len(scc.GetStateChanges())) + assert.Equal(t, []byte("txHash"), scc.GetStateChanges()[0].TxHash) + assert.Equal(t, numStateChanges, len(scc.GetStateChanges()[0].StateChanges)) + + stateChangesForTx := scc.GetStateChanges() + assert.Equal(t, 1, len(stateChangesForTx)) + assert.Equal(t, []byte("txHash"), stateChangesForTx[0].TxHash) + for i := 0; i < len(stateChangesForTx[0].StateChanges); i++ { + sc, ok := stateChangesForTx[0].StateChanges[i].(*StateChangeDTO) + require.True(t, ok) + + assert.Equal(t, []byte(strconv.Itoa(i)), sc.MainTrieKey) + } + }) + + t.Run("getStateChanges without tx hash", func(t *testing.T) { + t.Parallel() + + scc := NewStateChangesCollector() + assert.Equal(t, 0, len(scc.stateChanges)) + assert.Equal(t, 0, len(scc.GetStateChanges())) + + numStateChanges := 10 + for i := 0; i < numStateChanges; i++ { + scc.AddStateChange(&StateChangeDTO{ + MainTrieKey: []byte(strconv.Itoa(i)), + }) + } + assert.Equal(t, numStateChanges, len(scc.stateChanges)) + assert.Equal(t, 0, len(scc.GetStateChanges())) + + stateChangesForTx := scc.GetStateChanges() + assert.Equal(t, 0, len(stateChangesForTx)) + }) +} + +func TestStateChangesCollector_AddTxHashToCollectedStateChanges(t *testing.T) { + t.Parallel() + + scc := NewStateChangesCollector() + assert.Equal(t, 0, len(scc.stateChanges)) + assert.Equal(t, 0, len(scc.GetStateChanges())) + + scc.AddTxHashToCollectedStateChanges([]byte("txHash0"), &transaction.Transaction{}) + + stateChange := &StateChangeDTO{ + MainTrieKey: []byte("mainTrieKey"), + MainTrieVal: []byte("mainTrieVal"), + DataTrieChanges: []DataTrieChange{{Key: []byte("dataTrieKey"), Val: []byte("dataTrieVal")}}, + } + scc.AddStateChange(stateChange) + + assert.Equal(t, 1, len(scc.stateChanges)) + assert.Equal(t, 0, len(scc.GetStateChanges())) + scc.AddTxHashToCollectedStateChanges([]byte("txHash"), &transaction.Transaction{}) + assert.Equal(t, 1, len(scc.stateChanges)) + assert.Equal(t, 1, len(scc.GetStateChanges())) + + stateChangesForTx := scc.GetStateChanges() + assert.Equal(t, 1, len(stateChangesForTx)) + assert.Equal(t, []byte("txHash"), stateChangesForTx[0].TxHash) + assert.Equal(t, 1, len(stateChangesForTx[0].StateChanges)) + + sc, ok := stateChangesForTx[0].StateChanges[0].(*StateChangeDTO) + require.True(t, ok) + + assert.Equal(t, []byte("mainTrieKey"), sc.MainTrieKey) + assert.Equal(t, []byte("mainTrieVal"), sc.MainTrieVal) + assert.Equal(t, 1, len(sc.DataTrieChanges)) +} + +func TestStateChangesCollector_RevertToIndex_FailIfWrongIndex(t *testing.T) { + t.Parallel() + + scc := NewStateChangesCollector() + numStateChanges := len(scc.stateChanges) + + err := scc.RevertToIndex(-1) + require.Equal(t, ErrStateChangesIndexOutOfBounds, err) + + err = scc.RevertToIndex(numStateChanges + 1) + require.Equal(t, ErrStateChangesIndexOutOfBounds, err) +} + +func TestStateChangesCollector_RevertToIndex(t *testing.T) { + t.Parallel() + + scc := NewStateChangesCollector() + + numStateChanges := 10 + for i := 0; i < numStateChanges; i++ { + scc.AddStateChange(&StateChangeDTO{}) + scc.SetIndexToLastStateChange(i) + } + scc.AddTxHashToCollectedStateChanges([]byte("txHash1"), &transaction.Transaction{}) + + for i := numStateChanges; i < numStateChanges*2; i++ { + scc.AddStateChange(&StateChangeDTO{}) + scc.AddTxHashToCollectedStateChanges([]byte("txHash"+fmt.Sprintf("%d", i)), &transaction.Transaction{}) + } + scc.SetIndexToLastStateChange(numStateChanges) + + assert.Equal(t, numStateChanges*2, len(scc.stateChanges)) + + err := scc.RevertToIndex(numStateChanges) + require.Nil(t, err) + assert.Equal(t, numStateChanges*2-1, len(scc.stateChanges)) + + err = scc.RevertToIndex(numStateChanges - 1) + require.Nil(t, err) + assert.Equal(t, numStateChanges-1, len(scc.stateChanges)) + + err = scc.RevertToIndex(numStateChanges / 2) + require.Nil(t, err) + assert.Equal(t, numStateChanges/2, len(scc.stateChanges)) + + err = scc.RevertToIndex(1) + require.Nil(t, err) + assert.Equal(t, 1, len(scc.stateChanges)) + + err = scc.RevertToIndex(0) + require.Nil(t, err) + assert.Equal(t, 0, len(scc.stateChanges)) +} + +func TestStateChangesCollector_SetIndexToLastStateChange(t *testing.T) { + t.Parallel() + + t.Run("should fail if valid index", func(t *testing.T) { + t.Parallel() + + scc := NewStateChangesCollector() + + err := scc.SetIndexToLastStateChange(-1) + require.Equal(t, ErrStateChangesIndexOutOfBounds, err) + + numStateChanges := len(scc.stateChanges) + err = scc.SetIndexToLastStateChange(numStateChanges + 1) + require.Equal(t, ErrStateChangesIndexOutOfBounds, err) + }) + + t.Run("should work", func(t *testing.T) { + t.Parallel() + + scc := NewStateChangesCollector() + + numStateChanges := 10 + for i := 0; i < numStateChanges; i++ { + scc.AddStateChange(&StateChangeDTO{}) + err := scc.SetIndexToLastStateChange(i) + require.Nil(t, err) + } + scc.AddTxHashToCollectedStateChanges([]byte("txHash1"), &transaction.Transaction{}) + + for i := numStateChanges; i < numStateChanges*2; i++ { + scc.AddStateChange(&StateChangeDTO{}) + scc.AddTxHashToCollectedStateChanges([]byte("txHash"+fmt.Sprintf("%d", i)), &transaction.Transaction{}) + } + err := scc.SetIndexToLastStateChange(numStateChanges) + require.Nil(t, err) + + assert.Equal(t, numStateChanges*2, len(scc.stateChanges)) + }) +} + +func TestStateChangesCollector_Reset(t *testing.T) { + t.Parallel() + + scc := NewStateChangesCollector() + assert.Equal(t, 0, len(scc.stateChanges)) + + numStateChanges := 10 + for i := 0; i < numStateChanges; i++ { + scc.AddStateChange(&StateChangeDTO{}) + } + scc.AddTxHashToCollectedStateChanges([]byte("txHash"), &transaction.Transaction{}) + for i := numStateChanges; i < numStateChanges*2; i++ { + scc.AddStateChange(&StateChangeDTO{}) + } + assert.Equal(t, numStateChanges*2, len(scc.stateChanges)) + + assert.Equal(t, 1, len(scc.GetStateChanges())) + + scc.Reset() + assert.Equal(t, 0, len(scc.stateChanges)) + + assert.Equal(t, 0, len(scc.GetStateChanges())) +} diff --git a/state/trackableDataTrie/trackableDataTrie.go b/state/trackableDataTrie/trackableDataTrie.go index 6a4a7103d13..c83ad850034 100644 --- a/state/trackableDataTrie/trackableDataTrie.go +++ b/state/trackableDataTrie/trackableDataTrie.go @@ -14,6 +14,7 @@ import ( errorsCommon "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/state/dataTrieValue" + "github.com/multiversx/mx-chain-go/state/stateChanges" logger "github.com/multiversx/mx-chain-logger-go" vmcommon "github.com/multiversx/mx-chain-vm-common-go" ) @@ -99,11 +100,11 @@ func (tdt *trackableDataTrie) RetrieveValue(key []byte) ([]byte, uint32, error) log.Trace("retrieve value from trie", "key", key, "value", val, "account", tdt.identifier) - stateChange := state.StateChangeDTO{ + stateChange := &stateChanges.StateChangeDTO{ Type: "read", MainTrieKey: tdt.identifier, MainTrieVal: nil, - DataTrieChanges: []state.DataTrieChange{ + DataTrieChanges: []stateChanges.DataTrieChange{ { Type: "read", Key: key, @@ -240,9 +241,9 @@ func (tdt *trackableDataTrie) DataTrie() common.DataTrieHandler { } // SaveDirtyData saved the dirty data to the trie -func (tdt *trackableDataTrie) SaveDirtyData(mainTrie common.Trie) ([]state.DataTrieChange, []core.TrieData, error) { +func (tdt *trackableDataTrie) SaveDirtyData(mainTrie common.Trie) ([]stateChanges.DataTrieChange, []core.TrieData, error) { if len(tdt.dirtyData) == 0 { - return make([]state.DataTrieChange, 0), make([]core.TrieData, 0), nil + return make([]stateChanges.DataTrieChange, 0), make([]core.TrieData, 0), nil } if check.IfNil(tdt.tr) { @@ -262,10 +263,10 @@ func (tdt *trackableDataTrie) SaveDirtyData(mainTrie common.Trie) ([]state.DataT return tdt.updateTrie(dtr) } -func (tdt *trackableDataTrie) updateTrie(dtr state.DataTrie) ([]state.DataTrieChange, []core.TrieData, error) { +func (tdt *trackableDataTrie) updateTrie(dtr state.DataTrie) ([]stateChanges.DataTrieChange, []core.TrieData, error) { oldValues := make([]core.TrieData, len(tdt.dirtyData)) - newData := make([]state.DataTrieChange, len(tdt.dirtyData)) - deletedKeys := make([]state.DataTrieChange, 0) + newData := make([]stateChanges.DataTrieChange, len(tdt.dirtyData)) + deletedKeys := make([]stateChanges.DataTrieChange, 0) index := 0 for key, dataEntry := range tdt.dirtyData { @@ -282,7 +283,7 @@ func (tdt *trackableDataTrie) updateTrie(dtr state.DataTrie) ([]state.DataTrieCh if wasDeleted { deletedKeys = append(deletedKeys, - state.DataTrieChange{ + stateChanges.DataTrieChange{ Type: "write", Key: []byte(key), Val: nil, @@ -313,7 +314,7 @@ func (tdt *trackableDataTrie) updateTrie(dtr state.DataTrie) ([]state.DataTrieCh return nil, nil, fmt.Errorf("index out of range") } - newData[dataEntry.index] = state.DataTrieChange{ + newData[dataEntry.index] = stateChanges.DataTrieChange{ Type: "write", Key: dataTrieKey, Val: dataTrieVal, @@ -322,7 +323,7 @@ func (tdt *trackableDataTrie) updateTrie(dtr state.DataTrie) ([]state.DataTrieCh tdt.dirtyData = make(map[string]dirtyData) - stateChanges := make([]state.DataTrieChange, 0) + stateChanges := make([]stateChanges.DataTrieChange, 0) for i := range newData { if len(newData[i].Key) == 0 { continue diff --git a/testscommon/trie/dataTrieTrackerStub.go b/testscommon/trie/dataTrieTrackerStub.go index 60eca7ef978..9ee46a0a8e8 100644 --- a/testscommon/trie/dataTrieTrackerStub.go +++ b/testscommon/trie/dataTrieTrackerStub.go @@ -4,7 +4,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-go/common" - "github.com/multiversx/mx-chain-go/state" + "github.com/multiversx/mx-chain-go/state/stateChanges" vmcommon "github.com/multiversx/mx-chain-vm-common-go" ) @@ -16,7 +16,7 @@ type DataTrieTrackerStub struct { SaveKeyValueCalled func(key []byte, value []byte) error SetDataTrieCalled func(tr common.Trie) DataTrieCalled func() common.Trie - SaveDirtyDataCalled func(trie common.Trie) ([]state.DataTrieChange, []core.TrieData, error) + SaveDirtyDataCalled func(trie common.Trie) ([]stateChanges.DataTrieChange, []core.TrieData, error) SaveTrieDataCalled func(trieData core.TrieData) error MigrateDataTrieLeavesCalled func(args vmcommon.ArgsMigrateDataTrieLeaves) error } @@ -62,12 +62,12 @@ func (dtts *DataTrieTrackerStub) DataTrie() common.DataTrieHandler { } // SaveDirtyData - -func (dtts *DataTrieTrackerStub) SaveDirtyData(mainTrie common.Trie) ([]state.DataTrieChange, []core.TrieData, error) { +func (dtts *DataTrieTrackerStub) SaveDirtyData(mainTrie common.Trie) ([]stateChanges.DataTrieChange, []core.TrieData, error) { if dtts.SaveDirtyDataCalled != nil { return dtts.SaveDirtyDataCalled(mainTrie) } - return make([]state.DataTrieChange, 0), make([]core.TrieData, 0), nil + return make([]stateChanges.DataTrieChange, 0), make([]core.TrieData, 0), nil } // MigrateDataTrieLeaves - From ce8c1523142427504b96cb2a3f5e25acdabd3fe3 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Mon, 26 Aug 2024 15:26:41 +0300 Subject: [PATCH 16/55] fix referencies --- genesis/mock/userAccountMock.go | 4 ++-- genesis/process/memoryComponents.go | 2 +- integrationTests/testInitializer.go | 5 +++-- process/transactionEvaluator/simulationAccountsDB.go | 5 ----- state/accountsDB.go | 6 +----- state/stateChanges/dataAnalysisCollector.go | 6 +----- state/stateChanges/writeCollector.go | 4 +--- testscommon/factory/stateComponentsMock.go | 6 ++++++ testscommon/integrationtests/factory.go | 3 ++- testscommon/state/accountWrapperMock.go | 3 ++- testscommon/state/accountsAdapterStub.go | 10 ---------- testscommon/state/userAccountStub.go | 3 ++- 12 files changed, 21 insertions(+), 36 deletions(-) diff --git a/genesis/mock/userAccountMock.go b/genesis/mock/userAccountMock.go index 10362edba89..6a9e31c2dd0 100644 --- a/genesis/mock/userAccountMock.go +++ b/genesis/mock/userAccountMock.go @@ -7,7 +7,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-go/common" - "github.com/multiversx/mx-chain-go/state" + "github.com/multiversx/mx-chain-go/state/stateChanges" ) // ErrNegativeValue - @@ -148,7 +148,7 @@ func (uam *UserAccountMock) GetUserName() []byte { } // SaveDirtyData - -func (uam *UserAccountMock) SaveDirtyData(_ common.Trie) ([]state.DataTrieChange, []core.TrieData, error) { +func (uam *UserAccountMock) SaveDirtyData(_ common.Trie) ([]stateChanges.DataTrieChange, []core.TrieData, error) { return nil, nil, nil } diff --git a/genesis/process/memoryComponents.go b/genesis/process/memoryComponents.go index 2db8a0bd1ba..23bd7d3850c 100644 --- a/genesis/process/memoryComponents.go +++ b/genesis/process/memoryComponents.go @@ -34,7 +34,7 @@ func createAccountAdapter( StoragePruningManager: disabled.NewDisabledStoragePruningManager(), AddressConverter: addressConverter, SnapshotsManager: disabledState.NewDisabledSnapshotsManager(), - StateChangesCollector: state.NewStateChangesCollector(), + StateChangesCollector: disabledState.NewDisabledStateChangesCollector(), } adb, err := state.NewAccountsDB(args) diff --git a/integrationTests/testInitializer.go b/integrationTests/testInitializer.go index 76a826e70aa..df0863ebb6a 100644 --- a/integrationTests/testInitializer.go +++ b/integrationTests/testInitializer.go @@ -49,6 +49,7 @@ import ( "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/state/accounts" + "github.com/multiversx/mx-chain-go/state/disabled" "github.com/multiversx/mx-chain-go/state/factory" "github.com/multiversx/mx-chain-go/state/iteratorChannelsProvider" "github.com/multiversx/mx-chain-go/state/lastSnapshotMarker" @@ -486,7 +487,7 @@ func CreateAccountsDBWithEnableEpochsHandler( StoragePruningManager: spm, AddressConverter: &testscommon.PubkeyConverterMock{}, SnapshotsManager: snapshotsManager, - StateChangesCollector: state.NewStateChangesCollector(), + StateChangesCollector: disabled.NewDisabledStateChangesCollector(), } adb, _ := state.NewAccountsDB(args) @@ -969,7 +970,7 @@ func GenerateAddressJournalAccountAccountsDB() ([]byte, state.UserAccountHandler adb, _ := CreateAccountsDB(UserAccount, trieStorage) dtlp, _ := parsers.NewDataTrieLeafParser(adr, &marshallerMock.MarshalizerMock{}, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) - dtt, _ := trackableDataTrie.NewTrackableDataTrie(adr, &testscommon.HasherStub{}, &marshallerMock.MarshalizerMock{}, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + dtt, _ := trackableDataTrie.NewTrackableDataTrie(adr, &testscommon.HasherStub{}, &marshallerMock.MarshalizerMock{}, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, disabled.NewDisabledStateChangesCollector()) account, _ := accounts.NewUserAccount(adr, dtt, dtlp) diff --git a/process/transactionEvaluator/simulationAccountsDB.go b/process/transactionEvaluator/simulationAccountsDB.go index 21c045a2841..55ea394d2e6 100644 --- a/process/transactionEvaluator/simulationAccountsDB.go +++ b/process/transactionEvaluator/simulationAccountsDB.go @@ -178,11 +178,6 @@ func (r *simulationAccountsDB) SetTxHashForLatestStateChanges(txHash []byte, tx r.originalAccounts.SetTxHashForLatestStateChanges(txHash, tx) } -// ResetStateChangesCollector - -func (r *simulationAccountsDB) ResetStateChangesCollector() []state.StateChangesForTx { - return nil -} - // Close will handle the closing of the underlying components func (r *simulationAccountsDB) Close() error { return nil diff --git a/state/accountsDB.go b/state/accountsDB.go index d3fe05323f1..7700521ebe0 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -909,14 +909,10 @@ func (adb *AccountsDB) commit() ([]byte, error) { log.Trace("accountsDB.Commit started") adb.entries = make([]JournalEntry, 0) - // if len(stateChanges) != 0 { - // log.Warn("state changes collector is not empty", "state changes", stateChanges) - // adb.stateChangesCollector.Reset() - // } - err := adb.stateChangesCollector.Publish() if err != nil { log.Warn("failed to dump state changes to json file", "error", err) + return nil, err } adb.stateChangesCollector.Reset() diff --git a/state/stateChanges/dataAnalysisCollector.go b/state/stateChanges/dataAnalysisCollector.go index c9a004676c1..d9da80cf57e 100644 --- a/state/stateChanges/dataAnalysisCollector.go +++ b/state/stateChanges/dataAnalysisCollector.go @@ -17,7 +17,7 @@ const ( defaultStateChangesPath = "stateChanges" ) -// StateChangeDTO is used to collect state changes +// TODO: use proto stucts type DataAnalysisStateChangeDTO struct { StateChange Operation string `json:"operation"` @@ -31,7 +31,6 @@ type DataAnalysisStateChangeDTO struct { CodeMetadata bool `json:"codeMetadataChanged"` } -// StateChangesForTx is used to collect state changes for a transaction hash type DataAnalysisStateChangesForTx struct { TxHash []byte `json:"txHash"` Tx *transaction.Transaction `json:"tx"` @@ -56,7 +55,6 @@ type dataAnalysisCollector struct { storer storage.Persister } -// NewStateChangesCollector creates a new StateChangesCollector func NewDataAnalysisStateChangesCollector(storer storage.Persister) (*dataAnalysisCollector, error) { if check.IfNil(storer) { return nil, storage.ErrNilPersisterFactory @@ -145,8 +143,6 @@ func (scc *dataAnalysisCollector) getStateChangesForTxs() ([]DataAnalysisStateCh break } - log.Warn("txHash", "txHash", string(txHash)) - cachedTx, txOk := scc.cachedTxs[string(txHash)] if !txOk { return nil, fmt.Errorf("did not find tx in cache") diff --git a/state/stateChanges/writeCollector.go b/state/stateChanges/writeCollector.go index ad7d318e85a..99213c247cb 100644 --- a/state/stateChanges/writeCollector.go +++ b/state/stateChanges/writeCollector.go @@ -32,7 +32,7 @@ type StateChange interface { } // StateChangeDTO is used to collect state changes -// TODO: change to proto structs +// TODO: change to use proto structs type StateChangeDTO struct { Type string `json:"type"` Index int `json:"-"` @@ -104,8 +104,6 @@ func (scc *stateChangesCollector) getStateChangesForTxs() ([]StateChangesForTx, break } - log.Warn("txHash", "txHash", string(txHash)) - innerStateChangesForTx := make([]StateChange, 0) for j := i; j < len(scc.stateChanges); j++ { txHash2 := scc.stateChanges[j].GetTxHash() diff --git a/testscommon/factory/stateComponentsMock.go b/testscommon/factory/stateComponentsMock.go index 5aa541dffa0..eb2b586db9e 100644 --- a/testscommon/factory/stateComponentsMock.go +++ b/testscommon/factory/stateComponentsMock.go @@ -16,6 +16,7 @@ type StateComponentsMock struct { Tries common.TriesHolder StorageManagers map[string]common.StorageManager MissingNodesNotifier common.MissingTrieNodesNotifier + ChangesCollector state.StateChangesCollector } // NewStateComponentsMockFromRealComponent - @@ -89,6 +90,11 @@ func (scm *StateComponentsMock) MissingTrieNodesNotifier() common.MissingTrieNod return scm.MissingNodesNotifier } +// StateChangesCollector - +func (scm *StateComponentsMock) StateChangesCollector() state.StateChangesCollector { + return scm.ChangesCollector +} + // IsInterfaceNil - func (scm *StateComponentsMock) IsInterfaceNil() bool { return scm == nil diff --git a/testscommon/integrationtests/factory.go b/testscommon/integrationtests/factory.go index f69bc74bbaf..3211b79b48b 100644 --- a/testscommon/integrationtests/factory.go +++ b/testscommon/integrationtests/factory.go @@ -7,6 +7,7 @@ import ( "github.com/multiversx/mx-chain-go/common/statistics" "github.com/multiversx/mx-chain-go/config" "github.com/multiversx/mx-chain-go/state" + "github.com/multiversx/mx-chain-go/state/disabled" accountFactory "github.com/multiversx/mx-chain-go/state/factory" "github.com/multiversx/mx-chain-go/state/iteratorChannelsProvider" "github.com/multiversx/mx-chain-go/state/lastSnapshotMarker" @@ -131,7 +132,7 @@ func CreateAccountsDB(db storage.Storer, enableEpochs common.EnableEpochsHandler StoragePruningManager: spm, AddressConverter: &testscommon.PubkeyConverterMock{}, SnapshotsManager: snapshotsManager, - StateChangesCollector: state.NewStateChangesCollector(), + StateChangesCollector: disabled.NewDisabledStateChangesCollector(), } adb, _ := state.NewAccountsDB(argsAccountsDB) diff --git a/testscommon/state/accountWrapperMock.go b/testscommon/state/accountWrapperMock.go index 7dcabfec30d..5eecb17393f 100644 --- a/testscommon/state/accountWrapperMock.go +++ b/testscommon/state/accountWrapperMock.go @@ -10,6 +10,7 @@ import ( "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/state/disabled" + "github.com/multiversx/mx-chain-go/state/stateChanges" "github.com/multiversx/mx-chain-go/state/trackableDataTrie" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" @@ -198,7 +199,7 @@ func (awm *AccountWrapMock) DataTrie() common.DataTrieHandler { } // SaveDirtyData - -func (awm *AccountWrapMock) SaveDirtyData(trie common.Trie) ([]state.DataTrieChange, []core.TrieData, error) { +func (awm *AccountWrapMock) SaveDirtyData(trie common.Trie) ([]stateChanges.DataTrieChange, []core.TrieData, error) { return awm.trackableDataTrie.SaveDirtyData(trie) } diff --git a/testscommon/state/accountsAdapterStub.go b/testscommon/state/accountsAdapterStub.go index 85683ad7fe4..7a047c1c28e 100644 --- a/testscommon/state/accountsAdapterStub.go +++ b/testscommon/state/accountsAdapterStub.go @@ -41,7 +41,6 @@ type AccountsStub struct { SetSyncerCalled func(syncer state.AccountsDBSyncer) error StartSnapshotIfNeededCalled func() error SetTxHashForLatestStateChangesCalled func(txHash []byte, tx *transaction.Transaction) - ResetStateChangesCollectorCalled func() []state.StateChangesForTx } // CleanCache - @@ -267,15 +266,6 @@ func (as *AccountsStub) SetTxHashForLatestStateChanges(txHash []byte, tx *transa } } -// ResetStateChangesCollector - -func (as *AccountsStub) ResetStateChangesCollector() []state.StateChangesForTx { - if as.ResetStateChangesCollectorCalled != nil { - return as.ResetStateChangesCollectorCalled() - } - - return nil -} - // Close - func (as *AccountsStub) Close() error { if as.CloseCalled != nil { diff --git a/testscommon/state/userAccountStub.go b/testscommon/state/userAccountStub.go index ec386a68dbe..10bc9fa6932 100644 --- a/testscommon/state/userAccountStub.go +++ b/testscommon/state/userAccountStub.go @@ -8,6 +8,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/state" + "github.com/multiversx/mx-chain-go/state/stateChanges" vmcommon "github.com/multiversx/mx-chain-vm-common-go" ) @@ -189,7 +190,7 @@ func (u *UserAccountStub) IsGuarded() bool { } // SaveDirtyData - -func (u *UserAccountStub) SaveDirtyData(_ common.Trie) ([]state.DataTrieChange, []core.TrieData, error) { +func (u *UserAccountStub) SaveDirtyData(_ common.Trie) ([]stateChanges.DataTrieChange, []core.TrieData, error) { return nil, nil, nil } From d7c357e25943cc2f8b307e9930713acde5687c94 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Tue, 27 Aug 2024 15:17:51 +0300 Subject: [PATCH 17/55] update unit tests --- epochStart/metachain/baseRewards_test.go | 8 +- epochStart/metachain/systemSCs_test.go | 10 +- genesis/process/genesisBlockCreator_test.go | 10 +- process/block/argProcessor.go | 1 + process/block/baseProcess.go | 26 +- process/block/baseProcess_test.go | 2 + process/block/metablock.go | 1 + process/block/shardblock.go | 1 + process/rewardTransaction/process_test.go | 9 +- process/scToProtocol/stakingToPeer_test.go | 9 +- .../smartContract/processorV2/process_test.go | 8 +- .../tokensSuppliesProcessor_test.go | 9 +- state/accountsDB_test.go | 302 +++++++++--------- state/factory/accountCreator.go | 3 + state/factory/accountCreator_test.go | 8 +- .../factory/accountsAdapterAPICreator_test.go | 3 +- state/stateChanges/dataAnalysisCollector.go | 3 + state/stateChanges/writeCollector.go | 6 + state/stateChangesCollector_test.go | 234 -------------- .../storagePruningManager_test.go | 13 +- state/trackableDataTrie/trackableDataTrie.go | 4 + .../trackableDataTrie_test.go | 232 ++++++++++++-- 22 files changed, 448 insertions(+), 454 deletions(-) delete mode 100644 state/stateChangesCollector_test.go diff --git a/epochStart/metachain/baseRewards_test.go b/epochStart/metachain/baseRewards_test.go index 50aeb42e7ad..87ccb625643 100644 --- a/epochStart/metachain/baseRewards_test.go +++ b/epochStart/metachain/baseRewards_test.go @@ -20,6 +20,7 @@ import ( "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/state/factory" + "github.com/multiversx/mx-chain-go/state/stateChanges" "github.com/multiversx/mx-chain-go/testscommon" txExecOrderStub "github.com/multiversx/mx-chain-go/testscommon/common" dataRetrieverMock "github.com/multiversx/mx-chain-go/testscommon/dataRetriever" @@ -1177,9 +1178,10 @@ func getBaseRewardsArguments() BaseRewardsCreatorArgs { trieFactoryManager, _ := trie.CreateTrieStorageManager(storageManagerArgs, storage.GetStorageManagerOptions()) argsAccCreator := factory.ArgsAccountCreator{ - Hasher: hasher, - Marshaller: marshalizer, - EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + Hasher: hasher, + Marshaller: marshalizer, + EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + StateChangesCollector: stateChanges.NewStateChangesCollector(), } accCreator, _ := factory.NewAccountCreator(argsAccCreator) enableEpochsHandler := &enableEpochsHandlerMock.EnableEpochsHandlerStub{} diff --git a/epochStart/metachain/systemSCs_test.go b/epochStart/metachain/systemSCs_test.go index d203a2c1075..34415df74d7 100644 --- a/epochStart/metachain/systemSCs_test.go +++ b/epochStart/metachain/systemSCs_test.go @@ -38,6 +38,7 @@ import ( "github.com/multiversx/mx-chain-go/state" disabledState "github.com/multiversx/mx-chain-go/state/disabled" "github.com/multiversx/mx-chain-go/state/factory" + "github.com/multiversx/mx-chain-go/state/stateChanges" "github.com/multiversx/mx-chain-go/state/storagePruningManager" "github.com/multiversx/mx-chain-go/state/storagePruningManager/evictionWaitingList" "github.com/multiversx/mx-chain-go/storage" @@ -764,7 +765,7 @@ func createAccountsDB( StoragePruningManager: spm, AddressConverter: &testscommon.PubkeyConverterMock{}, SnapshotsManager: disabledState.NewDisabledSnapshotsManager(), - StateChangesCollector: state.NewStateChangesCollector(), + StateChangesCollector: stateChanges.NewStateChangesCollector(), } adb, _ := state.NewAccountsDB(args) return adb @@ -780,9 +781,10 @@ func createFullArgumentsForSystemSCProcessing(enableEpochsConfig config.EnableEp trieFactoryManager, _ := trie.CreateTrieStorageManager(storageManagerArgs, storageMock.GetStorageManagerOptions()) argsAccCreator := factory.ArgsAccountCreator{ - Hasher: hasher, - Marshaller: marshalizer, - EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + Hasher: hasher, + Marshaller: marshalizer, + EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + StateChangesCollector: stateChanges.NewStateChangesCollector(), } accCreator, _ := factory.NewAccountCreator(argsAccCreator) peerAccCreator := factory.NewPeerAccountCreator() diff --git a/genesis/process/genesisBlockCreator_test.go b/genesis/process/genesisBlockCreator_test.go index 7553025f369..bb3df433aef 100644 --- a/genesis/process/genesisBlockCreator_test.go +++ b/genesis/process/genesisBlockCreator_test.go @@ -27,6 +27,7 @@ import ( "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/state/accounts" factoryState "github.com/multiversx/mx-chain-go/state/factory" + "github.com/multiversx/mx-chain-go/state/stateChanges" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/testscommon" commonMocks "github.com/multiversx/mx-chain-go/testscommon/common" @@ -180,7 +181,7 @@ func createMockArgument( SCDeployEnableEpoch: unreachableEpoch, CleanUpInformativeSCRsEnableEpoch: unreachableEpoch, SCProcessorV2EnableEpoch: unreachableEpoch, - StakeLimitsEnableEpoch: 10, + StakeLimitsEnableEpoch: 10, }, }, RoundConfig: testscommon.GetDefaultRoundsConfig(), @@ -200,9 +201,10 @@ func createMockArgument( } argsAccCreator := factoryState.ArgsAccountCreator{ - Hasher: &hashingMocks.HasherMock{}, - Marshaller: &mock.MarshalizerMock{}, - EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + Hasher: &hashingMocks.HasherMock{}, + Marshaller: &mock.MarshalizerMock{}, + EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + StateChangesCollector: stateChanges.NewStateChangesCollector(), } accCreator, err := factoryState.NewAccountCreator(argsAccCreator) require.Nil(t, err) diff --git a/process/block/argProcessor.go b/process/block/argProcessor.go index df929214829..f5906980469 100644 --- a/process/block/argProcessor.go +++ b/process/block/argProcessor.go @@ -94,6 +94,7 @@ type ArgBaseProcessor struct { BlockProcessingCutoffHandler cutoff.BlockProcessingCutoffHandler ManagedPeersHolder common.ManagedPeersHolder SentSignaturesTracker process.SentSignaturesTracker + StateChangesCollector state.StateChangesCollector } // ArgShardProcessor holds all dependencies required by the process data factory in order to create diff --git a/process/block/baseProcess.go b/process/block/baseProcess.go index 9e1cf4f02f9..2dac2b25a74 100644 --- a/process/block/baseProcess.go +++ b/process/block/baseProcess.go @@ -39,7 +39,6 @@ import ( "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" "github.com/multiversx/mx-chain-go/state" - "github.com/multiversx/mx-chain-go/state/disabled" "github.com/multiversx/mx-chain-go/state/factory" "github.com/multiversx/mx-chain-go/state/parsers" "github.com/multiversx/mx-chain-go/storage/storageunit" @@ -102,17 +101,18 @@ type baseProcessor struct { blockProcessor blockProcessor txCounter *transactionCounter - outportHandler outport.OutportHandler - outportDataProvider outport.DataProviderOutport - historyRepo dblookupext.HistoryRepository - epochNotifier process.EpochNotifier - enableEpochsHandler common.EnableEpochsHandler - roundNotifier process.RoundNotifier - enableRoundsHandler process.EnableRoundsHandler - vmContainerFactory process.VirtualMachinesContainerFactory - vmContainer process.VirtualMachinesContainer - gasConsumedProvider gasConsumedProvider - economicsData process.EconomicsDataHandler + outportHandler outport.OutportHandler + outportDataProvider outport.DataProviderOutport + historyRepo dblookupext.HistoryRepository + epochNotifier process.EpochNotifier + enableEpochsHandler common.EnableEpochsHandler + roundNotifier process.RoundNotifier + enableRoundsHandler process.EnableRoundsHandler + vmContainerFactory process.VirtualMachinesContainerFactory + vmContainer process.VirtualMachinesContainer + gasConsumedProvider gasConsumedProvider + economicsData process.EconomicsDataHandler + stateChangesCollector state.StateChangesCollector processDataTriesOnCommitEpoch bool lastRestartNonce uint64 @@ -1776,7 +1776,7 @@ func (bp *baseProcessor) commitTrieEpochRootHashIfNeeded(metaBlock *block.MetaBl Hasher: bp.hasher, Marshaller: bp.marshalizer, EnableEpochsHandler: bp.enableEpochsHandler, - StateChangesCollector: disabled.NewDisabledStateChangesCollector(), + StateChangesCollector: bp.stateChangesCollector, } accountCreator, err := factory.NewAccountCreator(argsAccCreator) if err != nil { diff --git a/process/block/baseProcess_test.go b/process/block/baseProcess_test.go index f24a580bbc3..78dd5e23bc9 100644 --- a/process/block/baseProcess_test.go +++ b/process/block/baseProcess_test.go @@ -36,6 +36,7 @@ import ( "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" "github.com/multiversx/mx-chain-go/state" + "github.com/multiversx/mx-chain-go/state/disabled" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/database" "github.com/multiversx/mx-chain-go/storage/storageunit" @@ -128,6 +129,7 @@ func createArgBaseProcessor( BlockProcessingCutoffHandler: &testscommon.BlockProcessingCutoffStub{}, ManagedPeersHolder: &testscommon.ManagedPeersHolderStub{}, SentSignaturesTracker: &testscommon.SentSignatureTrackerStub{}, + StateChangesCollector: disabled.NewDisabledStateChangesCollector(), } } diff --git a/process/block/metablock.go b/process/block/metablock.go index 390e1cebf25..a341e25edc2 100644 --- a/process/block/metablock.go +++ b/process/block/metablock.go @@ -137,6 +137,7 @@ func NewMetaProcessor(arguments ArgMetaProcessor) (*metaProcessor, error) { blockProcessingCutoffHandler: arguments.BlockProcessingCutoffHandler, managedPeersHolder: arguments.ManagedPeersHolder, sentSignaturesTracker: arguments.SentSignaturesTracker, + stateChangesCollector: arguments.StateChangesCollector, extraDelayRequestBlockInfo: time.Duration(arguments.Config.EpochStartConfig.ExtraDelayForRequestBlockInfoInMilliseconds) * time.Millisecond, } diff --git a/process/block/shardblock.go b/process/block/shardblock.go index 11e62f63ff9..753e6eb152d 100644 --- a/process/block/shardblock.go +++ b/process/block/shardblock.go @@ -122,6 +122,7 @@ func NewShardProcessor(arguments ArgShardProcessor) (*shardProcessor, error) { blockProcessingCutoffHandler: arguments.BlockProcessingCutoffHandler, managedPeersHolder: arguments.ManagedPeersHolder, sentSignaturesTracker: arguments.SentSignaturesTracker, + stateChangesCollector: arguments.StateChangesCollector, extraDelayRequestBlockInfo: time.Duration(arguments.Config.EpochStartConfig.ExtraDelayForRequestBlockInfoInMilliseconds) * time.Millisecond, } diff --git a/process/rewardTransaction/process_test.go b/process/rewardTransaction/process_test.go index c75fa1a1cdc..85ec1cf6373 100644 --- a/process/rewardTransaction/process_test.go +++ b/process/rewardTransaction/process_test.go @@ -12,6 +12,7 @@ import ( "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/process/rewardTransaction" "github.com/multiversx/mx-chain-go/state/accounts" + "github.com/multiversx/mx-chain-go/state/disabled" "github.com/multiversx/mx-chain-go/state/trackableDataTrie" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" @@ -260,7 +261,13 @@ func TestRewardTxProcessor_ProcessRewardTransactionToASmartContractShouldWork(t address := []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6} - dtt, _ := trackableDataTrie.NewTrackableDataTrie(address, &hashingMocks.HasherMock{}, &marshallerMock.MarshalizerMock{}, enableEpochsHandlerMock.NewEnableEpochsHandlerStub()) + dtt, _ := trackableDataTrie.NewTrackableDataTrie( + address, + &hashingMocks.HasherMock{}, + &marshallerMock.MarshalizerMock{}, + enableEpochsHandlerMock.NewEnableEpochsHandlerStub(), + disabled.NewDisabledStateChangesCollector(), + ) userAccount, _ := accounts.NewUserAccount(address, dtt, &trie.TrieLeafParserStub{}) accountsDb := &stateMock.AccountsStub{ LoadAccountCalled: func(address []byte) (vmcommon.AccountHandler, error) { diff --git a/process/scToProtocol/stakingToPeer_test.go b/process/scToProtocol/stakingToPeer_test.go index f53495e92c9..1127bda29f7 100644 --- a/process/scToProtocol/stakingToPeer_test.go +++ b/process/scToProtocol/stakingToPeer_test.go @@ -20,6 +20,7 @@ import ( "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/state/accounts" + "github.com/multiversx/mx-chain-go/state/stateChanges" "github.com/multiversx/mx-chain-go/state/trackableDataTrie" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" @@ -61,7 +62,13 @@ func createBlockBody() *block.Body { } func createStakingScAccount() state.UserAccountHandler { - dtt, _ := trackableDataTrie.NewTrackableDataTrie(vm.StakingSCAddress, &hashingMocks.HasherMock{}, &marshallerMock.MarshalizerMock{}, enableEpochsHandlerMock.NewEnableEpochsHandlerStub()) + dtt, _ := trackableDataTrie.NewTrackableDataTrie( + vm.StakingSCAddress, + &hashingMocks.HasherMock{}, + &marshallerMock.MarshalizerMock{}, + enableEpochsHandlerMock.NewEnableEpochsHandlerStub(), + stateChanges.NewStateChangesCollector(), + ) userAcc, _ := accounts.NewUserAccount(vm.StakingSCAddress, dtt, &trie.TrieLeafParserStub{}) return userAcc diff --git a/process/smartContract/processorV2/process_test.go b/process/smartContract/processorV2/process_test.go index eedea17f1ad..f7672d66c38 100644 --- a/process/smartContract/processorV2/process_test.go +++ b/process/smartContract/processorV2/process_test.go @@ -27,6 +27,7 @@ import ( "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/state" stateFactory "github.com/multiversx/mx-chain-go/state/factory" + "github.com/multiversx/mx-chain-go/state/stateChanges" "github.com/multiversx/mx-chain-go/storage/storageunit" "github.com/multiversx/mx-chain-go/storage/txcache" "github.com/multiversx/mx-chain-go/testscommon" @@ -61,9 +62,10 @@ func createMockPubkeyConverter() *testscommon.PubkeyConverterMock { func createAccount(address []byte) state.UserAccountHandler { argsAccCreation := stateFactory.ArgsAccountCreator{ - Hasher: &hashingMocks.HasherMock{}, - Marshaller: &marshallerMock.MarshalizerMock{}, - EnableEpochsHandler: enableEpochsHandlerMock.NewEnableEpochsHandlerStub(), + Hasher: &hashingMocks.HasherMock{}, + Marshaller: &marshallerMock.MarshalizerMock{}, + EnableEpochsHandler: enableEpochsHandlerMock.NewEnableEpochsHandlerStub(), + StateChangesCollector: stateChanges.NewStateChangesCollector(), } accountFactory, _ := stateFactory.NewAccountCreator(argsAccCreation) account, _ := accountFactory.CreateAccount(address) diff --git a/process/sync/trieIterators/tokensSuppliesProcessor_test.go b/process/sync/trieIterators/tokensSuppliesProcessor_test.go index 9778ad72459..d791e28fe66 100644 --- a/process/sync/trieIterators/tokensSuppliesProcessor_test.go +++ b/process/sync/trieIterators/tokensSuppliesProcessor_test.go @@ -14,6 +14,7 @@ import ( coreEsdt "github.com/multiversx/mx-chain-go/dblookupext/esdtSupply" "github.com/multiversx/mx-chain-go/state/accounts" "github.com/multiversx/mx-chain-go/state/parsers" + "github.com/multiversx/mx-chain-go/state/stateChanges" "github.com/multiversx/mx-chain-go/state/trackableDataTrie" chainStorage "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" @@ -196,7 +197,13 @@ func TestTokensSuppliesProcessor_HandleTrieAccountIteration(t *testing.T) { args := getTokensSuppliesProcessorArgs() tsp, _ := NewTokensSuppliesProcessor(args) - dtt, _ := trackableDataTrie.NewTrackableDataTrie([]byte("addr"), &hashingMocks.HasherMock{}, &marshallerMock.MarshalizerMock{}, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + dtt, _ := trackableDataTrie.NewTrackableDataTrie( + []byte("addr"), + &hashingMocks.HasherMock{}, + &marshallerMock.MarshalizerMock{}, + &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + stateChanges.NewStateChangesCollector(), + ) dtlp, _ := parsers.NewDataTrieLeafParser([]byte("addr"), &marshallerMock.MarshalizerMock{}, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) userAcc, _ := accounts.NewUserAccount([]byte("addr"), dtt, dtlp) userAcc.SetRootHash([]byte("rootHash")) diff --git a/state/accountsDB_test.go b/state/accountsDB_test.go index 2751290cf5a..ee66a04648d 100644 --- a/state/accountsDB_test.go +++ b/state/accountsDB_test.go @@ -16,7 +16,6 @@ import ( "github.com/multiversx/mx-chain-core-go/core/atomic" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/core/keyValStorage" - "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/errChan" @@ -26,11 +25,11 @@ import ( "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/state/accounts" - "github.com/multiversx/mx-chain-go/state/dataTrieValue" "github.com/multiversx/mx-chain-go/state/factory" "github.com/multiversx/mx-chain-go/state/iteratorChannelsProvider" "github.com/multiversx/mx-chain-go/state/lastSnapshotMarker" "github.com/multiversx/mx-chain-go/state/parsers" + "github.com/multiversx/mx-chain-go/state/stateChanges" "github.com/multiversx/mx-chain-go/state/storagePruningManager" "github.com/multiversx/mx-chain-go/state/storagePruningManager/disabled" "github.com/multiversx/mx-chain-go/state/storagePruningManager/evictionWaitingList" @@ -82,7 +81,7 @@ func createMockAccountsDBArgs() state.ArgsAccountsDB { StoragePruningManager: disabled.NewDisabledStoragePruningManager(), AddressConverter: &testscommon.PubkeyConverterMock{}, SnapshotsManager: snapshotsManager, - StateChangesCollector: state.NewStateChangesCollector(), + StateChangesCollector: stateChanges.NewStateChangesCollector(), } } @@ -150,9 +149,10 @@ func getDefaultStateComponents( ewl, _ := evictionWaitingList.NewMemoryEvictionWaitingList(ewlArgs) spm, _ := storagePruningManager.NewStoragePruningManager(ewl, generalCfg.PruningBufferLen) argsAccCreator := factory.ArgsAccountCreator{ - Hasher: hasher, - Marshaller: marshaller, - EnableEpochsHandler: enableEpochsHandler, + Hasher: hasher, + Marshaller: marshaller, + EnableEpochsHandler: enableEpochsHandler, + StateChangesCollector: stateChanges.NewStateChangesCollector(), } accCreator, _ := factory.NewAccountCreator(argsAccCreator) @@ -176,7 +176,7 @@ func getDefaultStateComponents( StoragePruningManager: spm, AddressConverter: &testscommon.PubkeyConverterMock{}, SnapshotsManager: snapshotsManager, - StateChangesCollector: state.NewStateChangesCollector(), + StateChangesCollector: stateChanges.NewStateChangesCollector(), } adb, _ := state.NewAccountsDB(argsAccountsDB) @@ -372,8 +372,8 @@ func TestAccountsDB_SaveAccountSavesCodeAndDataTrieForUserAccount(t *testing.T) }) dtt := &trieMock.DataTrieTrackerStub{ - SaveDirtyDataCalled: func(_ common.Trie) ([]state.DataTrieChange, []core.TrieData, error) { - var stateChanges []state.DataTrieChange + SaveDirtyDataCalled: func(_ common.Trie) ([]stateChanges.DataTrieChange, []core.TrieData, error) { + var stateChanges []stateChanges.DataTrieChange oldVal := []core.TrieData{ { Key: []byte("key"), @@ -424,127 +424,127 @@ func TestAccountsDB_SaveAccountMalfunctionMarshallerShouldErr(t *testing.T) { assert.NotNil(t, err) } -func TestAccountsDB_SaveAccountCollectsAllStateChanges(t *testing.T) { - t.Parallel() - - autoBalanceFlagEnabled := false - enableEpochs := &enableEpochsHandlerMock.EnableEpochsHandlerStub{ - IsFlagEnabledCalled: func(flag core.EnableEpochFlag) bool { - return autoBalanceFlagEnabled - }, - } - _, adb := getDefaultStateComponentsWithCustomEnableEpochs(enableEpochs) - address := generateRandomByteArray(32) - - stepCreateAccountWithDataTrieAndCode(t, adb, address) - autoBalanceFlagEnabled = true - stepMigrateDataTrieValAndChangeCode(t, adb, address) -} - -func stepCreateAccountWithDataTrieAndCode( - t *testing.T, - adb *state.AccountsDB, - address []byte, -) { - marshaller := &marshallerMock.MarshalizerMock{} - - acc, _ := adb.LoadAccount(address) - userAcc := acc.(state.UserAccountHandler) - code := []byte("smart contract code") - key1 := []byte("key1") - key2 := []byte("key2") - userAcc.SetCode(code) - _ = userAcc.SaveKeyValue(key1, []byte("value")) - _ = userAcc.SaveKeyValue(key2, []byte("value")) - _ = adb.SaveAccount(userAcc) - adb.SetTxHashForLatestStateChanges([]byte("accountCreationTxHash"), &transaction.Transaction{}) - serializedAcc, _ := marshaller.Marshal(userAcc) - codeHash := userAcc.GetCodeHash() - - stateChangesForTx := adb.ResetStateChangesCollector() - assert.Equal(t, 1, len(stateChangesForTx)) - - stateChanges := stateChangesForTx[0].StateChanges - assert.Equal(t, 2, len(stateChanges)) - assert.Equal(t, []byte("accountCreationTxHash"), stateChangesForTx[0].TxHash) - - codeStateChange := stateChanges[0] - assert.Equal(t, codeHash, codeStateChange.MainTrieKey) - codeEntry := &state.CodeEntry{ - Code: code, - NumReferences: 1, - } - serializedCodeEntry, _ := marshaller.Marshal(codeEntry) - assert.Equal(t, serializedCodeEntry, codeStateChange.MainTrieVal) - assert.Equal(t, 0, len(codeStateChange.DataTrieChanges)) - - accountStateChange := stateChanges[1] - assert.Equal(t, address, accountStateChange.MainTrieKey) - assert.Equal(t, serializedAcc, accountStateChange.MainTrieVal) - assert.Equal(t, 2, len(accountStateChange.DataTrieChanges)) - assert.Equal(t, key1, accountStateChange.DataTrieChanges[0].Key) - valWithMetadata1 := append([]byte("value"), key1...) - valWithMetadata1 = append(valWithMetadata1, address...) - assert.Equal(t, valWithMetadata1, accountStateChange.DataTrieChanges[0].Val) - valWithMetadata2 := append([]byte("value"), key2...) - valWithMetadata2 = append(valWithMetadata2, address...) - assert.Equal(t, valWithMetadata2, accountStateChange.DataTrieChanges[1].Val) -} - -func stepMigrateDataTrieValAndChangeCode( - t *testing.T, - adb *state.AccountsDB, - address []byte, -) { - marshaller := &marshallerMock.MarshalizerMock{} - hasher := &hashingMocks.HasherMock{} - - acc, _ := adb.LoadAccount(address) - userAcc := acc.(state.UserAccountHandler) - oldCodeHash := userAcc.GetCodeHash() - code := []byte("new smart contract code") - userAcc.SetCode(code) - _ = userAcc.SaveKeyValue([]byte("key1"), []byte("value1")) - _ = adb.SaveAccount(userAcc) - adb.SetTxHashForLatestStateChanges([]byte("accountUpdateTxHash"), &transaction.Transaction{}) - - stateChangesForTx := adb.ResetStateChangesCollector() - assert.Equal(t, 1, len(stateChangesForTx)) - assert.Equal(t, 3, len(stateChangesForTx[0].StateChanges)) - assert.Equal(t, []byte("accountUpdateTxHash"), stateChangesForTx[0].TxHash) - - stateChanges := stateChangesForTx[0].StateChanges - oldCodeEntryChange := stateChanges[0] - assert.Equal(t, oldCodeHash, oldCodeEntryChange.MainTrieKey) - assert.Equal(t, []byte(nil), oldCodeEntryChange.MainTrieVal) - assert.Equal(t, 0, len(oldCodeEntryChange.DataTrieChanges)) - - newCodeEntryChange := stateChanges[1] - codeEntry := &state.CodeEntry{ - Code: code, - NumReferences: 1, - } - serializedCodeEntry, _ := marshaller.Marshal(codeEntry) - assert.Equal(t, userAcc.GetCodeHash(), newCodeEntryChange.MainTrieKey) - assert.Equal(t, serializedCodeEntry, newCodeEntryChange.MainTrieVal) - assert.Equal(t, 0, len(newCodeEntryChange.DataTrieChanges)) - - accountStateChange := stateChanges[2] - serializedAcc, _ := marshaller.Marshal(userAcc) - assert.Equal(t, address, accountStateChange.MainTrieKey) - assert.Equal(t, serializedAcc, accountStateChange.MainTrieVal) - assert.Equal(t, 2, len(accountStateChange.DataTrieChanges)) - trieVal := &dataTrieValue.TrieLeafData{ - Value: []byte("value1"), - Key: []byte("key1"), - Address: address, - } - serializedTrieVal, _ := marshaller.Marshal(trieVal) - assert.Equal(t, hasher.Compute("key1"), accountStateChange.DataTrieChanges[0].Key) - assert.Equal(t, serializedTrieVal, accountStateChange.DataTrieChanges[0].Val) - assert.Equal(t, []byte("key1"), accountStateChange.DataTrieChanges[1].Key) - assert.Equal(t, []byte(nil), accountStateChange.DataTrieChanges[1].Val) -} +// func TestAccountsDB_SaveAccountCollectsAllStateChanges(t *testing.T) { +// t.Parallel() + +// autoBalanceFlagEnabled := false +// enableEpochs := &enableEpochsHandlerMock.EnableEpochsHandlerStub{ +// IsFlagEnabledCalled: func(flag core.EnableEpochFlag) bool { +// return autoBalanceFlagEnabled +// }, +// } +// _, adb := getDefaultStateComponentsWithCustomEnableEpochs(enableEpochs) +// address := generateRandomByteArray(32) + +// stepCreateAccountWithDataTrieAndCode(t, adb, address) +// autoBalanceFlagEnabled = true +// stepMigrateDataTrieValAndChangeCode(t, adb, address) +// } + +// func stepCreateAccountWithDataTrieAndCode( +// t *testing.T, +// adb *state.AccountsDB, +// address []byte, +// ) { +// marshaller := &marshallerMock.MarshalizerMock{} + +// acc, _ := adb.LoadAccount(address) +// userAcc := acc.(state.UserAccountHandler) +// code := []byte("smart contract code") +// key1 := []byte("key1") +// key2 := []byte("key2") +// userAcc.SetCode(code) +// _ = userAcc.SaveKeyValue(key1, []byte("value")) +// _ = userAcc.SaveKeyValue(key2, []byte("value")) +// _ = adb.SaveAccount(userAcc) +// adb.SetTxHashForLatestStateChanges([]byte("accountCreationTxHash"), &transaction.Transaction{}) +// serializedAcc, _ := marshaller.Marshal(userAcc) +// codeHash := userAcc.GetCodeHash() + +// stateChangesForTx := adb.ResetStateChangesCollector() +// assert.Equal(t, 1, len(stateChangesForTx)) + +// stateChanges := stateChangesForTx[0].StateChanges +// assert.Equal(t, 2, len(stateChanges)) +// assert.Equal(t, []byte("accountCreationTxHash"), stateChangesForTx[0].TxHash) + +// codeStateChange := stateChanges[0] +// assert.Equal(t, codeHash, codeStateChange.MainTrieKey) +// codeEntry := &state.CodeEntry{ +// Code: code, +// NumReferences: 1, +// } +// serializedCodeEntry, _ := marshaller.Marshal(codeEntry) +// assert.Equal(t, serializedCodeEntry, codeStateChange.MainTrieVal) +// assert.Equal(t, 0, len(codeStateChange.DataTrieChanges)) + +// accountStateChange := stateChanges[1] +// assert.Equal(t, address, accountStateChange.MainTrieKey) +// assert.Equal(t, serializedAcc, accountStateChange.MainTrieVal) +// assert.Equal(t, 2, len(accountStateChange.DataTrieChanges)) +// assert.Equal(t, key1, accountStateChange.DataTrieChanges[0].Key) +// valWithMetadata1 := append([]byte("value"), key1...) +// valWithMetadata1 = append(valWithMetadata1, address...) +// assert.Equal(t, valWithMetadata1, accountStateChange.DataTrieChanges[0].Val) +// valWithMetadata2 := append([]byte("value"), key2...) +// valWithMetadata2 = append(valWithMetadata2, address...) +// assert.Equal(t, valWithMetadata2, accountStateChange.DataTrieChanges[1].Val) +// } + +// func stepMigrateDataTrieValAndChangeCode( +// t *testing.T, +// adb *state.AccountsDB, +// address []byte, +// ) { +// marshaller := &marshallerMock.MarshalizerMock{} +// hasher := &hashingMocks.HasherMock{} + +// acc, _ := adb.LoadAccount(address) +// userAcc := acc.(state.UserAccountHandler) +// oldCodeHash := userAcc.GetCodeHash() +// code := []byte("new smart contract code") +// userAcc.SetCode(code) +// _ = userAcc.SaveKeyValue([]byte("key1"), []byte("value1")) +// _ = adb.SaveAccount(userAcc) +// adb.SetTxHashForLatestStateChanges([]byte("accountUpdateTxHash"), &transaction.Transaction{}) + +// stateChangesForTx := adb.ResetStateChangesCollector() +// assert.Equal(t, 1, len(stateChangesForTx)) +// assert.Equal(t, 3, len(stateChangesForTx[0].StateChanges)) +// assert.Equal(t, []byte("accountUpdateTxHash"), stateChangesForTx[0].TxHash) + +// stateChanges := stateChangesForTx[0].StateChanges +// oldCodeEntryChange := stateChanges[0] +// assert.Equal(t, oldCodeHash, oldCodeEntryChange.MainTrieKey) +// assert.Equal(t, []byte(nil), oldCodeEntryChange.MainTrieVal) +// assert.Equal(t, 0, len(oldCodeEntryChange.DataTrieChanges)) + +// newCodeEntryChange := stateChanges[1] +// codeEntry := &state.CodeEntry{ +// Code: code, +// NumReferences: 1, +// } +// serializedCodeEntry, _ := marshaller.Marshal(codeEntry) +// assert.Equal(t, userAcc.GetCodeHash(), newCodeEntryChange.MainTrieKey) +// assert.Equal(t, serializedCodeEntry, newCodeEntryChange.MainTrieVal) +// assert.Equal(t, 0, len(newCodeEntryChange.DataTrieChanges)) + +// accountStateChange := stateChanges[2] +// serializedAcc, _ := marshaller.Marshal(userAcc) +// assert.Equal(t, address, accountStateChange.MainTrieKey) +// assert.Equal(t, serializedAcc, accountStateChange.MainTrieVal) +// assert.Equal(t, 2, len(accountStateChange.DataTrieChanges)) +// trieVal := &dataTrieValue.TrieLeafData{ +// Value: []byte("value1"), +// Key: []byte("key1"), +// Address: address, +// } +// serializedTrieVal, _ := marshaller.Marshal(trieVal) +// assert.Equal(t, hasher.Compute("key1"), accountStateChange.DataTrieChanges[0].Key) +// assert.Equal(t, serializedTrieVal, accountStateChange.DataTrieChanges[0].Val) +// assert.Equal(t, []byte("key1"), accountStateChange.DataTrieChanges[1].Key) +// assert.Equal(t, []byte(nil), accountStateChange.DataTrieChanges[1].Val) +// } func TestAccountsDB_SaveAccountWithSomeValuesShouldWork(t *testing.T) { t.Parallel() @@ -1920,9 +1920,10 @@ func TestAccountsDB_MainTrieAutomaticallyMarksCodeUpdatesForEviction(t *testing. argsAccountsDB.Hasher = hasher argsAccountsDB.Marshaller = marshaller argsAccCreator := factory.ArgsAccountCreator{ - Hasher: hasher, - Marshaller: marshaller, - EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + Hasher: hasher, + Marshaller: marshaller, + EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + StateChangesCollector: stateChanges.NewStateChangesCollector(), } argsAccountsDB.AccountFactory, _ = factory.NewAccountCreator(argsAccCreator) argsAccountsDB.StoragePruningManager = spm @@ -2001,9 +2002,10 @@ func TestAccountsDB_RemoveAccountMarksObsoleteHashesForEviction(t *testing.T) { argsAccountsDB.Hasher = hasher argsAccountsDB.Marshaller = marshaller argsAccCreator := factory.ArgsAccountCreator{ - Hasher: hasher, - Marshaller: marshaller, - EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + Hasher: hasher, + Marshaller: marshaller, + EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + StateChangesCollector: stateChanges.NewStateChangesCollector(), } argsAccountsDB.AccountFactory, _ = factory.NewAccountCreator(argsAccCreator) argsAccountsDB.StoragePruningManager = spm @@ -2203,9 +2205,10 @@ func TestAccountsDB_GetCode(t *testing.T) { argsAccountsDB.Hasher = hasher argsAccountsDB.Marshaller = marshaller argsAccCreator := factory.ArgsAccountCreator{ - Hasher: hasher, - Marshaller: marshaller, - EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + Hasher: hasher, + Marshaller: marshaller, + EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + StateChangesCollector: stateChanges.NewStateChangesCollector(), } argsAccountsDB.AccountFactory, _ = factory.NewAccountCreator(argsAccCreator) argsAccountsDB.StoragePruningManager = spm @@ -2370,9 +2373,10 @@ func TestAccountsDB_Close(t *testing.T) { argsAccountsDB.Hasher = hasher argsAccountsDB.Marshaller = marshaller argsAccCreator := factory.ArgsAccountCreator{ - Hasher: hasher, - Marshaller: marshaller, - EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + Hasher: hasher, + Marshaller: marshaller, + EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + StateChangesCollector: stateChanges.NewStateChangesCollector(), } argsAccountsDB.AccountFactory, _ = factory.NewAccountCreator(argsAccCreator) argsAccountsDB.StoragePruningManager = spm @@ -2658,9 +2662,10 @@ func BenchmarkAccountsDb_GetCodeEntry(b *testing.B) { argsAccountsDB.Hasher = hasher argsAccountsDB.Marshaller = marshaller argsAccCreator := factory.ArgsAccountCreator{ - Hasher: hasher, - Marshaller: marshaller, - EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + Hasher: hasher, + Marshaller: marshaller, + EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + StateChangesCollector: stateChanges.NewStateChangesCollector(), } argsAccountsDB.AccountFactory, _ = factory.NewAccountCreator(argsAccCreator) argsAccountsDB.StoragePruningManager = spm @@ -2978,9 +2983,10 @@ func TestAccountsDB_RevertTxWhichMigratesDataRemovesMigratedData(t *testing.T) { argsAccountsDB.Hasher = hasher argsAccountsDB.Marshaller = marshaller argsAccCreator := factory.ArgsAccountCreator{ - Hasher: hasher, - Marshaller: marshaller, - EnableEpochsHandler: enableEpochsHandler, + Hasher: hasher, + Marshaller: marshaller, + EnableEpochsHandler: enableEpochsHandler, + StateChangesCollector: stateChanges.NewStateChangesCollector(), } argsAccountsDB.AccountFactory, _ = factory.NewAccountCreator(argsAccCreator) argsAccountsDB.StoragePruningManager = spm diff --git a/state/factory/accountCreator.go b/state/factory/accountCreator.go index b8d97415b76..99a16e1b298 100644 --- a/state/factory/accountCreator.go +++ b/state/factory/accountCreator.go @@ -40,6 +40,9 @@ func NewAccountCreator(args ArgsAccountCreator) (state.AccountFactory, error) { if check.IfNil(args.EnableEpochsHandler) { return nil, errors.ErrNilEnableEpochsHandler } + if check.IfNil(args.StateChangesCollector) { + return nil, state.ErrNilStateChangesCollector + } return &accountCreator{ hasher: args.Hasher, diff --git a/state/factory/accountCreator_test.go b/state/factory/accountCreator_test.go index e5cba461d28..11490deae17 100644 --- a/state/factory/accountCreator_test.go +++ b/state/factory/accountCreator_test.go @@ -7,6 +7,7 @@ import ( "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/state/factory" + "github.com/multiversx/mx-chain-go/state/stateChanges" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" "github.com/multiversx/mx-chain-go/testscommon/marshallerMock" @@ -15,9 +16,10 @@ import ( func getDefaultArgs() factory.ArgsAccountCreator { return factory.ArgsAccountCreator{ - Hasher: &hashingMocks.HasherMock{}, - Marshaller: &marshallerMock.MarshalizerMock{}, - EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + Hasher: &hashingMocks.HasherMock{}, + Marshaller: &marshallerMock.MarshalizerMock{}, + EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + StateChangesCollector: stateChanges.NewStateChangesCollector(), } } diff --git a/state/factory/accountsAdapterAPICreator_test.go b/state/factory/accountsAdapterAPICreator_test.go index d1df498d6bc..fd524231e5d 100644 --- a/state/factory/accountsAdapterAPICreator_test.go +++ b/state/factory/accountsAdapterAPICreator_test.go @@ -7,6 +7,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/state" + "github.com/multiversx/mx-chain-go/state/stateChanges" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/testscommon/marshallerMock" mockState "github.com/multiversx/mx-chain-go/testscommon/state" @@ -28,7 +29,7 @@ func createMockAccountsArgs() state.ArgsAccountsDB { StoragePruningManager: &mockState.StoragePruningManagerStub{}, AddressConverter: &testscommon.PubkeyConverterMock{}, SnapshotsManager: &mockState.SnapshotsManagerStub{}, - StateChangesCollector: state.NewStateChangesCollector(), + StateChangesCollector: stateChanges.NewStateChangesCollector(), } } diff --git a/state/stateChanges/dataAnalysisCollector.go b/state/stateChanges/dataAnalysisCollector.go index d9da80cf57e..dc5d5e62a4f 100644 --- a/state/stateChanges/dataAnalysisCollector.go +++ b/state/stateChanges/dataAnalysisCollector.go @@ -55,6 +55,7 @@ type dataAnalysisCollector struct { storer storage.Persister } +// NewDataAnalysisStateChangesCollector will create a new instance of data analysis collector func NewDataAnalysisStateChangesCollector(storer storage.Persister) (*dataAnalysisCollector, error) { if check.IfNil(storer) { return nil, storage.ErrNilPersisterFactory @@ -69,6 +70,7 @@ func NewDataAnalysisStateChangesCollector(storer storage.Persister) (*dataAnalys }, nil } +// AddSaveAccountStateChange adds a new state change for the save account operation func (scc *dataAnalysisCollector) AddSaveAccountStateChange(oldAccount, account vmcommon.AccountHandler, stateChange StateChange) { dataAnalysisStateChange := &DataAnalysisStateChangeDTO{ StateChange: stateChange, @@ -180,6 +182,7 @@ func (scc *dataAnalysisCollector) Reset() { scc.cachedTxs = make(map[string]*transaction.Transaction) } +// Publish will export state changes func (scc *dataAnalysisCollector) Publish() error { stateChangesForTx, err := scc.getStateChangesForTxs() if err != nil { diff --git a/state/stateChanges/writeCollector.go b/state/stateChanges/writeCollector.go index 99213c247cb..188c8887160 100644 --- a/state/stateChanges/writeCollector.go +++ b/state/stateChanges/writeCollector.go @@ -79,6 +79,7 @@ func NewStateChangesCollector() *stateChangesCollector { } } +// AddSaveAccountStateChange adds a new state change for the save account operation func (scc *stateChangesCollector) AddSaveAccountStateChange(_, _ vmcommon.AccountHandler, stateChange StateChange) { scc.AddStateChange(stateChange) } @@ -134,6 +135,8 @@ func (scc *stateChangesCollector) Reset() { scc.stateChanges = make([]StateChange, 0) } +// AddTxHashToCollectedStateChanges will try to set txHash field to each state change +// if the field is not already set func (scc *stateChangesCollector) AddTxHashToCollectedStateChanges(txHash []byte, tx *transaction.Transaction) { scc.stateChangesMut.Lock() defer scc.stateChangesMut.Unlock() @@ -147,6 +150,7 @@ func (scc *stateChangesCollector) AddTxHashToCollectedStateChanges(txHash []byte } } +// SetIndexToLastStateChange will set index to the last state change func (scc *stateChangesCollector) SetIndexToLastStateChange(index int) error { if index > len(scc.stateChanges) || index < 0 { return ErrStateChangesIndexOutOfBounds @@ -160,6 +164,7 @@ func (scc *stateChangesCollector) SetIndexToLastStateChange(index int) error { return nil } +// RevertToIndex will revert to index func (scc *stateChangesCollector) RevertToIndex(index int) error { if index > len(scc.stateChanges) || index < 0 { return ErrStateChangesIndexOutOfBounds @@ -183,6 +188,7 @@ func (scc *stateChangesCollector) RevertToIndex(index int) error { return nil } +// Publish will export state changes func (scc *stateChangesCollector) Publish() error { stateChangesForTx, err := scc.getStateChangesForTxs() if err != nil { diff --git a/state/stateChangesCollector_test.go b/state/stateChangesCollector_test.go deleted file mode 100644 index 6853973bfcf..00000000000 --- a/state/stateChangesCollector_test.go +++ /dev/null @@ -1,234 +0,0 @@ -package state - -import ( - "fmt" - "strconv" - "testing" - - "github.com/multiversx/mx-chain-core-go/data/transaction" - "github.com/stretchr/testify/assert" - - "github.com/stretchr/testify/require" -) - -func TestNewStateChangesCollector(t *testing.T) { - t.Parallel() - - stateChangesCollector := NewStateChangesCollector() - require.False(t, stateChangesCollector.IsInterfaceNil()) -} - -func TestStateChangesCollector_AddStateChange(t *testing.T) { - t.Parallel() - - scc := NewStateChangesCollector() - assert.Equal(t, 0, len(scc.stateChanges)) - - numStateChanges := 10 - for i := 0; i < numStateChanges; i++ { - scc.AddStateChange(StateChangeDTO{}) - } - assert.Equal(t, numStateChanges, len(scc.stateChanges)) -} - -func TestStateChangesCollector_GetStateChanges(t *testing.T) { - t.Parallel() - - t.Run("getStateChanges with tx hash", func(t *testing.T) { - t.Parallel() - - scc := NewStateChangesCollector() - assert.Equal(t, 0, len(scc.stateChanges)) - assert.Equal(t, 0, len(scc.GetStateChanges())) - - numStateChanges := 10 - for i := 0; i < numStateChanges; i++ { - scc.AddStateChange(StateChangeDTO{ - MainTrieKey: []byte(strconv.Itoa(i)), - }) - } - assert.Equal(t, numStateChanges, len(scc.stateChanges)) - assert.Equal(t, 0, len(scc.GetStateChanges())) - scc.AddTxHashToCollectedStateChanges([]byte("txHash"), &transaction.Transaction{}) - assert.Equal(t, numStateChanges, len(scc.stateChanges)) - assert.Equal(t, 1, len(scc.GetStateChanges())) - assert.Equal(t, []byte("txHash"), scc.GetStateChanges()[0].TxHash) - assert.Equal(t, numStateChanges, len(scc.GetStateChanges()[0].StateChanges)) - - stateChangesForTx := scc.GetStateChanges() - assert.Equal(t, 1, len(stateChangesForTx)) - assert.Equal(t, []byte("txHash"), stateChangesForTx[0].TxHash) - for i := 0; i < len(stateChangesForTx[0].StateChanges); i++ { - assert.Equal(t, []byte(strconv.Itoa(i)), stateChangesForTx[0].StateChanges[i].MainTrieKey) - } - }) - - t.Run("getStateChanges without tx hash", func(t *testing.T) { - t.Parallel() - - scc := NewStateChangesCollector() - assert.Equal(t, 0, len(scc.stateChanges)) - assert.Equal(t, 0, len(scc.GetStateChanges())) - - numStateChanges := 10 - for i := 0; i < numStateChanges; i++ { - scc.AddStateChange(StateChangeDTO{ - MainTrieKey: []byte(strconv.Itoa(i)), - }) - } - assert.Equal(t, numStateChanges, len(scc.stateChanges)) - assert.Equal(t, 0, len(scc.GetStateChanges())) - - stateChangesForTx := scc.GetStateChanges() - assert.Equal(t, 0, len(stateChangesForTx)) - }) -} - -func TestStateChangesCollector_AddTxHashToCollectedStateChanges(t *testing.T) { - t.Parallel() - - scc := NewStateChangesCollector() - assert.Equal(t, 0, len(scc.stateChanges)) - assert.Equal(t, 0, len(scc.GetStateChanges())) - - scc.AddTxHashToCollectedStateChanges([]byte("txHash0"), &transaction.Transaction{}) - - stateChange := StateChangeDTO{ - MainTrieKey: []byte("mainTrieKey"), - MainTrieVal: []byte("mainTrieVal"), - DataTrieChanges: []DataTrieChange{{Key: []byte("dataTrieKey"), Val: []byte("dataTrieVal")}}, - } - scc.AddStateChange(stateChange) - - assert.Equal(t, 1, len(scc.stateChanges)) - assert.Equal(t, 0, len(scc.GetStateChanges())) - scc.AddTxHashToCollectedStateChanges([]byte("txHash"), &transaction.Transaction{}) - assert.Equal(t, 1, len(scc.stateChanges)) - assert.Equal(t, 1, len(scc.GetStateChanges())) - - stateChangesForTx := scc.GetStateChanges() - assert.Equal(t, 1, len(stateChangesForTx)) - assert.Equal(t, []byte("txHash"), stateChangesForTx[0].TxHash) - assert.Equal(t, 1, len(stateChangesForTx[0].StateChanges)) - assert.Equal(t, []byte("mainTrieKey"), stateChangesForTx[0].StateChanges[0].MainTrieKey) - assert.Equal(t, []byte("mainTrieVal"), stateChangesForTx[0].StateChanges[0].MainTrieVal) - assert.Equal(t, 1, len(stateChangesForTx[0].StateChanges[0].DataTrieChanges)) -} - -func TestStateChangesCollector_RevertToIndex_FailIfWrongIndex(t *testing.T) { - t.Parallel() - - scc := NewStateChangesCollector() - numStateChanges := len(scc.stateChanges) - - err := scc.RevertToIndex(-1) - require.Equal(t, ErrStateChangesIndexOutOfBounds, err) - - err = scc.RevertToIndex(numStateChanges + 1) - require.Equal(t, ErrStateChangesIndexOutOfBounds, err) -} - -func TestStateChangesCollector_RevertToIndex(t *testing.T) { - t.Parallel() - - scc := NewStateChangesCollector() - - numStateChanges := 10 - for i := 0; i < numStateChanges; i++ { - scc.AddStateChange(StateChangeDTO{}) - scc.SetIndexToLastStateChange(i) - } - scc.AddTxHashToCollectedStateChanges([]byte("txHash1"), &transaction.Transaction{}) - - for i := numStateChanges; i < numStateChanges*2; i++ { - scc.AddStateChange(StateChangeDTO{}) - scc.AddTxHashToCollectedStateChanges([]byte("txHash"+fmt.Sprintf("%d", i)), &transaction.Transaction{}) - } - scc.SetIndexToLastStateChange(numStateChanges) - - assert.Equal(t, numStateChanges*2, len(scc.stateChanges)) - - err := scc.RevertToIndex(numStateChanges) - require.Nil(t, err) - assert.Equal(t, numStateChanges*2-1, len(scc.stateChanges)) - - err = scc.RevertToIndex(numStateChanges - 1) - require.Nil(t, err) - assert.Equal(t, numStateChanges-1, len(scc.stateChanges)) - - err = scc.RevertToIndex(numStateChanges / 2) - require.Nil(t, err) - assert.Equal(t, numStateChanges/2, len(scc.stateChanges)) - - err = scc.RevertToIndex(1) - require.Nil(t, err) - assert.Equal(t, 1, len(scc.stateChanges)) - - err = scc.RevertToIndex(0) - require.Nil(t, err) - assert.Equal(t, 0, len(scc.stateChanges)) -} - -func TestStateChangesCollector_SetIndexToLastStateChange(t *testing.T) { - t.Parallel() - - t.Run("should fail if valid index", func(t *testing.T) { - t.Parallel() - - scc := NewStateChangesCollector() - - err := scc.SetIndexToLastStateChange(-1) - require.Equal(t, ErrStateChangesIndexOutOfBounds, err) - - numStateChanges := len(scc.stateChanges) - err = scc.SetIndexToLastStateChange(numStateChanges + 1) - require.Equal(t, ErrStateChangesIndexOutOfBounds, err) - }) - - t.Run("should work", func(t *testing.T) { - t.Parallel() - - scc := NewStateChangesCollector() - - numStateChanges := 10 - for i := 0; i < numStateChanges; i++ { - scc.AddStateChange(StateChangeDTO{}) - err := scc.SetIndexToLastStateChange(i) - require.Nil(t, err) - } - scc.AddTxHashToCollectedStateChanges([]byte("txHash1"), &transaction.Transaction{}) - - for i := numStateChanges; i < numStateChanges*2; i++ { - scc.AddStateChange(StateChangeDTO{}) - scc.AddTxHashToCollectedStateChanges([]byte("txHash"+fmt.Sprintf("%d", i)), &transaction.Transaction{}) - } - err := scc.SetIndexToLastStateChange(numStateChanges) - require.Nil(t, err) - - assert.Equal(t, numStateChanges*2, len(scc.stateChanges)) - }) -} - -func TestStateChangesCollector_Reset(t *testing.T) { - t.Parallel() - - scc := NewStateChangesCollector() - assert.Equal(t, 0, len(scc.stateChanges)) - - numStateChanges := 10 - for i := 0; i < numStateChanges; i++ { - scc.AddStateChange(StateChangeDTO{}) - } - scc.AddTxHashToCollectedStateChanges([]byte("txHash"), &transaction.Transaction{}) - for i := numStateChanges; i < numStateChanges*2; i++ { - scc.AddStateChange(StateChangeDTO{}) - } - assert.Equal(t, numStateChanges*2, len(scc.stateChanges)) - - assert.Equal(t, 1, len(scc.GetStateChanges())) - - scc.Reset() - assert.Equal(t, 0, len(scc.stateChanges)) - - assert.Equal(t, 0, len(scc.GetStateChanges())) -} diff --git a/state/storagePruningManager/storagePruningManager_test.go b/state/storagePruningManager/storagePruningManager_test.go index e10bf0250ac..d6dc193b940 100644 --- a/state/storagePruningManager/storagePruningManager_test.go +++ b/state/storagePruningManager/storagePruningManager_test.go @@ -10,6 +10,7 @@ import ( "github.com/multiversx/mx-chain-go/state/factory" "github.com/multiversx/mx-chain-go/state/iteratorChannelsProvider" "github.com/multiversx/mx-chain-go/state/lastSnapshotMarker" + "github.com/multiversx/mx-chain-go/state/stateChanges" "github.com/multiversx/mx-chain-go/state/storagePruningManager/evictionWaitingList" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" @@ -38,10 +39,14 @@ func getDefaultTrieAndAccountsDbAndStoragePruningManager() (common.Trie, *state. } ewl, _ := evictionWaitingList.NewMemoryEvictionWaitingList(ewlArgs) spm, _ := NewStoragePruningManager(ewl, generalCfg.PruningBufferLen) + + stateChangesCollector := stateChanges.NewStateChangesCollector() + argsAccCreator := factory.ArgsAccountCreator{ - Hasher: hasher, - Marshaller: marshaller, - EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + Hasher: hasher, + Marshaller: marshaller, + EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + StateChangesCollector: stateChangesCollector, } accCreator, _ := factory.NewAccountCreator(argsAccCreator) @@ -65,7 +70,7 @@ func getDefaultTrieAndAccountsDbAndStoragePruningManager() (common.Trie, *state. StoragePruningManager: spm, AddressConverter: &testscommon.PubkeyConverterMock{}, SnapshotsManager: snapshotsManager, - StateChangesCollector: state.NewStateChangesCollector(), + StateChangesCollector: stateChangesCollector, } adb, _ := state.NewAccountsDB(argsAccountsDB) diff --git a/state/trackableDataTrie/trackableDataTrie.go b/state/trackableDataTrie/trackableDataTrie.go index c83ad850034..f16964c79f2 100644 --- a/state/trackableDataTrie/trackableDataTrie.go +++ b/state/trackableDataTrie/trackableDataTrie.go @@ -55,6 +55,10 @@ func NewTrackableDataTrie( if check.IfNil(enableEpochsHandler) { return nil, state.ErrNilEnableEpochsHandler } + if check.IfNil(stateChangesCollector) { + return nil, state.ErrNilStateChangesCollector + } + err := core.CheckHandlerCompatibility(enableEpochsHandler, []core.EnableEpochFlag{ common.AutoBalanceDataTriesFlag, }) diff --git a/state/trackableDataTrie/trackableDataTrie_test.go b/state/trackableDataTrie/trackableDataTrie_test.go index db2d7f83b7a..c4e777851f3 100644 --- a/state/trackableDataTrie/trackableDataTrie_test.go +++ b/state/trackableDataTrie/trackableDataTrie_test.go @@ -11,6 +11,7 @@ import ( errorsCommon "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/state/dataTrieValue" + "github.com/multiversx/mx-chain-go/state/stateChanges" "github.com/multiversx/mx-chain-go/state/trackableDataTrie" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" @@ -21,13 +22,31 @@ import ( "github.com/stretchr/testify/assert" ) +func createDefaultTrackableDataTrie() state.DataTrieTracker { + tdt, _ := trackableDataTrie.NewTrackableDataTrie( + []byte("identifier"), + &hashingMocks.HasherMock{}, + &marshallerMock.MarshalizerMock{}, + enableEpochsHandlerMock.NewEnableEpochsHandlerStubWithNoFlagsDefined(), + stateChanges.NewStateChangesCollector(), + ) + + return tdt +} + func TestNewTrackableDataTrie(t *testing.T) { t.Parallel() t.Run("create with nil hasher", func(t *testing.T) { t.Parallel() - tdt, err := trackableDataTrie.NewTrackableDataTrie([]byte("identifier"), nil, &marshallerMock.MarshalizerMock{}, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tdt, err := trackableDataTrie.NewTrackableDataTrie( + []byte("identifier"), + nil, + &marshallerMock.MarshalizerMock{}, + &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + stateChanges.NewStateChangesCollector(), + ) assert.Equal(t, state.ErrNilHasher, err) assert.True(t, check.IfNil(tdt)) }) @@ -35,7 +54,13 @@ func TestNewTrackableDataTrie(t *testing.T) { t.Run("create with nil marshaller", func(t *testing.T) { t.Parallel() - tdt, err := trackableDataTrie.NewTrackableDataTrie([]byte("identifier"), &hashingMocks.HasherMock{}, nil, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tdt, err := trackableDataTrie.NewTrackableDataTrie( + []byte("identifier"), + &hashingMocks.HasherMock{}, + nil, + &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + stateChanges.NewStateChangesCollector(), + ) assert.Equal(t, state.ErrNilMarshalizer, err) assert.True(t, check.IfNil(tdt)) }) @@ -43,7 +68,13 @@ func TestNewTrackableDataTrie(t *testing.T) { t.Run("create with nil enableEpochsHandler", func(t *testing.T) { t.Parallel() - tdt, err := trackableDataTrie.NewTrackableDataTrie([]byte("identifier"), &hashingMocks.HasherMock{}, &marshallerMock.MarshalizerMock{}, nil) + tdt, err := trackableDataTrie.NewTrackableDataTrie( + []byte("identifier"), + &hashingMocks.HasherMock{}, + &marshallerMock.MarshalizerMock{}, + nil, + stateChanges.NewStateChangesCollector(), + ) assert.Equal(t, state.ErrNilEnableEpochsHandler, err) assert.True(t, check.IfNil(tdt)) }) @@ -55,7 +86,9 @@ func TestNewTrackableDataTrie(t *testing.T) { []byte("identifier"), &hashingMocks.HasherMock{}, &marshallerMock.MarshalizerMock{}, - enableEpochsHandlerMock.NewEnableEpochsHandlerStubWithNoFlagsDefined()) + enableEpochsHandlerMock.NewEnableEpochsHandlerStubWithNoFlagsDefined(), + stateChanges.NewStateChangesCollector(), + ) assert.True(t, errors.Is(err, core.ErrInvalidEnableEpochsHandler)) assert.True(t, check.IfNil(tdt)) }) @@ -63,7 +96,13 @@ func TestNewTrackableDataTrie(t *testing.T) { t.Run("should work", func(t *testing.T) { t.Parallel() - tdt, err := trackableDataTrie.NewTrackableDataTrie([]byte("identifier"), &hashingMocks.HasherMock{}, &marshallerMock.MarshalizerMock{}, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tdt, err := trackableDataTrie.NewTrackableDataTrie( + []byte("identifier"), + &hashingMocks.HasherMock{}, + &marshallerMock.MarshalizerMock{}, + &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + stateChanges.NewStateChangesCollector(), + ) assert.Nil(t, err) assert.False(t, check.IfNil(tdt)) }) @@ -75,7 +114,13 @@ func TestTrackableDataTrie_SaveKeyValue(t *testing.T) { t.Run("data too large", func(t *testing.T) { t.Parallel() - tdt, _ := trackableDataTrie.NewTrackableDataTrie([]byte("identifier"), &hashingMocks.HasherMock{}, &marshallerMock.MarshalizerMock{}, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tdt, _ := trackableDataTrie.NewTrackableDataTrie( + []byte("identifier"), + &hashingMocks.HasherMock{}, + &marshallerMock.MarshalizerMock{}, + &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + stateChanges.NewStateChangesCollector(), + ) err := tdt.SaveKeyValue([]byte("key"), make([]byte, core.MaxLeafSize+1)) assert.Equal(t, err, data.ErrLeafSizeTooBig) @@ -96,7 +141,13 @@ func TestTrackableDataTrie_SaveKeyValue(t *testing.T) { return nil, 0, nil }, } - tdt, _ := trackableDataTrie.NewTrackableDataTrie([]byte("identifier"), &hashingMocks.HasherMock{}, &marshallerMock.MarshalizerMock{}, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tdt, _ := trackableDataTrie.NewTrackableDataTrie( + []byte("identifier"), + &hashingMocks.HasherMock{}, + &marshallerMock.MarshalizerMock{}, + &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + stateChanges.NewStateChangesCollector(), + ) assert.NotNil(t, tdt) tdt.SetDataTrie(trie) @@ -129,13 +180,19 @@ func TestTrackableDataTrie_RetrieveValue(t *testing.T) { return nil, 0, nil }, } - tdt, _ := trackableDataTrie.NewTrackableDataTrie(identifier, &hashingMocks.HasherMock{}, &marshallerMock.MarshalizerMock{}, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tdt, _ := trackableDataTrie.NewTrackableDataTrie( + identifier, + &hashingMocks.HasherMock{}, + &marshallerMock.MarshalizerMock{}, + &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + stateChanges.NewStateChangesCollector(), + ) assert.NotNil(t, tdt) tdt.SetDataTrie(trie) valRecovered, _, err := tdt.RetrieveValue(key) - assert.Equal(t, retrievedTrieVal, valRecovered) assert.Nil(t, err) + assert.Equal(t, retrievedTrieVal, valRecovered) _ = tdt.SaveKeyValue(key, newTrieValue) valRecovered, _, err = tdt.RetrieveValue(key) @@ -146,7 +203,13 @@ func TestTrackableDataTrie_RetrieveValue(t *testing.T) { t.Run("nil data trie should err", func(t *testing.T) { t.Parallel() - tdt, err := trackableDataTrie.NewTrackableDataTrie([]byte("identifier"), &hashingMocks.HasherMock{}, &marshallerMock.MarshalizerMock{}, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tdt, err := trackableDataTrie.NewTrackableDataTrie( + []byte("identifier"), + &hashingMocks.HasherMock{}, + &marshallerMock.MarshalizerMock{}, + &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + stateChanges.NewStateChangesCollector(), + ) assert.Nil(t, err) assert.NotNil(t, tdt) @@ -176,7 +239,13 @@ func TestTrackableDataTrie_RetrieveValue(t *testing.T) { return flag == common.AutoBalanceDataTriesFlag }, } - tdt, _ := trackableDataTrie.NewTrackableDataTrie(identifier, &hashingMocks.HasherMock{}, &marshallerMock.MarshalizerMock{}, enableEpochsHandler) + tdt, _ := trackableDataTrie.NewTrackableDataTrie( + identifier, + &hashingMocks.HasherMock{}, + &marshallerMock.MarshalizerMock{}, + enableEpochsHandler, + stateChanges.NewStateChangesCollector(), + ) assert.NotNil(t, tdt) tdt.SetDataTrie(trie) @@ -211,7 +280,13 @@ func TestTrackableDataTrie_RetrieveValue(t *testing.T) { return false }, } - tdt, _ := trackableDataTrie.NewTrackableDataTrie(identifier, &hashingMocks.HasherMock{}, &marshallerMock.MarshalizerMock{}, enableEpochsHandler) + tdt, _ := trackableDataTrie.NewTrackableDataTrie( + identifier, + &hashingMocks.HasherMock{}, + &marshallerMock.MarshalizerMock{}, + enableEpochsHandler, + stateChanges.NewStateChangesCollector(), + ) assert.NotNil(t, tdt) tdt.SetDataTrie(trie) @@ -250,7 +325,13 @@ func TestTrackableDataTrie_RetrieveValue(t *testing.T) { return flag == common.AutoBalanceDataTriesFlag }, } - tdt, _ := trackableDataTrie.NewTrackableDataTrie(identifier, hasher, marshaller, enableEpochsHandler) + tdt, _ := trackableDataTrie.NewTrackableDataTrie( + identifier, + hasher, + marshaller, + enableEpochsHandler, + stateChanges.NewStateChangesCollector(), + ) assert.NotNil(t, tdt) tdt.SetDataTrie(trie) @@ -269,7 +350,13 @@ func TestTrackableDataTrie_RetrieveValue(t *testing.T) { return nil, 0, errExpected }, } - tdt, _ := trackableDataTrie.NewTrackableDataTrie([]byte("identifier"), &hashingMocks.HasherMock{}, &marshallerMock.MarshalizerMock{}, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tdt, _ := trackableDataTrie.NewTrackableDataTrie( + []byte("identifier"), + &hashingMocks.HasherMock{}, + &marshallerMock.MarshalizerMock{}, + &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + stateChanges.NewStateChangesCollector(), + ) assert.NotNil(t, tdt) tdt.SetDataTrie(trie) @@ -304,6 +391,7 @@ func TestTrackableDataTrie_RetrieveValue(t *testing.T) { hasher, marshaller, enableEpochsHandler, + stateChanges.NewStateChangesCollector(), ) assert.NotNil(t, tdt) tdt.SetDataTrie(trie) @@ -339,6 +427,7 @@ func TestTrackableDataTrie_RetrieveValue(t *testing.T) { hasher, marshaller, enableEpochsHandler, + stateChanges.NewStateChangesCollector(), ) assert.NotNil(t, tdt) tdt.SetDataTrie(trie) @@ -355,7 +444,13 @@ func TestTrackableDataTrie_SaveDirtyData(t *testing.T) { t.Run("no dirty data", func(t *testing.T) { t.Parallel() - tdt, _ := trackableDataTrie.NewTrackableDataTrie([]byte("identifier"), &hashingMocks.HasherMock{}, &marshallerMock.MarshalizerMock{}, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tdt, _ := trackableDataTrie.NewTrackableDataTrie( + []byte("identifier"), + &hashingMocks.HasherMock{}, + &marshallerMock.MarshalizerMock{}, + &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + stateChanges.NewStateChangesCollector(), + ) stateChanges, oldValues, err := tdt.SaveDirtyData(&trieMock.TrieStub{}) assert.Nil(t, err) @@ -380,7 +475,14 @@ func TestTrackableDataTrie_SaveDirtyData(t *testing.T) { }, nil }, } - tdt, _ := trackableDataTrie.NewTrackableDataTrie([]byte("identifier"), &hashingMocks.HasherMock{}, &marshallerMock.MarshalizerMock{}, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + + tdt, _ := trackableDataTrie.NewTrackableDataTrie( + []byte("identifier"), + &hashingMocks.HasherMock{}, + &marshallerMock.MarshalizerMock{}, + &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + stateChanges.NewStateChangesCollector(), + ) key := []byte("key") val := []byte("val") @@ -410,7 +512,13 @@ func TestTrackableDataTrie_SaveDirtyData(t *testing.T) { return flag == common.AutoBalanceDataTriesFlag }, } - tdt, _ := trackableDataTrie.NewTrackableDataTrie(identifier, hasher, marshaller, enableEpochsHandler) + tdt, _ := trackableDataTrie.NewTrackableDataTrie( + identifier, + hasher, + marshaller, + enableEpochsHandler, + stateChanges.NewStateChangesCollector(), + ) expectedKey := []byte("key") expectedVal := []byte("value") @@ -437,7 +545,7 @@ func TestTrackableDataTrie_SaveDirtyData(t *testing.T) { }, } - tdt, _ = trackableDataTrie.NewTrackableDataTrie(identifier, hasher, marshaller, enableEpochsHandler) + tdt, _ = trackableDataTrie.NewTrackableDataTrie(identifier, hasher, marshaller, enableEpochsHandler, stateChanges.NewStateChangesCollector()) tdt.SetDataTrie(trie) _ = tdt.SaveKeyValue(expectedKey, expectedVal) @@ -470,7 +578,13 @@ func TestTrackableDataTrie_SaveDirtyData(t *testing.T) { }, } - tdt, _ := trackableDataTrie.NewTrackableDataTrie(identifier, hasher, marshaller, enableEpochsHandler) + tdt, _ := trackableDataTrie.NewTrackableDataTrie( + identifier, + hasher, + marshaller, + enableEpochsHandler, + stateChanges.NewStateChangesCollector(), + ) expectedKey := []byte("key") val := []byte("value") @@ -495,7 +609,7 @@ func TestTrackableDataTrie_SaveDirtyData(t *testing.T) { }, } - tdt, _ = trackableDataTrie.NewTrackableDataTrie(identifier, hasher, marshaller, enableEpochsHandler) + tdt, _ = trackableDataTrie.NewTrackableDataTrie(identifier, hasher, marshaller, enableEpochsHandler, stateChanges.NewStateChangesCollector()) tdt.SetDataTrie(trie) _ = tdt.SaveKeyValue(expectedKey, val) @@ -522,7 +636,13 @@ func TestTrackableDataTrie_SaveDirtyData(t *testing.T) { return flag == common.AutoBalanceDataTriesFlag }, } - tdt, _ := trackableDataTrie.NewTrackableDataTrie(identifier, hasher, marshaller, enableEpochsHandler) + tdt, _ := trackableDataTrie.NewTrackableDataTrie( + identifier, + hasher, + marshaller, + enableEpochsHandler, + stateChanges.NewStateChangesCollector(), + ) expectedKey := []byte("key") newVal := []byte("value") @@ -549,7 +669,13 @@ func TestTrackableDataTrie_SaveDirtyData(t *testing.T) { }, } - tdt, _ = trackableDataTrie.NewTrackableDataTrie(identifier, hasher, marshaller, enableEpochsHandler) + tdt, _ = trackableDataTrie.NewTrackableDataTrie( + identifier, + hasher, + marshaller, + enableEpochsHandler, + stateChanges.NewStateChangesCollector(), + ) tdt.SetDataTrie(trie) _ = tdt.SaveKeyValue(expectedKey, newVal) @@ -576,7 +702,13 @@ func TestTrackableDataTrie_SaveDirtyData(t *testing.T) { return flag == common.AutoBalanceDataTriesFlag }, } - tdt, _ := trackableDataTrie.NewTrackableDataTrie(identifier, hasher, marshaller, enableEpochsHandler) + tdt, _ := trackableDataTrie.NewTrackableDataTrie( + identifier, + hasher, + marshaller, + enableEpochsHandler, + stateChanges.NewStateChangesCollector(), + ) expectedKey := []byte("key") newVal := []byte("value") @@ -598,7 +730,7 @@ func TestTrackableDataTrie_SaveDirtyData(t *testing.T) { }, } - tdt, _ = trackableDataTrie.NewTrackableDataTrie(identifier, hasher, marshaller, enableEpochsHandler) + tdt, _ = trackableDataTrie.NewTrackableDataTrie(identifier, hasher, marshaller, enableEpochsHandler, stateChanges.NewStateChangesCollector()) tdt.SetDataTrie(trie) _ = tdt.SaveKeyValue(expectedKey, newVal) @@ -628,7 +760,7 @@ func TestTrackableDataTrie_SaveDirtyData(t *testing.T) { }, } - tdt, _ := trackableDataTrie.NewTrackableDataTrie([]byte("identifier"), &hashingMocks.HasherMock{}, &marshallerMock.MarshalizerMock{}, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tdt, _ := trackableDataTrie.NewTrackableDataTrie([]byte("identifier"), &hashingMocks.HasherMock{}, &marshallerMock.MarshalizerMock{}, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, stateChanges.NewStateChangesCollector()) tdt.SetDataTrie(trie) _ = tdt.SaveKeyValue(expectedKey, val) @@ -653,7 +785,7 @@ func TestTrackableDataTrie_SaveDirtyData(t *testing.T) { }, } - tdt, _ := trackableDataTrie.NewTrackableDataTrie([]byte("identifier"), &hashingMocks.HasherMock{}, &marshallerMock.MarshalizerMock{}, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tdt, _ := trackableDataTrie.NewTrackableDataTrie([]byte("identifier"), &hashingMocks.HasherMock{}, &marshallerMock.MarshalizerMock{}, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, stateChanges.NewStateChangesCollector()) tdt.SetDataTrie(trie) _ = tdt.SaveKeyValue(expectedKey, nil) @@ -682,7 +814,7 @@ func TestTrackableDataTrie_SaveDirtyData(t *testing.T) { }, } - tdt, _ := trackableDataTrie.NewTrackableDataTrie([]byte("identifier"), &hashingMocks.HasherMock{}, &marshallerMock.MarshalizerMock{}, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tdt, _ := trackableDataTrie.NewTrackableDataTrie([]byte("identifier"), &hashingMocks.HasherMock{}, &marshallerMock.MarshalizerMock{}, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, stateChanges.NewStateChangesCollector()) tdt.SetDataTrie(trie) _ = tdt.SaveKeyValue(expectedKey, nil) @@ -719,7 +851,7 @@ func TestTrackableDataTrie_SaveDirtyData(t *testing.T) { return flag == common.AutoBalanceDataTriesFlag }, } - tdt, _ := trackableDataTrie.NewTrackableDataTrie([]byte("identifier"), &hashingMocks.HasherMock{}, &marshallerMock.MarshalizerMock{}, enableEpchs) + tdt, _ := trackableDataTrie.NewTrackableDataTrie([]byte("identifier"), &hashingMocks.HasherMock{}, &marshallerMock.MarshalizerMock{}, enableEpchs, stateChanges.NewStateChangesCollector()) tdt.SetDataTrie(trie) _ = tdt.SaveKeyValue(expectedKey, nil) @@ -757,7 +889,7 @@ func TestTrackableDataTrie_SaveDirtyData(t *testing.T) { return flag == common.AutoBalanceDataTriesFlag }, } - tdt, _ := trackableDataTrie.NewTrackableDataTrie([]byte("identifier"), &hashingMocks.HasherMock{}, &marshallerMock.MarshalizerMock{}, enableEpchs) + tdt, _ := trackableDataTrie.NewTrackableDataTrie([]byte("identifier"), &hashingMocks.HasherMock{}, &marshallerMock.MarshalizerMock{}, enableEpchs, stateChanges.NewStateChangesCollector()) tdt.SetDataTrie(trie) _ = tdt.SaveKeyValue(expectedKey, nil) @@ -787,6 +919,7 @@ func TestTrackableDataTrie_SaveDirtyData(t *testing.T) { hasher, marshaller, enableEpochsHandler, + stateChanges.NewStateChangesCollector(), ) expectedKey := []byte("key") @@ -838,6 +971,7 @@ func TestTrackableDataTrie_SaveDirtyData(t *testing.T) { hasher, marshaller, enableEpochsHandler, + stateChanges.NewStateChangesCollector(), ) key1 := "key1" @@ -903,7 +1037,13 @@ func TestTrackableDataTrie_MigrateDataTrieLeaves(t *testing.T) { t.Run("nil trie", func(t *testing.T) { t.Parallel() - tdt, _ := trackableDataTrie.NewTrackableDataTrie([]byte("identifier"), &hashingMocks.HasherMock{}, &marshallerMock.MarshalizerMock{}, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tdt, _ := trackableDataTrie.NewTrackableDataTrie( + []byte("identifier"), + &hashingMocks.HasherMock{}, + &marshallerMock.MarshalizerMock{}, + &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + stateChanges.NewStateChangesCollector(), + ) args := vmcommon.ArgsMigrateDataTrieLeaves{ OldVersion: core.NotSpecified, NewVersion: core.AutoBalanceEnabled, @@ -916,7 +1056,13 @@ func TestTrackableDataTrie_MigrateDataTrieLeaves(t *testing.T) { t.Run("nil trie migrator", func(t *testing.T) { t.Parallel() - tdt, _ := trackableDataTrie.NewTrackableDataTrie([]byte("identifier"), &hashingMocks.HasherMock{}, &marshallerMock.MarshalizerMock{}, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tdt, _ := trackableDataTrie.NewTrackableDataTrie( + []byte("identifier"), + &hashingMocks.HasherMock{}, + &marshallerMock.MarshalizerMock{}, + &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + stateChanges.NewStateChangesCollector(), + ) tdt.SetDataTrie(&trieMock.TrieStub{}) args := vmcommon.ArgsMigrateDataTrieLeaves{ @@ -938,7 +1084,13 @@ func TestTrackableDataTrie_MigrateDataTrieLeaves(t *testing.T) { }, } - tdt, _ := trackableDataTrie.NewTrackableDataTrie([]byte("identifier"), &hashingMocks.HasherMock{}, &marshallerMock.MarshalizerMock{}, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tdt, _ := trackableDataTrie.NewTrackableDataTrie( + []byte("identifier"), + &hashingMocks.HasherMock{}, + &marshallerMock.MarshalizerMock{}, + &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + stateChanges.NewStateChangesCollector(), + ) tdt.SetDataTrie(tr) args := vmcommon.ArgsMigrateDataTrieLeaves{ OldVersion: core.NotSpecified, @@ -987,7 +1139,13 @@ func TestTrackableDataTrie_MigrateDataTrieLeaves(t *testing.T) { }, } - tdt, _ := trackableDataTrie.NewTrackableDataTrie(address, &hashingMocks.HasherMock{}, &marshallerMock.MarshalizerMock{}, enableEpchs) + tdt, _ := trackableDataTrie.NewTrackableDataTrie( + address, + &hashingMocks.HasherMock{}, + &marshallerMock.MarshalizerMock{}, + enableEpchs, + stateChanges.NewStateChangesCollector(), + ) tdt.SetDataTrie(tr) args := vmcommon.ArgsMigrateDataTrieLeaves{ OldVersion: core.NotSpecified, @@ -1010,7 +1168,13 @@ func TestTrackableDataTrie_MigrateDataTrieLeaves(t *testing.T) { func TestTrackableDataTrie_SetAndGetDataTrie(t *testing.T) { t.Parallel() - tdt, _ := trackableDataTrie.NewTrackableDataTrie([]byte("identifier"), &hashingMocks.HasherMock{}, &marshallerMock.MarshalizerMock{}, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tdt, _ := trackableDataTrie.NewTrackableDataTrie( + []byte("identifier"), + &hashingMocks.HasherMock{}, + &marshallerMock.MarshalizerMock{}, + &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + stateChanges.NewStateChangesCollector(), + ) newTrie := &trieMock.TrieStub{} tdt.SetDataTrie(newTrie) From a3cc4af03fbb3f0d902fd9529ad62eb52c9a4a88 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Tue, 27 Aug 2024 16:58:26 +0300 Subject: [PATCH 18/55] fix integration tests --- factory/mock/stateComponentsHolderStub.go | 10 ++++++++++ factory/processing/blockProcessorCreator.go | 2 ++ factory/processing/blockProcessorCreator_test.go | 10 ++++++---- .../state/stateTrie/stateTrie_test.go | 15 +++++++++------ integrationTests/testInitializer.go | 7 ++++--- integrationTests/testProcessorNode.go | 2 ++ integrationTests/testSyncNode.go | 2 ++ .../vm/staking/componentsHolderCreator.go | 7 ++++--- .../vm/staking/metaBlockProcessorCreator.go | 2 ++ integrationTests/vm/wasm/wasmvm/mockContracts.go | 8 +++++--- testscommon/integrationtests/factory.go | 7 ++++--- 11 files changed, 50 insertions(+), 22 deletions(-) diff --git a/factory/mock/stateComponentsHolderStub.go b/factory/mock/stateComponentsHolderStub.go index c851fdc6dac..1775ad915f0 100644 --- a/factory/mock/stateComponentsHolderStub.go +++ b/factory/mock/stateComponentsHolderStub.go @@ -14,6 +14,7 @@ type StateComponentsHolderStub struct { TriesContainerCalled func() common.TriesHolder TrieStorageManagersCalled func() map[string]common.StorageManager MissingTrieNodesNotifierCalled func() common.MissingTrieNodesNotifier + StateChangesCollectorCalled func() state.StateChangesCollector } // PeerAccounts - @@ -79,6 +80,15 @@ func (s *StateComponentsHolderStub) MissingTrieNodesNotifier() common.MissingTri return nil } +// StateChangesCollector - +func (s *StateComponentsHolderStub) StateChangesCollector() state.StateChangesCollector { + if s.StateChangesCollectorCalled != nil { + return s.StateChangesCollectorCalled() + } + + return nil +} + // Close - func (s *StateComponentsHolderStub) Close() error { return nil diff --git a/factory/processing/blockProcessorCreator.go b/factory/processing/blockProcessorCreator.go index 7db9e20cf7d..c3618eff970 100644 --- a/factory/processing/blockProcessorCreator.go +++ b/factory/processing/blockProcessorCreator.go @@ -435,6 +435,7 @@ func (pcf *processComponentsFactory) newShardBlockProcessor( BlockProcessingCutoffHandler: blockProcessingCutoffHandler, ManagedPeersHolder: pcf.crypto.ManagedPeersHolder(), SentSignaturesTracker: sentSignaturesTracker, + StateChangesCollector: pcf.state.StateChangesCollector(), } arguments := block.ArgShardProcessor{ ArgBaseProcessor: argumentsBaseProcessor, @@ -871,6 +872,7 @@ func (pcf *processComponentsFactory) newMetaBlockProcessor( BlockProcessingCutoffHandler: blockProcessingCutoffhandler, ManagedPeersHolder: pcf.crypto.ManagedPeersHolder(), SentSignaturesTracker: sentSignaturesTracker, + StateChangesCollector: pcf.state.StateChangesCollector(), } esdtOwnerAddress, err := pcf.coreData.AddressPubKeyConverter().Decode(pcf.systemSCConfig.ESDTSystemSCConfig.OwnerAddress) diff --git a/factory/processing/blockProcessorCreator_test.go b/factory/processing/blockProcessorCreator_test.go index 92941e4778e..1f271db8bdd 100644 --- a/factory/processing/blockProcessorCreator_test.go +++ b/factory/processing/blockProcessorCreator_test.go @@ -16,6 +16,7 @@ import ( "github.com/multiversx/mx-chain-go/state/accounts" disabledState "github.com/multiversx/mx-chain-go/state/disabled" factoryState "github.com/multiversx/mx-chain-go/state/factory" + "github.com/multiversx/mx-chain-go/state/stateChanges" "github.com/multiversx/mx-chain-go/state/storagePruningManager/disabled" "github.com/multiversx/mx-chain-go/testscommon" componentsMock "github.com/multiversx/mx-chain-go/testscommon/components" @@ -100,9 +101,10 @@ func Test_newBlockProcessorCreatorForMeta(t *testing.T) { trieStorageManagers[dataRetriever.PeerAccountsUnit.String()] = storageManagerPeer argsAccCreator := factoryState.ArgsAccountCreator{ - Hasher: coreComponents.Hasher(), - Marshaller: coreComponents.InternalMarshalizer(), - EnableEpochsHandler: coreComponents.EnableEpochsHandler(), + Hasher: coreComponents.Hasher(), + Marshaller: coreComponents.InternalMarshalizer(), + EnableEpochsHandler: coreComponents.EnableEpochsHandler(), + StateChangesCollector: stateChanges.NewStateChangesCollector(), } accCreator, _ := factoryState.NewAccountCreator(argsAccCreator) @@ -208,7 +210,7 @@ func createAccountAdapter( StoragePruningManager: disabled.NewDisabledStoragePruningManager(), AddressConverter: &testscommon.PubkeyConverterMock{}, SnapshotsManager: disabledState.NewDisabledSnapshotsManager(), - StateChangesCollector: state.NewStateChangesCollector(), + StateChangesCollector: disabledState.NewDisabledStateChangesCollector(), } adb, err := state.NewAccountsDB(args) if err != nil { diff --git a/integrationTests/state/stateTrie/stateTrie_test.go b/integrationTests/state/stateTrie/stateTrie_test.go index 0ac4b86c7ee..ab58bc57b8a 100644 --- a/integrationTests/state/stateTrie/stateTrie_test.go +++ b/integrationTests/state/stateTrie/stateTrie_test.go @@ -38,6 +38,7 @@ import ( "github.com/multiversx/mx-chain-go/state/factory" "github.com/multiversx/mx-chain-go/state/iteratorChannelsProvider" "github.com/multiversx/mx-chain-go/state/lastSnapshotMarker" + "github.com/multiversx/mx-chain-go/state/stateChanges" "github.com/multiversx/mx-chain-go/state/storagePruningManager" "github.com/multiversx/mx-chain-go/state/storagePruningManager/evictionWaitingList" "github.com/multiversx/mx-chain-go/storage" @@ -1058,9 +1059,10 @@ func createAccounts( tr, _ := trie.NewTrie(trieStorage, integrationTests.TestMarshalizer, integrationTests.TestHasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, maxTrieLevelInMemory) spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 10) argsAccCreator := factory.ArgsAccountCreator{ - Hasher: integrationTests.TestHasher, - Marshaller: integrationTests.TestMarshalizer, - EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + Hasher: integrationTests.TestHasher, + Marshaller: integrationTests.TestMarshalizer, + EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + StateChangesCollector: stateChanges.NewStateChangesCollector(), } accCreator, _ := factory.NewAccountCreator(argsAccCreator) snapshotsManager, _ := state.NewSnapshotsManager(state.ArgsNewSnapshotsManager{ @@ -2731,9 +2733,10 @@ func createAccountsDBTestSetup() *state.AccountsDB { tr, _ := trie.NewTrie(trieStorage, integrationTests.TestMarshalizer, integrationTests.TestHasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, maxTrieLevelInMemory) spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 10) argsAccCreator := factory.ArgsAccountCreator{ - Hasher: integrationTests.TestHasher, - Marshaller: integrationTests.TestMarshalizer, - EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + Hasher: integrationTests.TestHasher, + Marshaller: integrationTests.TestMarshalizer, + EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + StateChangesCollector: stateChanges.NewStateChangesCollector(), } accCreator, _ := factory.NewAccountCreator(argsAccCreator) diff --git a/integrationTests/testInitializer.go b/integrationTests/testInitializer.go index df0863ebb6a..860e7afb2d2 100644 --- a/integrationTests/testInitializer.go +++ b/integrationTests/testInitializer.go @@ -498,9 +498,10 @@ func getAccountFactory(accountType Type, enableEpochsHandler common.EnableEpochs switch accountType { case UserAccount: argsAccCreator := factory.ArgsAccountCreator{ - Hasher: TestHasher, - Marshaller: TestMarshalizer, - EnableEpochsHandler: enableEpochsHandler, + Hasher: TestHasher, + Marshaller: TestMarshalizer, + EnableEpochsHandler: enableEpochsHandler, + StateChangesCollector: disabled.NewDisabledStateChangesCollector(), } return factory.NewAccountCreator(argsAccCreator) case ValidatorAccount: diff --git a/integrationTests/testProcessorNode.go b/integrationTests/testProcessorNode.go index b52cc3585a8..962812a9851 100644 --- a/integrationTests/testProcessorNode.go +++ b/integrationTests/testProcessorNode.go @@ -98,6 +98,7 @@ import ( "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/state/blockInfoProviders" + stateDisabled "github.com/multiversx/mx-chain-go/state/disabled" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/cache" "github.com/multiversx/mx-chain-go/storage/storageunit" @@ -2211,6 +2212,7 @@ func (tpn *TestProcessorNode) initBlockProcessor() { BlockProcessingCutoffHandler: &testscommon.BlockProcessingCutoffStub{}, ManagedPeersHolder: &testscommon.ManagedPeersHolderStub{}, SentSignaturesTracker: &testscommon.SentSignatureTrackerStub{}, + StateChangesCollector: stateDisabled.NewDisabledStateChangesCollector(), } if check.IfNil(tpn.EpochStartNotifier) { diff --git a/integrationTests/testSyncNode.go b/integrationTests/testSyncNode.go index b28d5e3f953..80329c3eb6b 100644 --- a/integrationTests/testSyncNode.go +++ b/integrationTests/testSyncNode.go @@ -15,6 +15,7 @@ import ( "github.com/multiversx/mx-chain-go/process/block/bootstrapStorage" "github.com/multiversx/mx-chain-go/process/sync" "github.com/multiversx/mx-chain-go/state" + stateDisabled "github.com/multiversx/mx-chain-go/state/disabled" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/testscommon/dblookupext" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" @@ -105,6 +106,7 @@ func (tpn *TestProcessorNode) initBlockProcessorWithSync() { BlockProcessingCutoffHandler: &testscommon.BlockProcessingCutoffStub{}, ManagedPeersHolder: &testscommon.ManagedPeersHolderStub{}, SentSignaturesTracker: &testscommon.SentSignatureTrackerStub{}, + StateChangesCollector: stateDisabled.NewDisabledStateChangesCollector(), } if tpn.ShardCoordinator.SelfId() == core.MetachainShardId { diff --git a/integrationTests/vm/staking/componentsHolderCreator.go b/integrationTests/vm/staking/componentsHolderCreator.go index 251e7eec511..8be321e03e3 100644 --- a/integrationTests/vm/staking/componentsHolderCreator.go +++ b/integrationTests/vm/staking/componentsHolderCreator.go @@ -158,9 +158,10 @@ func createStateComponents(coreComponents factory.CoreComponentsHolder) factory. trieFactoryManager, _ := trie.NewTrieStorageManagerWithoutPruning(tsm) argsAccCreator := stateFactory.ArgsAccountCreator{ - Hasher: coreComponents.Hasher(), - Marshaller: coreComponents.InternalMarshalizer(), - EnableEpochsHandler: coreComponents.EnableEpochsHandler(), + Hasher: coreComponents.Hasher(), + Marshaller: coreComponents.InternalMarshalizer(), + EnableEpochsHandler: coreComponents.EnableEpochsHandler(), + StateChangesCollector: disabledState.NewDisabledStateChangesCollector(), } accCreator, _ := stateFactory.NewAccountCreator(argsAccCreator) diff --git a/integrationTests/vm/staking/metaBlockProcessorCreator.go b/integrationTests/vm/staking/metaBlockProcessorCreator.go index 759458cf30e..ab96a156e9e 100644 --- a/integrationTests/vm/staking/metaBlockProcessorCreator.go +++ b/integrationTests/vm/staking/metaBlockProcessorCreator.go @@ -22,6 +22,7 @@ import ( "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" "github.com/multiversx/mx-chain-go/state" + "github.com/multiversx/mx-chain-go/state/disabled" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/testscommon/dblookupext" factory2 "github.com/multiversx/mx-chain-go/testscommon/factory" @@ -104,6 +105,7 @@ func createMetaBlockProcessor( ManagedPeersHolder: &testscommon.ManagedPeersHolderStub{}, BlockProcessingCutoffHandler: &testscommon.BlockProcessingCutoffStub{}, SentSignaturesTracker: &testscommon.SentSignatureTrackerStub{}, + StateChangesCollector: disabled.NewDisabledStateChangesCollector(), }, SCToProtocol: stakingToPeer, PendingMiniBlocksHandler: &mock.PendingMiniBlocksHandlerStub{}, diff --git a/integrationTests/vm/wasm/wasmvm/mockContracts.go b/integrationTests/vm/wasm/wasmvm/mockContracts.go index 4e1b2b2b2c2..d8eeccdbd25 100644 --- a/integrationTests/vm/wasm/wasmvm/mockContracts.go +++ b/integrationTests/vm/wasm/wasmvm/mockContracts.go @@ -13,6 +13,7 @@ import ( "github.com/multiversx/mx-chain-go/process/factory" "github.com/multiversx/mx-chain-go/state" stateFactory "github.com/multiversx/mx-chain-go/state/factory" + "github.com/multiversx/mx-chain-go/state/stateChanges" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" "github.com/multiversx/mx-chain-go/testscommon/marshallerMock" @@ -106,9 +107,10 @@ func GetAddressForNewAccountOnWalletAndNodeWithVM( address := net.NewAddressWithVM(wallet, vmType) argsAccCreation := stateFactory.ArgsAccountCreator{ - Hasher: &hashingMocks.HasherMock{}, - Marshaller: &marshallerMock.MarshalizerMock{}, - EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + Hasher: &hashingMocks.HasherMock{}, + Marshaller: &marshallerMock.MarshalizerMock{}, + EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + StateChangesCollector: stateChanges.NewStateChangesCollector(), } accountFactory, _ := stateFactory.NewAccountCreator(argsAccCreation) diff --git a/testscommon/integrationtests/factory.go b/testscommon/integrationtests/factory.go index 3211b79b48b..f8e705a6d11 100644 --- a/testscommon/integrationtests/factory.go +++ b/testscommon/integrationtests/factory.go @@ -106,9 +106,10 @@ func CreateAccountsDB(db storage.Storer, enableEpochs common.EnableEpochsHandler spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 10) argsAccCreator := accountFactory.ArgsAccountCreator{ - Hasher: TestHasher, - Marshaller: TestMarshalizer, - EnableEpochsHandler: enableEpochs, + Hasher: TestHasher, + Marshaller: TestMarshalizer, + EnableEpochsHandler: enableEpochs, + StateChangesCollector: disabled.NewDisabledStateChangesCollector(), } accCreator, _ := accountFactory.NewAccountCreator(argsAccCreator) From 55ebfd64aba9d4288dafc9049b98cdbaef31f53c Mon Sep 17 00:00:00 2001 From: ssd04 Date: Wed, 28 Aug 2024 13:46:39 +0300 Subject: [PATCH 19/55] fix more tests --- factory/api/apiResolverFactory_test.go | 4 ++++ integrationTests/state/stateTrie/stateTrie_test.go | 4 ++-- node/node_test.go | 3 ++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/factory/api/apiResolverFactory_test.go b/factory/api/apiResolverFactory_test.go index e929d66e701..ec9a39a4aba 100644 --- a/factory/api/apiResolverFactory_test.go +++ b/factory/api/apiResolverFactory_test.go @@ -19,6 +19,7 @@ import ( "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/sync/disabled" "github.com/multiversx/mx-chain-go/state" + stateDisabled "github.com/multiversx/mx-chain-go/state/disabled" "github.com/multiversx/mx-chain-go/testscommon" componentsMock "github.com/multiversx/mx-chain-go/testscommon/components" "github.com/multiversx/mx-chain-go/testscommon/dataRetriever" @@ -343,6 +344,9 @@ func createMockSCQueryElementArgs() api.SCQueryElementArgs { PeerAccountsCalled: func() state.AccountsAdapter { return &stateMocks.AccountsStub{} }, + StateChangesCollectorCalled: func() state.StateChangesCollector { + return stateDisabled.NewDisabledStateChangesCollector() + }, }, StatusCoreComponents: &factory.StatusCoreComponentsStub{ AppStatusHandlerCalled: func() core.AppStatusHandler { diff --git a/integrationTests/state/stateTrie/stateTrie_test.go b/integrationTests/state/stateTrie/stateTrie_test.go index ab58bc57b8a..97c782a570d 100644 --- a/integrationTests/state/stateTrie/stateTrie_test.go +++ b/integrationTests/state/stateTrie/stateTrie_test.go @@ -1084,7 +1084,7 @@ func createAccounts( StoragePruningManager: spm, AddressConverter: &testscommon.PubkeyConverterMock{}, SnapshotsManager: snapshotsManager, - StateChangesCollector: state.NewStateChangesCollector(), + StateChangesCollector: stateChanges.NewStateChangesCollector(), } adb, _ := state.NewAccountsDB(argsAccountsDB) @@ -2760,7 +2760,7 @@ func createAccountsDBTestSetup() *state.AccountsDB { StoragePruningManager: spm, AddressConverter: &testscommon.PubkeyConverterMock{}, SnapshotsManager: snapshotsManager, - StateChangesCollector: state.NewStateChangesCollector(), + StateChangesCollector: stateChanges.NewStateChangesCollector(), } adb, _ := state.NewAccountsDB(argsAccountsDB) diff --git a/node/node_test.go b/node/node_test.go index 2cde11d08a0..6f2d46fa05f 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -45,6 +45,7 @@ import ( "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/state/accounts" + "github.com/multiversx/mx-chain-go/state/disabled" "github.com/multiversx/mx-chain-go/state/parsers" "github.com/multiversx/mx-chain-go/state/trackableDataTrie" "github.com/multiversx/mx-chain-go/storage" @@ -100,7 +101,7 @@ func createMockPubkeyConverter() *testscommon.PubkeyConverterMock { func createAcc(address []byte) state.UserAccountHandler { dtlp, _ := parsers.NewDataTrieLeafParser(address, &marshallerMock.MarshalizerMock{}, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) - dtt, _ := trackableDataTrie.NewTrackableDataTrie(address, &testscommon.HasherStub{}, &marshallerMock.MarshalizerMock{}, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + dtt, _ := trackableDataTrie.NewTrackableDataTrie(address, &testscommon.HasherStub{}, &marshallerMock.MarshalizerMock{}, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, disabled.NewDisabledStateChangesCollector()) acc, _ := accounts.NewUserAccount(address, dtt, dtlp) return acc From 03102683d950a530a0faea4e0c75cf3a88363f54 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Mon, 2 Sep 2024 11:51:22 +0300 Subject: [PATCH 20/55] added state changes collector to hostDriver. --- factory/status/statusComponents.go | 8 +++-- outport/factory/hostDriverFactory.go | 15 +++++--- outport/host/driver.go | 39 ++++++++++++++------- outport/host/errors.go | 3 ++ outport/interface.go | 3 ++ outport/mock/driverStub.go | 11 ++++++ outport/notifier/eventNotifier.go | 5 +++ outport/outport.go | 26 ++++++++++++++ state/accountsDB.go | 8 +++-- state/interface.go | 4 ++- state/stateChanges/dataAnalysisCollector.go | 7 +++- state/stateChanges/writeCollector.go | 4 +++ 12 files changed, 107 insertions(+), 26 deletions(-) diff --git a/factory/status/statusComponents.go b/factory/status/statusComponents.go index b8c5cf29163..cea175fdfaf 100644 --- a/factory/status/statusComponents.go +++ b/factory/status/statusComponents.go @@ -9,6 +9,8 @@ import ( outportCore "github.com/multiversx/mx-chain-core-go/data/outport" factoryMarshalizer "github.com/multiversx/mx-chain-core-go/marshal/factory" indexerFactory "github.com/multiversx/mx-chain-es-indexer-go/process/factory" + logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/statistics" swVersionFactory "github.com/multiversx/mx-chain-go/common/statistics/softwareVersion/factory" @@ -23,7 +25,6 @@ import ( "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" - logger "github.com/multiversx/mx-chain-logger-go" ) type statusComponents struct { @@ -286,8 +287,9 @@ func (scf *statusComponentsFactory) makeHostDriversArgs() ([]outportDriverFactor } argsHostDriverFactorySlice = append(argsHostDriverFactorySlice, outportDriverFactory.ArgsHostDriverFactory{ - Marshaller: marshaller, - HostConfig: hostConfig, + Marshaller: marshaller, + HostConfig: hostConfig, + StateChangesCollector: scf.stateComponents.StateChangesCollector(), }) } diff --git a/outport/factory/hostDriverFactory.go b/outport/factory/hostDriverFactory.go index 42dfdeadcf7..895c5577688 100644 --- a/outport/factory/hostDriverFactory.go +++ b/outport/factory/hostDriverFactory.go @@ -4,15 +4,19 @@ import ( "github.com/multiversx/mx-chain-communication-go/websocket/data" "github.com/multiversx/mx-chain-communication-go/websocket/factory" "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/multiversx/mx-chain-go/config" "github.com/multiversx/mx-chain-go/outport" "github.com/multiversx/mx-chain-go/outport/host" + "github.com/multiversx/mx-chain-go/state" + logger "github.com/multiversx/mx-chain-logger-go" ) type ArgsHostDriverFactory struct { - HostConfig config.HostDriversConfig - Marshaller marshal.Marshalizer + HostConfig config.HostDriversConfig + Marshaller marshal.Marshalizer + StateChangesCollector state.StateChangesCollector } var log = logger.GetOrCreate("outport/factory/hostdriver") @@ -38,8 +42,9 @@ func CreateHostDriver(args ArgsHostDriverFactory) (outport.Driver, error) { } return host.NewHostDriver(host.ArgsHostDriver{ - Marshaller: args.Marshaller, - SenderHost: wsHost, - Log: log, + Marshaller: args.Marshaller, + SenderHost: wsHost, + Log: log, + StateChangesCollector: args.StateChangesCollector, }) } diff --git a/outport/host/driver.go b/outport/host/driver.go index d15771f28c4..e73871c2c71 100644 --- a/outport/host/driver.go +++ b/outport/host/driver.go @@ -8,21 +8,25 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/marshal" + + "github.com/multiversx/mx-chain-go/state" ) // ArgsHostDriver holds the arguments needed for creating a new hostDriver type ArgsHostDriver struct { - Marshaller marshal.Marshalizer - SenderHost SenderHost - Log core.Logger + Marshaller marshal.Marshalizer + SenderHost SenderHost + Log core.Logger + StateChangesCollector state.StateChangesCollector } type hostDriver struct { - marshaller marshal.Marshalizer - senderHost SenderHost - isClosed atomic.Flag - log core.Logger - payloadProc payloadProcessorHandler + marshaller marshal.Marshalizer + senderHost SenderHost + isClosed atomic.Flag + log core.Logger + payloadProc payloadProcessorHandler + stateChangesCollector state.StateChangesCollector } // NewHostDriver will create a new instance of hostDriver @@ -36,6 +40,9 @@ func NewHostDriver(args ArgsHostDriver) (*hostDriver, error) { if check.IfNil(args.Log) { return nil, core.ErrNilLogger } + if check.IfNil(args.StateChangesCollector) { + return nil, ErrNilStateChangesCollector + } payloadProc, err := newPayloadProcessor(args.Log) if err != nil { @@ -48,11 +55,12 @@ func NewHostDriver(args ArgsHostDriver) (*hostDriver, error) { } return &hostDriver{ - marshaller: args.Marshaller, - senderHost: args.SenderHost, - log: args.Log, - isClosed: atomic.Flag{}, - payloadProc: payloadProc, + marshaller: args.Marshaller, + senderHost: args.SenderHost, + log: args.Log, + isClosed: atomic.Flag{}, + payloadProc: payloadProc, + stateChangesCollector: args.StateChangesCollector, }, nil } @@ -124,6 +132,11 @@ func (o *hostDriver) SetCurrentSettings(config outport.OutportConfig) error { return o.handleAction(&config, outport.TopicSettings) } +func (o *hostDriver) SaveStateChanges() error { + //TODO: add this as a constant + return o.handleAction(o.stateChangesCollector.GetStateChanges(), "SaveStateChanges") +} + // Close will handle the closing of the outport driver web socket sender func (o *hostDriver) Close() error { o.isClosed.SetValue(true) diff --git a/outport/host/errors.go b/outport/host/errors.go index de45f08acef..8b092a1a102 100644 --- a/outport/host/errors.go +++ b/outport/host/errors.go @@ -7,3 +7,6 @@ var ErrHostIsClosed = errors.New("server is closed") // ErrNilHost signals that a nil host has been provided var ErrNilHost = errors.New("nil host provided") + +// ErrNilStateChangesCollector signals that a nil state change collector has been provided. +var ErrNilStateChangesCollector = errors.New("nil state changes collector provided") diff --git a/outport/interface.go b/outport/interface.go index 768920a7709..d4cd4eb205d 100644 --- a/outport/interface.go +++ b/outport/interface.go @@ -3,6 +3,7 @@ package outport import ( outportcore "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/multiversx/mx-chain-go/outport/process" ) @@ -15,6 +16,7 @@ type Driver interface { SaveValidatorsPubKeys(validatorsPubKeys *outportcore.ValidatorsPubKeys) error SaveValidatorsRating(validatorsRating *outportcore.ValidatorsRating) error SaveAccounts(accounts *outportcore.Accounts) error + SaveStateChanges() error FinalizedBlock(finalizedBlock *outportcore.FinalizedBlock) error GetMarshaller() marshal.Marshalizer SetCurrentSettings(config outportcore.OutportConfig) error @@ -32,6 +34,7 @@ type OutportHandler interface { SaveValidatorsPubKeys(validatorsPubKeys *outportcore.ValidatorsPubKeys) SaveValidatorsRating(validatorsRating *outportcore.ValidatorsRating) SaveAccounts(accounts *outportcore.Accounts) + //SaveStateChanges() FinalizedBlock(finalizedBlock *outportcore.FinalizedBlock) SubscribeDriver(driver Driver) error HasDrivers() bool diff --git a/outport/mock/driverStub.go b/outport/mock/driverStub.go index e142eb9f5aa..2b7cfe35159 100644 --- a/outport/mock/driverStub.go +++ b/outport/mock/driverStub.go @@ -3,6 +3,7 @@ package mock import ( outportcore "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/multiversx/mx-chain-go/testscommon/marshallerMock" ) @@ -18,6 +19,7 @@ type DriverStub struct { CloseCalled func() error RegisterHandlerCalled func(handlerFunction func() error, topic string) error SetCurrentSettingsCalled func(config outportcore.OutportConfig) error + SaveStateChangesCalled func() error } // SaveBlock - @@ -106,6 +108,15 @@ func (d *DriverStub) RegisterHandler(handlerFunction func() error, topic string) return nil } +// SaveStateChanges - +func (d *DriverStub) SaveStateChanges() error { + if d.SaveStateChangesCalled != nil { + return d.SaveStateChangesCalled() + } + + return nil +} + // Close - func (d *DriverStub) Close() error { if d.CloseCalled != nil { diff --git a/outport/notifier/eventNotifier.go b/outport/notifier/eventNotifier.go index 828b027c88e..247fd7e3d23 100644 --- a/outport/notifier/eventNotifier.go +++ b/outport/notifier/eventNotifier.go @@ -115,6 +115,11 @@ func (en *eventNotifier) SaveAccounts(_ *outport.Accounts) error { return nil } +// SaveStateChanges does nothing +func (en *eventNotifier) SaveStateChanges() error { + return nil +} + // GetMarshaller returns internal marshaller func (en *eventNotifier) GetMarshaller() marshal.Marshalizer { return en.marshalizer diff --git a/outport/outport.go b/outport/outport.go index edcecc0691a..17773f1639e 100644 --- a/outport/outport.go +++ b/outport/outport.go @@ -310,6 +310,32 @@ func (o *outport) saveAccountsBlocking(accounts *outportcore.Accounts, driver Dr } } +func (o *outport) SaveStateChanges() { + o.mutex.RLock() + defer o.mutex.RUnlock() + + for _, driver := range o.drivers { + o.saveStateChangesBlocking(driver) + } +} + +func (o *outport) saveStateChangesBlocking(driver Driver) { + ch := o.monitorCompletionOnDriver("finalizedBlockBlocking", driver) + defer close(ch) + + for { + err := driver.SaveStateChanges() + if err == nil { + return + } + + log.Error("error calling SaveStateChanges, will retry", + "driver", driverString(driver), + "retrial in", o.retrialInterval, + "error", err) + } +} + // FinalizedBlock will call all the drivers that a block is finalized func (o *outport) FinalizedBlock(finalizedBlock *outportcore.FinalizedBlock) { o.mutex.RLock() diff --git a/state/accountsDB.go b/state/accountsDB.go index 7700521ebe0..56bb820f42d 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -16,6 +16,9 @@ import ( "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" + logger "github.com/multiversx/mx-chain-logger-go" + vmcommon "github.com/multiversx/mx-chain-vm-common-go" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/errChan" "github.com/multiversx/mx-chain-go/common/holders" @@ -23,8 +26,6 @@ import ( "github.com/multiversx/mx-chain-go/state/stateChanges" "github.com/multiversx/mx-chain-go/trie/keyBuilder" "github.com/multiversx/mx-chain-go/trie/statistics" - logger "github.com/multiversx/mx-chain-logger-go" - vmcommon "github.com/multiversx/mx-chain-vm-common-go" ) const ( @@ -915,7 +916,8 @@ func (adb *AccountsDB) commit() ([]byte, error) { return nil, err } - adb.stateChangesCollector.Reset() + //TODO: discuss the workflow. If reset here, the outport driver won't be able to pick up the changes. + //adb.stateChangesCollector.Reset() oldHashes := make(common.ModifiedHashes) newHashes := make(common.ModifiedHashes) diff --git a/state/interface.go b/state/interface.go index ae2100aa7a3..80e3a6baca3 100644 --- a/state/interface.go +++ b/state/interface.go @@ -7,9 +7,10 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data/api" "github.com/multiversx/mx-chain-core-go/data/transaction" + vmcommon "github.com/multiversx/mx-chain-vm-common-go" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/state/stateChanges" - vmcommon "github.com/multiversx/mx-chain-vm-common-go" ) // AccountFactory creates an account of different types @@ -363,5 +364,6 @@ type StateChangesCollector interface { SetIndexToLastStateChange(index int) error RevertToIndex(index int) error Publish() error + GetStateChanges() []stateChanges.StateChange IsInterfaceNil() bool } diff --git a/state/stateChanges/dataAnalysisCollector.go b/state/stateChanges/dataAnalysisCollector.go index dc5d5e62a4f..85b8fd39638 100644 --- a/state/stateChanges/dataAnalysisCollector.go +++ b/state/stateChanges/dataAnalysisCollector.go @@ -8,8 +8,9 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data/transaction" - "github.com/multiversx/mx-chain-go/storage" vmcommon "github.com/multiversx/mx-chain-vm-common-go" + + "github.com/multiversx/mx-chain-go/storage" ) const ( @@ -204,6 +205,10 @@ func (scc *dataAnalysisCollector) Publish() error { return nil } +func (scc *dataAnalysisCollector) GetStateChanges() []StateChange { + return scc.stateChanges +} + // IsInterfaceNil returns true if there is no value under the interface func (scc *dataAnalysisCollector) IsInterfaceNil() bool { return scc == nil diff --git a/state/stateChanges/writeCollector.go b/state/stateChanges/writeCollector.go index 188c8887160..89bfad57617 100644 --- a/state/stateChanges/writeCollector.go +++ b/state/stateChanges/writeCollector.go @@ -213,6 +213,10 @@ func printStateChanges(stateChanges []StateChangesForTx) { } } +func (scc *stateChangesCollector) GetStateChanges() []StateChange { + return scc.stateChanges +} + // IsInterfaceNil returns true if there is no value under the interface func (scc *stateChangesCollector) IsInterfaceNil() bool { return scc == nil From 682e4178be307c075f794902193610fc5bec1043 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Mon, 2 Sep 2024 12:30:02 +0300 Subject: [PATCH 21/55] some fixes. --- outport/disabled/disabledOutport.go | 6 ++++++ outport/factory/outportFactory.go | 7 +++++-- outport/interface.go | 2 +- outport/outport.go | 3 ++- process/block/metablock.go | 6 +++++- state/disabled/disabledStateChangesCollector.go | 7 ++++++- 6 files changed, 25 insertions(+), 6 deletions(-) diff --git a/outport/disabled/disabledOutport.go b/outport/disabled/disabledOutport.go index 97be7894c9b..c4e861ec210 100644 --- a/outport/disabled/disabledOutport.go +++ b/outport/disabled/disabledOutport.go @@ -2,6 +2,7 @@ package disabled import ( outportcore "github.com/multiversx/mx-chain-core-go/data/outport" + "github.com/multiversx/mx-chain-go/outport" ) @@ -38,6 +39,11 @@ func (n *disabledOutport) SaveValidatorsRating(_ *outportcore.ValidatorsRating) func (n *disabledOutport) SaveAccounts(_ *outportcore.Accounts) { } +// SaveStateChanges does nothing +func (n *disabledOutport) SaveStateChanges() { + return +} + // FinalizedBlock does nothing func (n *disabledOutport) FinalizedBlock(_ *outportcore.FinalizedBlock) { } diff --git a/outport/factory/outportFactory.go b/outport/factory/outportFactory.go index e7259bfc69f..84d963131ae 100644 --- a/outport/factory/outportFactory.go +++ b/outport/factory/outportFactory.go @@ -6,6 +6,7 @@ import ( outportcore "github.com/multiversx/mx-chain-core-go/data/outport" indexerFactory "github.com/multiversx/mx-chain-es-indexer-go/process/factory" + "github.com/multiversx/mx-chain-go/outport" ) @@ -73,12 +74,14 @@ func createAndSubscribeElasticDriverIfNeeded( return nil } - elasticDriver, err := indexerFactory.NewIndexer(args) + //TODO: this needs discussed. + _, err := indexerFactory.NewIndexer(args) if err != nil { return err } - return outport.SubscribeDriver(elasticDriver) + return nil + //return outport.SubscribeDriver(elasticDriver) } func createAndSubscribeEventNotifierIfNeeded( diff --git a/outport/interface.go b/outport/interface.go index d4cd4eb205d..645721b402d 100644 --- a/outport/interface.go +++ b/outport/interface.go @@ -34,7 +34,7 @@ type OutportHandler interface { SaveValidatorsPubKeys(validatorsPubKeys *outportcore.ValidatorsPubKeys) SaveValidatorsRating(validatorsRating *outportcore.ValidatorsRating) SaveAccounts(accounts *outportcore.Accounts) - //SaveStateChanges() + SaveStateChanges() FinalizedBlock(finalizedBlock *outportcore.FinalizedBlock) SubscribeDriver(driver Driver) error HasDrivers() bool diff --git a/outport/outport.go b/outport/outport.go index 17773f1639e..b5cfdabfc83 100644 --- a/outport/outport.go +++ b/outport/outport.go @@ -279,7 +279,7 @@ func (o *outport) saveValidatorsRatingBlocking(validatorsRating *outportcore.Val } } -// SaveAccounts will save accounts for every driver +// SaveAccounts will save accounts for every driver func (o *outport) SaveAccounts(accounts *outportcore.Accounts) { o.mutex.RLock() defer o.mutex.RUnlock() @@ -310,6 +310,7 @@ func (o *outport) saveAccountsBlocking(accounts *outportcore.Accounts, driver Dr } } +// SaveStateChanges will save the state changes to every driver func (o *outport) SaveStateChanges() { o.mutex.RLock() defer o.mutex.RUnlock() diff --git a/process/block/metablock.go b/process/block/metablock.go index a341e25edc2..adbdfa72d3e 100644 --- a/process/block/metablock.go +++ b/process/block/metablock.go @@ -13,6 +13,8 @@ import ( "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/headerVersionData" + logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/dataRetriever" processOutport "github.com/multiversx/mx-chain-go/outport/process" @@ -21,7 +23,6 @@ import ( "github.com/multiversx/mx-chain-go/process/block/helpers" "github.com/multiversx/mx-chain-go/process/block/processedMb" "github.com/multiversx/mx-chain-go/state" - logger "github.com/multiversx/mx-chain-logger-go" ) const firstHeaderNonce = uint64(1) @@ -646,6 +647,9 @@ func (mp *metaProcessor) indexBlock( return } + //TODO: this can be removed. this is only to the test the integration. + mp.outportHandler.SaveStateChanges() + log.Debug("indexed block", "hash", headerHash, "nonce", metaBlock.GetNonce(), "round", metaBlock.GetRound()) indexRoundInfo(mp.outportHandler, mp.nodesCoordinator, core.MetachainShardId, metaBlock, lastMetaBlock, argSaveBlock.SignersIndexes) diff --git a/state/disabled/disabledStateChangesCollector.go b/state/disabled/disabledStateChangesCollector.go index 8405bd3821a..380c27428fa 100644 --- a/state/disabled/disabledStateChangesCollector.go +++ b/state/disabled/disabledStateChangesCollector.go @@ -2,9 +2,10 @@ package disabled import ( "github.com/multiversx/mx-chain-core-go/data/transaction" + vmcommon "github.com/multiversx/mx-chain-vm-common-go" + "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/state/stateChanges" - vmcommon "github.com/multiversx/mx-chain-vm-common-go" ) // disabledStateChangesCollector is a state changes collector that does nothing @@ -47,6 +48,10 @@ func (d *disabledStateChangesCollector) Publish() error { return nil } +func (d *disabledStateChangesCollector) GetStateChanges() []stateChanges.StateChange { + return nil +} + // IsInterfaceNil returns true if there is no value under the interface func (d *disabledStateChangesCollector) IsInterfaceNil() bool { return d == nil From 265cbe82317bdb82c1313c04bd35ec4360cae4be Mon Sep 17 00:00:00 2001 From: ssd04 Date: Wed, 4 Sep 2024 17:11:30 +0300 Subject: [PATCH 22/55] added todo comment for db config variable --- factory/state/stateComponents.go | 1 + 1 file changed, 1 insertion(+) diff --git a/factory/state/stateComponents.go b/factory/state/stateComponents.go index de6b9d4a4cf..e9ce1270e49 100644 --- a/factory/state/stateComponents.go +++ b/factory/state/stateComponents.go @@ -130,6 +130,7 @@ func (scf *stateComponentsFactory) createStateChangesCollector() (state.StateCha return stateChanges.NewStateChangesCollector(), nil } + // TODO: move to toml config file dbConfig := config.DBConfig{ FilePath: "stateChanges", Type: "LvlDBSerial", From f6bff485430c0d4baaf550f57957fa5aae388114 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Thu, 5 Sep 2024 15:18:54 +0300 Subject: [PATCH 23/55] migrated stateChanges to proto structure in mx-chain-core-go. --- go.mod | 2 +- go.sum | 4 +- process/block/preprocess/basePreProcess.go | 1 + process/block/preprocess/transactions.go | 5 ++- state/accountsDB.go | 32 +++++++-------- state/accountsDBApi.go | 3 +- state/accountsDBApiWithHistory.go | 3 +- state/interface.go | 5 ++- state/stateChanges/writeCollector.go | 43 ++------------------ state/trackableDataTrie/trackableDataTrie.go | 28 +++++++------ testscommon/trie/dataTrieTrackerStub.go | 11 ++--- 11 files changed, 55 insertions(+), 82 deletions(-) diff --git a/go.mod b/go.mod index 86225522dcc..33bdddeb976 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/klauspost/cpuid/v2 v2.2.5 github.com/mitchellh/mapstructure v1.5.0 github.com/multiversx/mx-chain-communication-go v1.0.13-0.20240126121117-627adccf10ad - github.com/multiversx/mx-chain-core-go v1.2.19-0.20240222081523-011c96ab2548 + github.com/multiversx/mx-chain-core-go v1.2.21-0.20240905114358-3e9985d6cb80 github.com/multiversx/mx-chain-crypto-go v1.2.10-0.20231206065052-38843c1f1479 github.com/multiversx/mx-chain-es-indexer-go v1.4.19-0.20240129150813-a772c480d33a github.com/multiversx/mx-chain-logger-go v1.0.14-0.20240129144507-d00e967c890c diff --git a/go.sum b/go.sum index f12ab723392..252cb9a2c86 100644 --- a/go.sum +++ b/go.sum @@ -387,8 +387,8 @@ github.com/multiversx/concurrent-map v0.1.4 h1:hdnbM8VE4b0KYJaGY5yJS2aNIW9TFFsUY github.com/multiversx/concurrent-map v0.1.4/go.mod h1:8cWFRJDOrWHOTNSqgYCUvwT7c7eFQ4U2vKMOp4A/9+o= github.com/multiversx/mx-chain-communication-go v1.0.13-0.20240126121117-627adccf10ad h1:izxTyKCxvT7z2mhXCWAZibSxwRVgLmq/kDovs4Nx/6Y= github.com/multiversx/mx-chain-communication-go v1.0.13-0.20240126121117-627adccf10ad/go.mod h1:n4E8BWIV0g3AcNGe1gf+vcjUC8A2QCJ4ARQSbiUDGrI= -github.com/multiversx/mx-chain-core-go v1.2.19-0.20240222081523-011c96ab2548 h1:WQoVgQG9YWiYM5Q3MmnbnxeoQkfHr63iFJZScFYsMxk= -github.com/multiversx/mx-chain-core-go v1.2.19-0.20240222081523-011c96ab2548/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= +github.com/multiversx/mx-chain-core-go v1.2.21-0.20240905114358-3e9985d6cb80 h1:5ogEZZrKvTAh4i2TC8udXftoSwREw9f5dvSxODDSkHY= +github.com/multiversx/mx-chain-core-go v1.2.21-0.20240905114358-3e9985d6cb80/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= github.com/multiversx/mx-chain-crypto-go v1.2.10-0.20231206065052-38843c1f1479 h1:beVIhs5ysylwNplQ/bZ0h5DoDlqKNWgpWE/NMHHNmAw= github.com/multiversx/mx-chain-crypto-go v1.2.10-0.20231206065052-38843c1f1479/go.mod h1:Ap6p7QZFtwPlb++OvCG+85BfuZ+bLP/JtQp6EwjWJsI= github.com/multiversx/mx-chain-es-indexer-go v1.4.19-0.20240129150813-a772c480d33a h1:mOMUhbsjTq7n5oAv4KkVnL67ngS0+wkqmkiv1XJfBIY= diff --git a/process/block/preprocess/basePreProcess.go b/process/block/preprocess/basePreProcess.go index 58534fe4395..3519bcad6f6 100644 --- a/process/block/preprocess/basePreProcess.go +++ b/process/block/preprocess/basePreProcess.go @@ -12,6 +12,7 @@ import ( "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/process" diff --git a/process/block/preprocess/transactions.go b/process/block/preprocess/transactions.go index fd53f95aad5..c8f3a9b4eb5 100644 --- a/process/block/preprocess/transactions.go +++ b/process/block/preprocess/transactions.go @@ -15,6 +15,9 @@ import ( "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" + logger "github.com/multiversx/mx-chain-logger-go" + vmcommon "github.com/multiversx/mx-chain-vm-common-go" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/process" @@ -23,8 +26,6 @@ import ( "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/txcache" - logger "github.com/multiversx/mx-chain-logger-go" - vmcommon "github.com/multiversx/mx-chain-vm-common-go" ) var _ process.DataMarshalizer = (*transactions)(nil) diff --git a/state/accountsDB.go b/state/accountsDB.go index 56bb820f42d..9420183b7be 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -13,6 +13,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-core-go/data/stateChange" "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" @@ -23,7 +24,6 @@ import ( "github.com/multiversx/mx-chain-go/common/errChan" "github.com/multiversx/mx-chain-go/common/holders" "github.com/multiversx/mx-chain-go/state/parsers" - "github.com/multiversx/mx-chain-go/state/stateChanges" "github.com/multiversx/mx-chain-go/trie/keyBuilder" "github.com/multiversx/mx-chain-go/trie/statistics" ) @@ -210,7 +210,7 @@ func (adb *AccountsDB) GetCode(codeHash []byte) []byte { return nil } - stateChange := &stateChanges.StateChangeDTO{ + stateChange := &stateChange.StateChange{ Type: "read", MainTrieKey: codeHash, MainTrieVal: val, @@ -285,7 +285,7 @@ func (adb *AccountsDB) SaveAccount(account vmcommon.AccountHandler) error { return err } - stateChange := &stateChanges.StateChangeDTO{ + stateChange := &stateChange.StateChange{ Type: "write", MainTrieKey: account.AddressBytes(), MainTrieVal: marshalledAccount, @@ -298,12 +298,12 @@ func (adb *AccountsDB) SaveAccount(account vmcommon.AccountHandler) error { return err } -func (adb *AccountsDB) saveCodeAndDataTrie(oldAcc, newAcc vmcommon.AccountHandler) ([]stateChanges.DataTrieChange, error) { +func (adb *AccountsDB) saveCodeAndDataTrie(oldAcc, newAcc vmcommon.AccountHandler) ([]*stateChange.DataTrieChange, error) { baseNewAcc, newAccOk := newAcc.(baseAccountHandler) baseOldAccount, _ := oldAcc.(baseAccountHandler) if !newAccOk { - return make([]stateChanges.DataTrieChange, 0), nil + return make([]*stateChange.DataTrieChange, 0), nil } newValues, err := adb.saveDataTrie(baseNewAcc) @@ -372,14 +372,14 @@ func (adb *AccountsDB) updateOldCodeEntry(oldCodeHash []byte) (*CodeEntry, error return nil, nil } - stateChange := &stateChanges.StateChangeDTO{ + sc := &stateChange.StateChange{ Type: "read", MainTrieKey: oldCodeHash, MainTrieVal: nil, Operation: "getCode", DataTrieChanges: nil, } - adb.stateChangesCollector.AddStateChange(stateChange) + adb.stateChangesCollector.AddStateChange(sc) unmodifiedOldCodeEntry := &CodeEntry{ Code: oldCodeEntry.Code, @@ -392,14 +392,14 @@ func (adb *AccountsDB) updateOldCodeEntry(oldCodeHash []byte) (*CodeEntry, error return nil, err } - stateChange := &stateChanges.StateChangeDTO{ + sc1 := &stateChange.StateChange{ Type: "write", MainTrieKey: oldCodeHash, MainTrieVal: nil, Operation: "writeCode", DataTrieChanges: nil, } - adb.stateChangesCollector.AddStateChange(stateChange) + adb.stateChangesCollector.AddStateChange(sc1) return unmodifiedOldCodeEntry, nil } @@ -410,14 +410,14 @@ func (adb *AccountsDB) updateOldCodeEntry(oldCodeHash []byte) (*CodeEntry, error return nil, err } - stateChange = &stateChanges.StateChangeDTO{ + sc = &stateChange.StateChange{ Type: "write", MainTrieKey: oldCodeHash, MainTrieVal: codeEntryBytes, Operation: "writeCode", DataTrieChanges: nil, } - adb.stateChangesCollector.AddStateChange(stateChange) + adb.stateChangesCollector.AddStateChange(sc) return unmodifiedOldCodeEntry, nil } @@ -444,14 +444,14 @@ func (adb *AccountsDB) updateNewCodeEntry(newCodeHash []byte, newCode []byte) er return err } - stateChange := &stateChanges.StateChangeDTO{ + sc := &stateChange.StateChange{ Type: "write", MainTrieKey: newCodeHash, MainTrieVal: codeEntryBytes, Operation: "writeCode", DataTrieChanges: nil, } - adb.stateChangesCollector.AddStateChange(stateChange) + adb.stateChangesCollector.AddStateChange(sc) return nil } @@ -517,7 +517,7 @@ func (adb *AccountsDB) loadDataTrieConcurrentSafe(accountHandler baseAccountHand // SaveDataTrie is used to save the data trie (not committing it) and to recompute the new Root value // If data is not dirtied, method will not create its JournalEntries to keep track of data modification -func (adb *AccountsDB) saveDataTrie(accountHandler baseAccountHandler) ([]stateChanges.DataTrieChange, error) { +func (adb *AccountsDB) saveDataTrie(accountHandler baseAccountHandler) ([]*stateChange.DataTrieChange, error) { newValues, oldValues, err := accountHandler.SaveDirtyData(adb.mainTrie) if err != nil { return nil, err @@ -651,13 +651,13 @@ func (adb *AccountsDB) removeDataTrie(baseAcc baseAccountHandler) error { } adb.journalize(entry) - stateChange := &stateChanges.StateChangeDTO{ + sc := &stateChange.StateChange{ Type: "write", MainTrieKey: baseAcc.AddressBytes(), Operation: "removeDataTrie", DataTrieChanges: nil, } - adb.stateChangesCollector.AddStateChange(stateChange) + adb.stateChangesCollector.AddStateChange(sc) return nil } diff --git a/state/accountsDBApi.go b/state/accountsDBApi.go index a9d42146a64..7b53e4ad85c 100644 --- a/state/accountsDBApi.go +++ b/state/accountsDBApi.go @@ -7,9 +7,10 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data/transaction" + vmcommon "github.com/multiversx/mx-chain-vm-common-go" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/holders" - vmcommon "github.com/multiversx/mx-chain-vm-common-go" ) type accountsDBApi struct { diff --git a/state/accountsDBApiWithHistory.go b/state/accountsDBApiWithHistory.go index 96d43a9cd31..c078143e3cc 100644 --- a/state/accountsDBApiWithHistory.go +++ b/state/accountsDBApiWithHistory.go @@ -7,9 +7,10 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data/transaction" + vmcommon "github.com/multiversx/mx-chain-vm-common-go" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/holders" - vmcommon "github.com/multiversx/mx-chain-vm-common-go" ) type accountsDBApiWithHistory struct { diff --git a/state/interface.go b/state/interface.go index 80e3a6baca3..f7583adc5b4 100644 --- a/state/interface.go +++ b/state/interface.go @@ -6,6 +6,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data/api" + "github.com/multiversx/mx-chain-core-go/data/stateChange" "github.com/multiversx/mx-chain-core-go/data/transaction" vmcommon "github.com/multiversx/mx-chain-vm-common-go" @@ -163,7 +164,7 @@ type baseAccountHandler interface { GetRootHash() []byte SetDataTrie(trie common.Trie) DataTrie() common.DataTrieHandler - SaveDirtyData(trie common.Trie) ([]stateChanges.DataTrieChange, []core.TrieData, error) + SaveDirtyData(trie common.Trie) ([]*stateChange.DataTrieChange, []core.TrieData, error) IsInterfaceNil() bool } @@ -261,7 +262,7 @@ type DataTrieTracker interface { SaveKeyValue(key []byte, value []byte) error SetDataTrie(tr common.Trie) DataTrie() common.DataTrieHandler - SaveDirtyData(common.Trie) ([]stateChanges.DataTrieChange, []core.TrieData, error) + SaveDirtyData(common.Trie) ([]*stateChange.DataTrieChange, []core.TrieData, error) MigrateDataTrieLeaves(args vmcommon.ArgsMigrateDataTrieLeaves) error IsInterfaceNil() bool } diff --git a/state/stateChanges/writeCollector.go b/state/stateChanges/writeCollector.go index 89bfad57617..7c794845832 100644 --- a/state/stateChanges/writeCollector.go +++ b/state/stateChanges/writeCollector.go @@ -14,49 +14,14 @@ import ( var log = logger.GetOrCreate("state/stateChanges") -// DataTrieChange represents a change in the data trie -type DataTrieChange struct { - Type string `json:"type"` - Key []byte `json:"key"` - Val []byte `json:"-"` -} - // ErrStateChangesIndexOutOfBounds signals that the state changes index is out of bounds var ErrStateChangesIndexOutOfBounds = errors.New("state changes index out of bounds") type StateChange interface { GetTxHash() []byte SetTxHash(txHash []byte) - GetIndex() int - SetIndex(index int) -} - -// StateChangeDTO is used to collect state changes -// TODO: change to use proto structs -type StateChangeDTO struct { - Type string `json:"type"` - Index int `json:"-"` - TxHash []byte `json:"-"` - MainTrieKey []byte `json:"mainTrieKey"` - MainTrieVal []byte `json:"-"` - Operation string `json:"operation"` - DataTrieChanges []DataTrieChange `json:"dataTrieChanges"` -} - -func (sc *StateChangeDTO) GetIndex() int { - return sc.Index -} - -func (sc *StateChangeDTO) SetIndex(index int) { - sc.Index = index -} - -func (sc *StateChangeDTO) GetTxHash() []byte { - return sc.TxHash -} - -func (sc *StateChangeDTO) SetTxHash(txHash []byte) { - sc.TxHash = txHash + GetIndex() int32 + SetIndex(index int32) } // StateChangesForTx is used to collect state changes for a transaction hash @@ -159,7 +124,7 @@ func (scc *stateChangesCollector) SetIndexToLastStateChange(index int) error { scc.stateChangesMut.Lock() defer scc.stateChangesMut.Unlock() - scc.stateChanges[len(scc.stateChanges)-1].SetIndex(index) + scc.stateChanges[len(scc.stateChanges)-1].SetIndex(int32(index)) return nil } @@ -179,7 +144,7 @@ func (scc *stateChangesCollector) RevertToIndex(index int) error { defer scc.stateChangesMut.Unlock() for i := len(scc.stateChanges) - 1; i >= 0; i-- { - if scc.stateChanges[i].GetIndex() == index { + if scc.stateChanges[i].GetIndex() == int32(index) { scc.stateChanges = scc.stateChanges[:i] break } diff --git a/state/trackableDataTrie/trackableDataTrie.go b/state/trackableDataTrie/trackableDataTrie.go index f16964c79f2..2d356ffab5d 100644 --- a/state/trackableDataTrie/trackableDataTrie.go +++ b/state/trackableDataTrie/trackableDataTrie.go @@ -8,15 +8,17 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data" + "github.com/multiversx/mx-chain-core-go/data/stateChange" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" + + logger "github.com/multiversx/mx-chain-logger-go" + vmcommon "github.com/multiversx/mx-chain-vm-common-go" + "github.com/multiversx/mx-chain-go/common" errorsCommon "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/state/dataTrieValue" - "github.com/multiversx/mx-chain-go/state/stateChanges" - logger "github.com/multiversx/mx-chain-logger-go" - vmcommon "github.com/multiversx/mx-chain-vm-common-go" ) var log = logger.GetOrCreate("state/trackableDataTrie") @@ -104,11 +106,11 @@ func (tdt *trackableDataTrie) RetrieveValue(key []byte) ([]byte, uint32, error) log.Trace("retrieve value from trie", "key", key, "value", val, "account", tdt.identifier) - stateChange := &stateChanges.StateChangeDTO{ + stateChange := &stateChange.StateChange{ Type: "read", MainTrieKey: tdt.identifier, MainTrieVal: nil, - DataTrieChanges: []stateChanges.DataTrieChange{ + DataTrieChanges: []*stateChange.DataTrieChange{ { Type: "read", Key: key, @@ -245,9 +247,9 @@ func (tdt *trackableDataTrie) DataTrie() common.DataTrieHandler { } // SaveDirtyData saved the dirty data to the trie -func (tdt *trackableDataTrie) SaveDirtyData(mainTrie common.Trie) ([]stateChanges.DataTrieChange, []core.TrieData, error) { +func (tdt *trackableDataTrie) SaveDirtyData(mainTrie common.Trie) ([]*stateChange.DataTrieChange, []core.TrieData, error) { if len(tdt.dirtyData) == 0 { - return make([]stateChanges.DataTrieChange, 0), make([]core.TrieData, 0), nil + return make([]*stateChange.DataTrieChange, 0), make([]core.TrieData, 0), nil } if check.IfNil(tdt.tr) { @@ -267,10 +269,10 @@ func (tdt *trackableDataTrie) SaveDirtyData(mainTrie common.Trie) ([]stateChange return tdt.updateTrie(dtr) } -func (tdt *trackableDataTrie) updateTrie(dtr state.DataTrie) ([]stateChanges.DataTrieChange, []core.TrieData, error) { +func (tdt *trackableDataTrie) updateTrie(dtr state.DataTrie) ([]*stateChange.DataTrieChange, []core.TrieData, error) { oldValues := make([]core.TrieData, len(tdt.dirtyData)) - newData := make([]stateChanges.DataTrieChange, len(tdt.dirtyData)) - deletedKeys := make([]stateChanges.DataTrieChange, 0) + newData := make([]*stateChange.DataTrieChange, len(tdt.dirtyData)) + deletedKeys := make([]*stateChange.DataTrieChange, 0) index := 0 for key, dataEntry := range tdt.dirtyData { @@ -287,7 +289,7 @@ func (tdt *trackableDataTrie) updateTrie(dtr state.DataTrie) ([]stateChanges.Dat if wasDeleted { deletedKeys = append(deletedKeys, - stateChanges.DataTrieChange{ + &stateChange.DataTrieChange{ Type: "write", Key: []byte(key), Val: nil, @@ -318,7 +320,7 @@ func (tdt *trackableDataTrie) updateTrie(dtr state.DataTrie) ([]stateChanges.Dat return nil, nil, fmt.Errorf("index out of range") } - newData[dataEntry.index] = stateChanges.DataTrieChange{ + newData[dataEntry.index] = &stateChange.DataTrieChange{ Type: "write", Key: dataTrieKey, Val: dataTrieVal, @@ -327,7 +329,7 @@ func (tdt *trackableDataTrie) updateTrie(dtr state.DataTrie) ([]stateChanges.Dat tdt.dirtyData = make(map[string]dirtyData) - stateChanges := make([]stateChanges.DataTrieChange, 0) + stateChanges := make([]*stateChange.DataTrieChange, 0) for i := range newData { if len(newData[i].Key) == 0 { continue diff --git a/testscommon/trie/dataTrieTrackerStub.go b/testscommon/trie/dataTrieTrackerStub.go index 9ee46a0a8e8..73c083b30c1 100644 --- a/testscommon/trie/dataTrieTrackerStub.go +++ b/testscommon/trie/dataTrieTrackerStub.go @@ -3,9 +3,10 @@ package trie import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" - "github.com/multiversx/mx-chain-go/common" - "github.com/multiversx/mx-chain-go/state/stateChanges" + "github.com/multiversx/mx-chain-core-go/data/stateChange" vmcommon "github.com/multiversx/mx-chain-vm-common-go" + + "github.com/multiversx/mx-chain-go/common" ) // DataTrieTrackerStub - @@ -16,7 +17,7 @@ type DataTrieTrackerStub struct { SaveKeyValueCalled func(key []byte, value []byte) error SetDataTrieCalled func(tr common.Trie) DataTrieCalled func() common.Trie - SaveDirtyDataCalled func(trie common.Trie) ([]stateChanges.DataTrieChange, []core.TrieData, error) + SaveDirtyDataCalled func(trie common.Trie) ([]*stateChange.DataTrieChange, []core.TrieData, error) SaveTrieDataCalled func(trieData core.TrieData) error MigrateDataTrieLeavesCalled func(args vmcommon.ArgsMigrateDataTrieLeaves) error } @@ -62,12 +63,12 @@ func (dtts *DataTrieTrackerStub) DataTrie() common.DataTrieHandler { } // SaveDirtyData - -func (dtts *DataTrieTrackerStub) SaveDirtyData(mainTrie common.Trie) ([]stateChanges.DataTrieChange, []core.TrieData, error) { +func (dtts *DataTrieTrackerStub) SaveDirtyData(mainTrie common.Trie) ([]*stateChange.DataTrieChange, []core.TrieData, error) { if dtts.SaveDirtyDataCalled != nil { return dtts.SaveDirtyDataCalled(mainTrie) } - return make([]stateChanges.DataTrieChange, 0), make([]core.TrieData, 0), nil + return make([]*stateChange.DataTrieChange, 0), make([]core.TrieData, 0), nil } // MigrateDataTrieLeaves - From a57053e058e0068e0ec7fdba472ae6ec4cefd4ef Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Fri, 6 Sep 2024 14:18:51 +0300 Subject: [PATCH 24/55] moved all the state changes in mx-chain-core-go. --- factory/processing/blockProcessorCreator.go | 8 +- go.mod | 2 +- go.sum | 4 +- outport/factory/hostDriverFactory.go | 7 +- outport/host/driver.go | 39 ++--- outport/interface.go | 2 - outport/outport.go | 27 ---- .../factory/outportDataProviderFactory.go | 5 +- outport/process/outportDataProvider.go | 11 +- process/block/metablock.go | 4 +- scripts/testnet/variables.sh | 10 +- .../disabled/disabledStateChangesCollector.go | 3 +- state/interface.go | 2 +- state/stateChanges/dataAnalysisCollector.go | 2 +- state/stateChanges/export_test.go | 2 +- state/stateChanges/writeCollector.go | 145 +++++++++++------- 16 files changed, 137 insertions(+), 136 deletions(-) diff --git a/factory/processing/blockProcessorCreator.go b/factory/processing/blockProcessorCreator.go index c3618eff970..54df8094660 100644 --- a/factory/processing/blockProcessorCreator.go +++ b/factory/processing/blockProcessorCreator.go @@ -6,6 +6,10 @@ import ( "github.com/multiversx/mx-chain-core-go/core" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" + logger "github.com/multiversx/mx-chain-logger-go" + vmcommon "github.com/multiversx/mx-chain-vm-common-go" + "github.com/multiversx/mx-chain-vm-common-go/parsers" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/config" "github.com/multiversx/mx-chain-go/dataRetriever" @@ -42,9 +46,6 @@ import ( "github.com/multiversx/mx-chain-go/state/syncer" "github.com/multiversx/mx-chain-go/storage/txcache" "github.com/multiversx/mx-chain-go/vm" - logger "github.com/multiversx/mx-chain-logger-go" - vmcommon "github.com/multiversx/mx-chain-vm-common-go" - "github.com/multiversx/mx-chain-vm-common-go/parsers" ) type blockProcessorAndVmFactories struct { @@ -1031,6 +1032,7 @@ func (pcf *processComponentsFactory) createOutportDataProvider( MbsStorer: mbsStorer, EnableEpochsHandler: pcf.coreData.EnableEpochsHandler(), ExecutionOrderGetter: pcf.txExecutionOrderHandler, + StateChangesCollector: pcf.state.StateChangesCollector(), }) } diff --git a/go.mod b/go.mod index 33bdddeb976..68fb7ae914d 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/klauspost/cpuid/v2 v2.2.5 github.com/mitchellh/mapstructure v1.5.0 github.com/multiversx/mx-chain-communication-go v1.0.13-0.20240126121117-627adccf10ad - github.com/multiversx/mx-chain-core-go v1.2.21-0.20240905114358-3e9985d6cb80 + github.com/multiversx/mx-chain-core-go v1.2.21-0.20240906093529-4da59a927b44 github.com/multiversx/mx-chain-crypto-go v1.2.10-0.20231206065052-38843c1f1479 github.com/multiversx/mx-chain-es-indexer-go v1.4.19-0.20240129150813-a772c480d33a github.com/multiversx/mx-chain-logger-go v1.0.14-0.20240129144507-d00e967c890c diff --git a/go.sum b/go.sum index 252cb9a2c86..22d508569c2 100644 --- a/go.sum +++ b/go.sum @@ -387,8 +387,8 @@ github.com/multiversx/concurrent-map v0.1.4 h1:hdnbM8VE4b0KYJaGY5yJS2aNIW9TFFsUY github.com/multiversx/concurrent-map v0.1.4/go.mod h1:8cWFRJDOrWHOTNSqgYCUvwT7c7eFQ4U2vKMOp4A/9+o= github.com/multiversx/mx-chain-communication-go v1.0.13-0.20240126121117-627adccf10ad h1:izxTyKCxvT7z2mhXCWAZibSxwRVgLmq/kDovs4Nx/6Y= github.com/multiversx/mx-chain-communication-go v1.0.13-0.20240126121117-627adccf10ad/go.mod h1:n4E8BWIV0g3AcNGe1gf+vcjUC8A2QCJ4ARQSbiUDGrI= -github.com/multiversx/mx-chain-core-go v1.2.21-0.20240905114358-3e9985d6cb80 h1:5ogEZZrKvTAh4i2TC8udXftoSwREw9f5dvSxODDSkHY= -github.com/multiversx/mx-chain-core-go v1.2.21-0.20240905114358-3e9985d6cb80/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= +github.com/multiversx/mx-chain-core-go v1.2.21-0.20240906093529-4da59a927b44 h1:3GdhKb2Q5KeW9A8uM3a/K9Ejjm8NwRJmpWOw1TQtCk0= +github.com/multiversx/mx-chain-core-go v1.2.21-0.20240906093529-4da59a927b44/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= github.com/multiversx/mx-chain-crypto-go v1.2.10-0.20231206065052-38843c1f1479 h1:beVIhs5ysylwNplQ/bZ0h5DoDlqKNWgpWE/NMHHNmAw= github.com/multiversx/mx-chain-crypto-go v1.2.10-0.20231206065052-38843c1f1479/go.mod h1:Ap6p7QZFtwPlb++OvCG+85BfuZ+bLP/JtQp6EwjWJsI= github.com/multiversx/mx-chain-es-indexer-go v1.4.19-0.20240129150813-a772c480d33a h1:mOMUhbsjTq7n5oAv4KkVnL67ngS0+wkqmkiv1XJfBIY= diff --git a/outport/factory/hostDriverFactory.go b/outport/factory/hostDriverFactory.go index 895c5577688..90b5b562dac 100644 --- a/outport/factory/hostDriverFactory.go +++ b/outport/factory/hostDriverFactory.go @@ -42,9 +42,8 @@ func CreateHostDriver(args ArgsHostDriverFactory) (outport.Driver, error) { } return host.NewHostDriver(host.ArgsHostDriver{ - Marshaller: args.Marshaller, - SenderHost: wsHost, - Log: log, - StateChangesCollector: args.StateChangesCollector, + Marshaller: args.Marshaller, + SenderHost: wsHost, + Log: log, }) } diff --git a/outport/host/driver.go b/outport/host/driver.go index e73871c2c71..d15771f28c4 100644 --- a/outport/host/driver.go +++ b/outport/host/driver.go @@ -8,25 +8,21 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/marshal" - - "github.com/multiversx/mx-chain-go/state" ) // ArgsHostDriver holds the arguments needed for creating a new hostDriver type ArgsHostDriver struct { - Marshaller marshal.Marshalizer - SenderHost SenderHost - Log core.Logger - StateChangesCollector state.StateChangesCollector + Marshaller marshal.Marshalizer + SenderHost SenderHost + Log core.Logger } type hostDriver struct { - marshaller marshal.Marshalizer - senderHost SenderHost - isClosed atomic.Flag - log core.Logger - payloadProc payloadProcessorHandler - stateChangesCollector state.StateChangesCollector + marshaller marshal.Marshalizer + senderHost SenderHost + isClosed atomic.Flag + log core.Logger + payloadProc payloadProcessorHandler } // NewHostDriver will create a new instance of hostDriver @@ -40,9 +36,6 @@ func NewHostDriver(args ArgsHostDriver) (*hostDriver, error) { if check.IfNil(args.Log) { return nil, core.ErrNilLogger } - if check.IfNil(args.StateChangesCollector) { - return nil, ErrNilStateChangesCollector - } payloadProc, err := newPayloadProcessor(args.Log) if err != nil { @@ -55,12 +48,11 @@ func NewHostDriver(args ArgsHostDriver) (*hostDriver, error) { } return &hostDriver{ - marshaller: args.Marshaller, - senderHost: args.SenderHost, - log: args.Log, - isClosed: atomic.Flag{}, - payloadProc: payloadProc, - stateChangesCollector: args.StateChangesCollector, + marshaller: args.Marshaller, + senderHost: args.SenderHost, + log: args.Log, + isClosed: atomic.Flag{}, + payloadProc: payloadProc, }, nil } @@ -132,11 +124,6 @@ func (o *hostDriver) SetCurrentSettings(config outport.OutportConfig) error { return o.handleAction(&config, outport.TopicSettings) } -func (o *hostDriver) SaveStateChanges() error { - //TODO: add this as a constant - return o.handleAction(o.stateChangesCollector.GetStateChanges(), "SaveStateChanges") -} - // Close will handle the closing of the outport driver web socket sender func (o *hostDriver) Close() error { o.isClosed.SetValue(true) diff --git a/outport/interface.go b/outport/interface.go index 645721b402d..70a3673cd0c 100644 --- a/outport/interface.go +++ b/outport/interface.go @@ -16,7 +16,6 @@ type Driver interface { SaveValidatorsPubKeys(validatorsPubKeys *outportcore.ValidatorsPubKeys) error SaveValidatorsRating(validatorsRating *outportcore.ValidatorsRating) error SaveAccounts(accounts *outportcore.Accounts) error - SaveStateChanges() error FinalizedBlock(finalizedBlock *outportcore.FinalizedBlock) error GetMarshaller() marshal.Marshalizer SetCurrentSettings(config outportcore.OutportConfig) error @@ -34,7 +33,6 @@ type OutportHandler interface { SaveValidatorsPubKeys(validatorsPubKeys *outportcore.ValidatorsPubKeys) SaveValidatorsRating(validatorsRating *outportcore.ValidatorsRating) SaveAccounts(accounts *outportcore.Accounts) - SaveStateChanges() FinalizedBlock(finalizedBlock *outportcore.FinalizedBlock) SubscribeDriver(driver Driver) error HasDrivers() bool diff --git a/outport/outport.go b/outport/outport.go index b5cfdabfc83..e3210361d56 100644 --- a/outport/outport.go +++ b/outport/outport.go @@ -310,33 +310,6 @@ func (o *outport) saveAccountsBlocking(accounts *outportcore.Accounts, driver Dr } } -// SaveStateChanges will save the state changes to every driver -func (o *outport) SaveStateChanges() { - o.mutex.RLock() - defer o.mutex.RUnlock() - - for _, driver := range o.drivers { - o.saveStateChangesBlocking(driver) - } -} - -func (o *outport) saveStateChangesBlocking(driver Driver) { - ch := o.monitorCompletionOnDriver("finalizedBlockBlocking", driver) - defer close(ch) - - for { - err := driver.SaveStateChanges() - if err == nil { - return - } - - log.Error("error calling SaveStateChanges, will retry", - "driver", driverString(driver), - "retrial in", o.retrialInterval, - "error", err) - } -} - // FinalizedBlock will call all the drivers that a block is finalized func (o *outport) FinalizedBlock(finalizedBlock *outportcore.FinalizedBlock) { o.mutex.RLock() diff --git a/outport/process/factory/outportDataProviderFactory.go b/outport/process/factory/outportDataProviderFactory.go index 68546df1c50..f0a083f6665 100644 --- a/outport/process/factory/outportDataProviderFactory.go +++ b/outport/process/factory/outportDataProviderFactory.go @@ -4,6 +4,8 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" + vmcommon "github.com/multiversx/mx-chain-vm-common-go" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/outport" "github.com/multiversx/mx-chain-go/outport/process" @@ -15,7 +17,6 @@ import ( "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/storage" - vmcommon "github.com/multiversx/mx-chain-vm-common-go" ) // ArgOutportDataProviderFactory holds the arguments needed for creating a new instance of outport.DataProviderOutport @@ -36,6 +37,7 @@ type ArgOutportDataProviderFactory struct { MbsStorer storage.Storer EnableEpochsHandler common.EnableEpochsHandler ExecutionOrderGetter common.ExecutionOrderGetter + StateChangesCollector state.StateChangesCollector } // CreateOutportDataProvider will create a new instance of outport.DataProviderOutport @@ -82,5 +84,6 @@ func CreateOutportDataProvider(arg ArgOutportDataProviderFactory) (outport.DataP ExecutionOrderHandler: arg.ExecutionOrderGetter, Hasher: arg.Hasher, Marshaller: arg.Marshaller, + StateChangesCollector: arg.StateChangesCollector, }) } diff --git a/outport/process/outportDataProvider.go b/outport/process/outportDataProvider.go index a99e0bc4827..adaee8ea35f 100644 --- a/outport/process/outportDataProvider.go +++ b/outport/process/outportDataProvider.go @@ -16,12 +16,14 @@ import ( "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" + logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/outport/process/alteredaccounts/shared" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" - logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-go/state" ) var log = logger.GetOrCreate("outport/process/outportDataProvider") @@ -39,6 +41,7 @@ type ArgOutportDataProvider struct { Marshaller marshal.Marshalizer Hasher hashing.Hasher ExecutionOrderHandler common.ExecutionOrderGetter + StateChangesCollector state.StateChangesCollector } // ArgPrepareOutportSaveBlockData holds the arguments needed for prepare outport save block data @@ -67,6 +70,7 @@ type outportDataProvider struct { executionOrderHandler common.ExecutionOrderGetter marshaller marshal.Marshalizer hasher hashing.Hasher + stateChangesCollector state.StateChangesCollector } // NewOutportDataProvider will create a new instance of outportDataProvider @@ -83,6 +87,7 @@ func NewOutportDataProvider(arg ArgOutportDataProvider) (*outportDataProvider, e executionOrderHandler: arg.ExecutionOrderHandler, marshaller: arg.Marshaller, hasher: arg.Hasher, + stateChangesCollector: arg.StateChangesCollector, }, nil } @@ -134,6 +139,9 @@ func (odp *outportDataProvider) PrepareOutportSaveBlockData(arg ArgPrepareOutpor return nil, err } + stateChanges := odp.stateChangesCollector.GetStateChangesForTxs() + odp.stateChangesCollector.Reset() + return &outportcore.OutportBlockWithHeaderAndBody{ OutportBlock: &outportcore.OutportBlock{ ShardID: odp.shardID, @@ -145,6 +153,7 @@ func (odp *outportDataProvider) PrepareOutportSaveBlockData(arg ArgPrepareOutpor GasPenalized: odp.gasConsumedProvider.TotalGasPenalized(), MaxGasPerBlock: odp.economicsData.MaxGasLimitPerBlock(odp.shardID), }, + StateChanges: stateChanges, AlteredAccounts: alteredAccounts, NotarizedHeadersHashes: arg.NotarizedHeadersHashes, NumberOfShards: odp.numOfShards, diff --git a/process/block/metablock.go b/process/block/metablock.go index adbdfa72d3e..073a0046282 100644 --- a/process/block/metablock.go +++ b/process/block/metablock.go @@ -635,6 +635,7 @@ func (mp *metaProcessor) indexBlock( HighestFinalBlockNonce: mp.forkDetector.GetHighestFinalBlockNonce(), HighestFinalBlockHash: mp.forkDetector.GetHighestFinalBlockHash(), }) + if err != nil { log.Error("metaProcessor.indexBlock cannot prepare argSaveBlock", "error", err.Error(), "hash", headerHash, "nonce", metaBlock.GetNonce(), "round", metaBlock.GetRound()) @@ -647,9 +648,6 @@ func (mp *metaProcessor) indexBlock( return } - //TODO: this can be removed. this is only to the test the integration. - mp.outportHandler.SaveStateChanges() - log.Debug("indexed block", "hash", headerHash, "nonce", metaBlock.GetNonce(), "round", metaBlock.GetRound()) indexRoundInfo(mp.outportHandler, mp.nodesCoordinator, core.MetachainShardId, metaBlock, lastMetaBlock, argSaveBlock.SignersIndexes) diff --git a/scripts/testnet/variables.sh b/scripts/testnet/variables.sh index f3fb44c5866..fa201f67c4c 100644 --- a/scripts/testnet/variables.sh +++ b/scripts/testnet/variables.sh @@ -12,7 +12,7 @@ export USE_PROXY=1 # Enable the MultiversX Transaction Generator. Note that this is a private # repository (mx-chain-txgen-go). -export USE_TXGEN=0 +export USE_TXGEN=1 # Path where the testnet will be instantiated. This folder is assumed to not # exist, but it doesn't matter if it already does. It will be created if not, @@ -52,13 +52,13 @@ export GENESIS_STAKE_TYPE="direct" #'delegated' or 'direct' as in direct stake export OBSERVERS_ANTIFLOOD_DISABLE=0 # Shard structure -export SHARDCOUNT=2 -export SHARD_VALIDATORCOUNT=3 +export SHARDCOUNT=1 +export SHARD_VALIDATORCOUNT=1 export SHARD_OBSERVERCOUNT=1 -export SHARD_CONSENSUS_SIZE=3 +export SHARD_CONSENSUS_SIZE=1 # Metashard structure -export META_VALIDATORCOUNT=3 +export META_VALIDATORCOUNT=1 export META_OBSERVERCOUNT=1 export META_CONSENSUS_SIZE=$META_VALIDATORCOUNT diff --git a/state/disabled/disabledStateChangesCollector.go b/state/disabled/disabledStateChangesCollector.go index 380c27428fa..836b92776b1 100644 --- a/state/disabled/disabledStateChangesCollector.go +++ b/state/disabled/disabledStateChangesCollector.go @@ -1,6 +1,7 @@ package disabled import ( + "github.com/multiversx/mx-chain-core-go/data/stateChange" "github.com/multiversx/mx-chain-core-go/data/transaction" vmcommon "github.com/multiversx/mx-chain-vm-common-go" @@ -48,7 +49,7 @@ func (d *disabledStateChangesCollector) Publish() error { return nil } -func (d *disabledStateChangesCollector) GetStateChanges() []stateChanges.StateChange { +func (d *disabledStateChangesCollector) GetStateChangesForTxs() map[string]*stateChange.StateChangesForTx { return nil } diff --git a/state/interface.go b/state/interface.go index f7583adc5b4..97c264407f4 100644 --- a/state/interface.go +++ b/state/interface.go @@ -365,6 +365,6 @@ type StateChangesCollector interface { SetIndexToLastStateChange(index int) error RevertToIndex(index int) error Publish() error - GetStateChanges() []stateChanges.StateChange + GetStateChangesForTxs() map[string]*stateChange.StateChangesForTx IsInterfaceNil() bool } diff --git a/state/stateChanges/dataAnalysisCollector.go b/state/stateChanges/dataAnalysisCollector.go index 85b8fd39638..69aaf49fa2f 100644 --- a/state/stateChanges/dataAnalysisCollector.go +++ b/state/stateChanges/dataAnalysisCollector.go @@ -205,7 +205,7 @@ func (scc *dataAnalysisCollector) Publish() error { return nil } -func (scc *dataAnalysisCollector) GetStateChanges() []StateChange { +func (scc *dataAnalysisCollector) GetStateChangesForTx() []StateChange { return scc.stateChanges } diff --git a/state/stateChanges/export_test.go b/state/stateChanges/export_test.go index 86350f45a5c..3e52d0985b2 100644 --- a/state/stateChanges/export_test.go +++ b/state/stateChanges/export_test.go @@ -2,6 +2,6 @@ package stateChanges // GetStateChanges - func (scc *stateChangesCollector) GetStateChanges() []StateChangesForTx { - scs, _ := scc.getStateChangesForTxs() + scs, _ := scc.GetStateChangesForTxs() return scs } diff --git a/state/stateChanges/writeCollector.go b/state/stateChanges/writeCollector.go index 7c794845832..2ed84f263ed 100644 --- a/state/stateChanges/writeCollector.go +++ b/state/stateChanges/writeCollector.go @@ -1,12 +1,10 @@ package stateChanges import ( - "bytes" - "encoding/hex" "errors" - "fmt" "sync" + data "github.com/multiversx/mx-chain-core-go/data/stateChange" "github.com/multiversx/mx-chain-core-go/data/transaction" logger "github.com/multiversx/mx-chain-logger-go" vmcommon "github.com/multiversx/mx-chain-vm-common-go" @@ -18,17 +16,23 @@ var log = logger.GetOrCreate("state/stateChanges") var ErrStateChangesIndexOutOfBounds = errors.New("state changes index out of bounds") type StateChange interface { + GetType() string + GetIndex() int32 GetTxHash() []byte + GetMainTrieKey() []byte + GetMainTrieVal() []byte + GetOperation() string + GetDataTrieChanges() []*data.DataTrieChange + SetTxHash(txHash []byte) - GetIndex() int32 SetIndex(index int32) } -// StateChangesForTx is used to collect state changes for a transaction hash -type StateChangesForTx struct { - TxHash []byte `json:"txHash"` - StateChanges []StateChange `json:"stateChanges"` -} +//// StateChangesForTx is used to collect state changes for a transaction hash +//type StateChangesForTx struct { +// TxHash []byte `json:"txHash"` +// StateChanges []StateChange `json:"stateChanges"` +//} type stateChangesCollector struct { stateChanges []StateChange @@ -56,40 +60,71 @@ func (scc *stateChangesCollector) AddStateChange(stateChange StateChange) { scc.stateChangesMut.Unlock() } -func (scc *stateChangesCollector) getStateChangesForTxs() ([]StateChangesForTx, error) { +func (scc *stateChangesCollector) GetStateChangesForTxs() map[string]*data.StateChangesForTx { scc.stateChangesMut.Lock() defer scc.stateChangesMut.Unlock() - stateChangesForTxs := make([]StateChangesForTx, 0) - - for i := 0; i < len(scc.stateChanges); i++ { - txHash := scc.stateChanges[i].GetTxHash() - - if len(txHash) == 0 { - log.Warn("empty tx hash, state change event not associated to a transaction") - break - } - - innerStateChangesForTx := make([]StateChange, 0) - for j := i; j < len(scc.stateChanges); j++ { - txHash2 := scc.stateChanges[j].GetTxHash() - if !bytes.Equal(txHash, txHash2) { - i = j - break + stateChangesForTxs := make(map[string]*data.StateChangesForTx) + + for _, stateChange := range scc.stateChanges { + txHash := string(stateChange.GetTxHash()) + + if sc, ok := stateChangesForTxs[txHash]; !ok { + stateChangesForTxs[txHash] = &data.StateChangesForTx{StateChanges: []*data.StateChange{ + { + Type: stateChange.GetType(), + Index: stateChange.GetIndex(), + TxHash: stateChange.GetTxHash(), + MainTrieKey: stateChange.GetMainTrieKey(), + MainTrieVal: stateChange.GetMainTrieVal(), + Operation: stateChange.GetOperation(), + DataTrieChanges: stateChange.GetDataTrieChanges(), + }, + }, } - - innerStateChangesForTx = append(innerStateChangesForTx, scc.stateChanges[j]) - i = j + } else { + stateChangesForTxs[txHash].StateChanges = append(sc.StateChanges, + &data.StateChange{ + Type: stateChange.GetType(), + Index: stateChange.GetIndex(), + TxHash: stateChange.GetTxHash(), + MainTrieKey: stateChange.GetMainTrieKey(), + MainTrieVal: stateChange.GetMainTrieVal(), + Operation: stateChange.GetOperation(), + DataTrieChanges: stateChange.GetDataTrieChanges(), + }, + ) } - - stateChangesForTx := StateChangesForTx{ - TxHash: txHash, - StateChanges: innerStateChangesForTx, - } - stateChangesForTxs = append(stateChangesForTxs, stateChangesForTx) } - return stateChangesForTxs, nil + //for i := 0; i < len(scc.stateChanges); i++ { + // txHash := scc.stateChanges[i].GetTxHash() + // + // if len(txHash) == 0 { + // log.Warn("empty tx hash, state change event not associated to a transaction") + // break + // } + // + // innerStateChangesForTx := make([]StateChange, 0) + // for j := i; j < len(scc.stateChanges); j++ { + // txHash2 := scc.stateChanges[j].GetTxHash() + // if !bytes.Equal(txHash, txHash2) { + // i = j + // break + // } + // + // innerStateChangesForTx = append(innerStateChangesForTx, scc.stateChanges[j]) + // i = j + // } + // + // stateChangesForTx := StateChangesForTx{ + // TxHash: txHash, + // StateChanges: innerStateChangesForTx, + // } + // stateChangesForTxs = append(stateChangesForTxs, stateChangesForTx) + //} + + return stateChangesForTxs } // Reset resets the state changes collector @@ -155,32 +190,28 @@ func (scc *stateChangesCollector) RevertToIndex(index int) error { // Publish will export state changes func (scc *stateChangesCollector) Publish() error { - stateChangesForTx, err := scc.getStateChangesForTxs() - if err != nil { - return err - } + //stateChangesForTx, err := scc.GetStateChangesForTxs() + //if err != nil { + // return err + //} - printStateChanges(stateChangesForTx) + //printStateChanges(stateChangesForTx) return nil } -func printStateChanges(stateChanges []StateChangesForTx) { - for _, stateChange := range stateChanges { - - if stateChange.TxHash != nil { - fmt.Println(hex.EncodeToString(stateChange.TxHash)) - } - - for _, st := range stateChange.StateChanges { - fmt.Println(st) - } - } -} - -func (scc *stateChangesCollector) GetStateChanges() []StateChange { - return scc.stateChanges -} +//func printStateChanges(stateChanges []StateChangesForTx) { +// for _, stateChange := range stateChanges { +// +// if stateChange.TxHash != nil { +// fmt.Println(hex.EncodeToString(stateChange.TxHash)) +// } +// +// for _, st := range stateChange.StateChanges { +// fmt.Println(st) +// } +// } +//} // IsInterfaceNil returns true if there is no value under the interface func (scc *stateChangesCollector) IsInterfaceNil() bool { From f8c9ba57cb647c17e8c8b6f70612518db90e56f9 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Fri, 6 Sep 2024 15:52:59 +0300 Subject: [PATCH 25/55] fixed nil pointer reference. --- state/trackableDataTrie/trackableDataTrie.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/state/trackableDataTrie/trackableDataTrie.go b/state/trackableDataTrie/trackableDataTrie.go index 2d356ffab5d..5d1d890ecdc 100644 --- a/state/trackableDataTrie/trackableDataTrie.go +++ b/state/trackableDataTrie/trackableDataTrie.go @@ -331,6 +331,9 @@ func (tdt *trackableDataTrie) updateTrie(dtr state.DataTrie) ([]*stateChange.Dat stateChanges := make([]*stateChange.DataTrieChange, 0) for i := range newData { + if newData[i] == nil { + continue + } if len(newData[i].Key) == 0 { continue } From 09802ae5e0f422bbbc6bf7aa86f45e2ee0b127d5 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Fri, 6 Sep 2024 17:48:33 +0300 Subject: [PATCH 26/55] moved collected state changes in outport block. --- go.mod | 2 +- go.sum | 4 ++-- state/disabled/disabledStateChangesCollector.go | 2 +- state/interface.go | 3 ++- state/stateChanges/writeCollector.go | 6 +++--- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 68fb7ae914d..b32c7dfae9d 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/klauspost/cpuid/v2 v2.2.5 github.com/mitchellh/mapstructure v1.5.0 github.com/multiversx/mx-chain-communication-go v1.0.13-0.20240126121117-627adccf10ad - github.com/multiversx/mx-chain-core-go v1.2.21-0.20240906093529-4da59a927b44 + github.com/multiversx/mx-chain-core-go v1.2.21-0.20240906142519-0afffde1cb6b github.com/multiversx/mx-chain-crypto-go v1.2.10-0.20231206065052-38843c1f1479 github.com/multiversx/mx-chain-es-indexer-go v1.4.19-0.20240129150813-a772c480d33a github.com/multiversx/mx-chain-logger-go v1.0.14-0.20240129144507-d00e967c890c diff --git a/go.sum b/go.sum index 22d508569c2..2e34d2a65ab 100644 --- a/go.sum +++ b/go.sum @@ -387,8 +387,8 @@ github.com/multiversx/concurrent-map v0.1.4 h1:hdnbM8VE4b0KYJaGY5yJS2aNIW9TFFsUY github.com/multiversx/concurrent-map v0.1.4/go.mod h1:8cWFRJDOrWHOTNSqgYCUvwT7c7eFQ4U2vKMOp4A/9+o= github.com/multiversx/mx-chain-communication-go v1.0.13-0.20240126121117-627adccf10ad h1:izxTyKCxvT7z2mhXCWAZibSxwRVgLmq/kDovs4Nx/6Y= github.com/multiversx/mx-chain-communication-go v1.0.13-0.20240126121117-627adccf10ad/go.mod h1:n4E8BWIV0g3AcNGe1gf+vcjUC8A2QCJ4ARQSbiUDGrI= -github.com/multiversx/mx-chain-core-go v1.2.21-0.20240906093529-4da59a927b44 h1:3GdhKb2Q5KeW9A8uM3a/K9Ejjm8NwRJmpWOw1TQtCk0= -github.com/multiversx/mx-chain-core-go v1.2.21-0.20240906093529-4da59a927b44/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= +github.com/multiversx/mx-chain-core-go v1.2.21-0.20240906142519-0afffde1cb6b h1:41RtfXzxUZ34lRhtoVT3kJo/oUVDFvHToucX3fKdFs0= +github.com/multiversx/mx-chain-core-go v1.2.21-0.20240906142519-0afffde1cb6b/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= github.com/multiversx/mx-chain-crypto-go v1.2.10-0.20231206065052-38843c1f1479 h1:beVIhs5ysylwNplQ/bZ0h5DoDlqKNWgpWE/NMHHNmAw= github.com/multiversx/mx-chain-crypto-go v1.2.10-0.20231206065052-38843c1f1479/go.mod h1:Ap6p7QZFtwPlb++OvCG+85BfuZ+bLP/JtQp6EwjWJsI= github.com/multiversx/mx-chain-es-indexer-go v1.4.19-0.20240129150813-a772c480d33a h1:mOMUhbsjTq7n5oAv4KkVnL67ngS0+wkqmkiv1XJfBIY= diff --git a/state/disabled/disabledStateChangesCollector.go b/state/disabled/disabledStateChangesCollector.go index 836b92776b1..e6e0f23990f 100644 --- a/state/disabled/disabledStateChangesCollector.go +++ b/state/disabled/disabledStateChangesCollector.go @@ -49,7 +49,7 @@ func (d *disabledStateChangesCollector) Publish() error { return nil } -func (d *disabledStateChangesCollector) GetStateChangesForTxs() map[string]*stateChange.StateChangesForTx { +func (d *disabledStateChangesCollector) GetStateChangesForTxs() map[string]*stateChange.StateChanges { return nil } diff --git a/state/interface.go b/state/interface.go index 97c264407f4..dadc450706f 100644 --- a/state/interface.go +++ b/state/interface.go @@ -7,6 +7,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data/api" "github.com/multiversx/mx-chain-core-go/data/stateChange" + data "github.com/multiversx/mx-chain-core-go/data/stateChange" "github.com/multiversx/mx-chain-core-go/data/transaction" vmcommon "github.com/multiversx/mx-chain-vm-common-go" @@ -365,6 +366,6 @@ type StateChangesCollector interface { SetIndexToLastStateChange(index int) error RevertToIndex(index int) error Publish() error - GetStateChangesForTxs() map[string]*stateChange.StateChangesForTx + GetStateChangesForTxs() map[string]*data.StateChanges IsInterfaceNil() bool } diff --git a/state/stateChanges/writeCollector.go b/state/stateChanges/writeCollector.go index 2ed84f263ed..da58f2634a0 100644 --- a/state/stateChanges/writeCollector.go +++ b/state/stateChanges/writeCollector.go @@ -60,17 +60,17 @@ func (scc *stateChangesCollector) AddStateChange(stateChange StateChange) { scc.stateChangesMut.Unlock() } -func (scc *stateChangesCollector) GetStateChangesForTxs() map[string]*data.StateChangesForTx { +func (scc *stateChangesCollector) GetStateChangesForTxs() map[string]*data.StateChanges { scc.stateChangesMut.Lock() defer scc.stateChangesMut.Unlock() - stateChangesForTxs := make(map[string]*data.StateChangesForTx) + stateChangesForTxs := make(map[string]*data.StateChanges) for _, stateChange := range scc.stateChanges { txHash := string(stateChange.GetTxHash()) if sc, ok := stateChangesForTxs[txHash]; !ok { - stateChangesForTxs[txHash] = &data.StateChangesForTx{StateChanges: []*data.StateChange{ + stateChangesForTxs[txHash] = &data.StateChanges{StateChanges: []*data.StateChange{ { Type: stateChange.GetType(), Index: stateChange.GetIndex(), From 28f38a8476014f5f68da71bdcbf9c1436f6f57b1 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Fri, 6 Sep 2024 17:52:47 +0300 Subject: [PATCH 27/55] update core-go. --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index b32c7dfae9d..f94c99e7488 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/klauspost/cpuid/v2 v2.2.5 github.com/mitchellh/mapstructure v1.5.0 github.com/multiversx/mx-chain-communication-go v1.0.13-0.20240126121117-627adccf10ad - github.com/multiversx/mx-chain-core-go v1.2.21-0.20240906142519-0afffde1cb6b + github.com/multiversx/mx-chain-core-go v1.2.21-0.20240906145048-894187a86b01 github.com/multiversx/mx-chain-crypto-go v1.2.10-0.20231206065052-38843c1f1479 github.com/multiversx/mx-chain-es-indexer-go v1.4.19-0.20240129150813-a772c480d33a github.com/multiversx/mx-chain-logger-go v1.0.14-0.20240129144507-d00e967c890c From c2b2857547f12e8e0f1e794971eaeb45a22f8eae Mon Sep 17 00:00:00 2001 From: ssd04 Date: Fri, 6 Sep 2024 18:49:06 +0300 Subject: [PATCH 28/55] refactor to remove duplicated code --- state/stateChanges/dataAnalysisCollector.go | 54 +++++++-------------- state/stateChanges/writeCollector.go | 25 ++++++++-- state/stateChanges/writeCollector_test.go | 23 ++++++--- 3 files changed, 55 insertions(+), 47 deletions(-) diff --git a/state/stateChanges/dataAnalysisCollector.go b/state/stateChanges/dataAnalysisCollector.go index dc5d5e62a4f..e687cd86dff 100644 --- a/state/stateChanges/dataAnalysisCollector.go +++ b/state/stateChanges/dataAnalysisCollector.go @@ -12,11 +12,6 @@ import ( vmcommon "github.com/multiversx/mx-chain-vm-common-go" ) -const ( - workingDir = "." - defaultStateChangesPath = "stateChanges" -) - // TODO: use proto stucts type DataAnalysisStateChangeDTO struct { StateChange @@ -32,9 +27,8 @@ type DataAnalysisStateChangeDTO struct { } type DataAnalysisStateChangesForTx struct { - TxHash []byte `json:"txHash"` - Tx *transaction.Transaction `json:"tx"` - StateChanges []StateChange `json:"stateChanges"` + StateChangesForTx + Tx *transaction.Transaction `json:"tx"` } type userAccountHandler interface { @@ -127,50 +121,38 @@ func checkAccountChanges(oldAcc, newAcc vmcommon.AccountHandler, stateChange *Da // AddStateChange adds a new state change to the collector func (scc *dataAnalysisCollector) AddStateChange(stateChange StateChange) { scc.stateChangesMut.Lock() + defer scc.stateChangesMut.Unlock() + scc.stateChanges = append(scc.stateChanges, stateChange) - scc.stateChangesMut.Unlock() } -func (scc *dataAnalysisCollector) getStateChangesForTxs() ([]DataAnalysisStateChangesForTx, error) { +func (scc *dataAnalysisCollector) getDataAnalysisStateChangesForTxs() ([]DataAnalysisStateChangesForTx, error) { scc.stateChangesMut.Lock() defer scc.stateChangesMut.Unlock() - stateChangesForTxs := make([]DataAnalysisStateChangesForTx, 0) + stateChangesForTxs, err := scc.getStateChangesForTxs() + if err != nil { + return nil, err + } - for i := 0; i < len(scc.stateChanges); i++ { - txHash := scc.stateChanges[i].GetTxHash() + dataAnalysisStateChangesForTxs := make([]DataAnalysisStateChangesForTx, len(stateChangesForTxs)) - if len(txHash) == 0 { - log.Warn("empty tx hash, state change event not associated to a transaction") - break - } + for _, stateChangeForTx := range stateChangesForTxs { + txHash := string(stateChangeForTx.TxHash) cachedTx, txOk := scc.cachedTxs[string(txHash)] if !txOk { return nil, fmt.Errorf("did not find tx in cache") } - innerStateChangesForTx := make([]StateChange, 0) - for j := i; j < len(scc.stateChanges); j++ { - txHash2 := scc.stateChanges[j].GetTxHash() - if !bytes.Equal(txHash, txHash2) { - i = j - break - } - - innerStateChangesForTx = append(innerStateChangesForTx, scc.stateChanges[j]) - i = j - } - stateChangesForTx := DataAnalysisStateChangesForTx{ - TxHash: txHash, - Tx: cachedTx, - StateChanges: innerStateChangesForTx, + StateChangesForTx: stateChangeForTx, + Tx: cachedTx, } - stateChangesForTxs = append(stateChangesForTxs, stateChangesForTx) + dataAnalysisStateChangesForTxs = append(dataAnalysisStateChangesForTxs, stateChangesForTx) } - return stateChangesForTxs, nil + return dataAnalysisStateChangesForTxs, nil } // Reset resets the state changes collector @@ -178,13 +160,13 @@ func (scc *dataAnalysisCollector) Reset() { scc.stateChangesMut.Lock() defer scc.stateChangesMut.Unlock() - scc.stateChanges = make([]StateChange, 0) + scc.resetStateChangesUnprotected() scc.cachedTxs = make(map[string]*transaction.Transaction) } // Publish will export state changes func (scc *dataAnalysisCollector) Publish() error { - stateChangesForTx, err := scc.getStateChangesForTxs() + stateChangesForTx, err := scc.getDataAnalysisStateChangesForTxs() if err != nil { return err } diff --git a/state/stateChanges/writeCollector.go b/state/stateChanges/writeCollector.go index 188c8887160..3a500683645 100644 --- a/state/stateChanges/writeCollector.go +++ b/state/stateChanges/writeCollector.go @@ -29,6 +29,7 @@ type StateChange interface { SetTxHash(txHash []byte) GetIndex() int SetIndex(index int) + GetType() string } // StateChangeDTO is used to collect state changes @@ -43,6 +44,10 @@ type StateChangeDTO struct { DataTrieChanges []DataTrieChange `json:"dataTrieChanges"` } +func (sc *StateChangeDTO) GetType() string { + return sc.Type +} + func (sc *StateChangeDTO) GetIndex() int { return sc.Index } @@ -87,8 +92,12 @@ func (scc *stateChangesCollector) AddSaveAccountStateChange(_, _ vmcommon.Accoun // AddStateChange adds a new state change to the collector func (scc *stateChangesCollector) AddStateChange(stateChange StateChange) { scc.stateChangesMut.Lock() - scc.stateChanges = append(scc.stateChanges, stateChange) - scc.stateChangesMut.Unlock() + defer scc.stateChangesMut.Unlock() + + // TODO: add custom type for stateChange type + if stateChange.GetType() == "write" { + scc.stateChanges = append(scc.stateChanges, stateChange) + } } func (scc *stateChangesCollector) getStateChangesForTxs() ([]StateChangesForTx, error) { @@ -132,6 +141,10 @@ func (scc *stateChangesCollector) Reset() { scc.stateChangesMut.Lock() defer scc.stateChangesMut.Unlock() + scc.resetStateChangesUnprotected() +} + +func (scc *stateChangesCollector) resetStateChangesUnprotected() { scc.stateChanges = make([]StateChange, 0) } @@ -152,12 +165,16 @@ func (scc *stateChangesCollector) AddTxHashToCollectedStateChanges(txHash []byte // SetIndexToLastStateChange will set index to the last state change func (scc *stateChangesCollector) SetIndexToLastStateChange(index int) error { + scc.stateChangesMut.Lock() + defer scc.stateChangesMut.Unlock() + if index > len(scc.stateChanges) || index < 0 { return ErrStateChangesIndexOutOfBounds } - scc.stateChangesMut.Lock() - defer scc.stateChangesMut.Unlock() + if len(scc.stateChanges) == 0 { + return nil + } scc.stateChanges[len(scc.stateChanges)-1].SetIndex(index) diff --git a/state/stateChanges/writeCollector_test.go b/state/stateChanges/writeCollector_test.go index 91ae533f969..29b19326131 100644 --- a/state/stateChanges/writeCollector_test.go +++ b/state/stateChanges/writeCollector_test.go @@ -11,6 +11,12 @@ import ( "github.com/stretchr/testify/require" ) +func getDefaultStateChange() *StateChangeDTO { + return &StateChangeDTO{ + Type: "write", + } +} + func TestNewStateChangesCollector(t *testing.T) { t.Parallel() @@ -26,7 +32,7 @@ func TestStateChangesCollector_AddStateChange(t *testing.T) { numStateChanges := 10 for i := 0; i < numStateChanges; i++ { - scc.AddStateChange(&StateChangeDTO{}) + scc.AddStateChange(getDefaultStateChange()) } assert.Equal(t, numStateChanges, len(scc.stateChanges)) } @@ -44,6 +50,7 @@ func TestStateChangesCollector_GetStateChanges(t *testing.T) { numStateChanges := 10 for i := 0; i < numStateChanges; i++ { scc.AddStateChange(&StateChangeDTO{ + Type: "write", MainTrieKey: []byte(strconv.Itoa(i)), }) } @@ -76,6 +83,7 @@ func TestStateChangesCollector_GetStateChanges(t *testing.T) { numStateChanges := 10 for i := 0; i < numStateChanges; i++ { scc.AddStateChange(&StateChangeDTO{ + Type: "write", MainTrieKey: []byte(strconv.Itoa(i)), }) } @@ -97,6 +105,7 @@ func TestStateChangesCollector_AddTxHashToCollectedStateChanges(t *testing.T) { scc.AddTxHashToCollectedStateChanges([]byte("txHash0"), &transaction.Transaction{}) stateChange := &StateChangeDTO{ + Type: "write", MainTrieKey: []byte("mainTrieKey"), MainTrieVal: []byte("mainTrieVal"), DataTrieChanges: []DataTrieChange{{Key: []byte("dataTrieKey"), Val: []byte("dataTrieVal")}}, @@ -142,13 +151,13 @@ func TestStateChangesCollector_RevertToIndex(t *testing.T) { numStateChanges := 10 for i := 0; i < numStateChanges; i++ { - scc.AddStateChange(&StateChangeDTO{}) + scc.AddStateChange(getDefaultStateChange()) scc.SetIndexToLastStateChange(i) } scc.AddTxHashToCollectedStateChanges([]byte("txHash1"), &transaction.Transaction{}) for i := numStateChanges; i < numStateChanges*2; i++ { - scc.AddStateChange(&StateChangeDTO{}) + scc.AddStateChange(getDefaultStateChange()) scc.AddTxHashToCollectedStateChanges([]byte("txHash"+fmt.Sprintf("%d", i)), &transaction.Transaction{}) } scc.SetIndexToLastStateChange(numStateChanges) @@ -199,14 +208,14 @@ func TestStateChangesCollector_SetIndexToLastStateChange(t *testing.T) { numStateChanges := 10 for i := 0; i < numStateChanges; i++ { - scc.AddStateChange(&StateChangeDTO{}) + scc.AddStateChange(getDefaultStateChange()) err := scc.SetIndexToLastStateChange(i) require.Nil(t, err) } scc.AddTxHashToCollectedStateChanges([]byte("txHash1"), &transaction.Transaction{}) for i := numStateChanges; i < numStateChanges*2; i++ { - scc.AddStateChange(&StateChangeDTO{}) + scc.AddStateChange(getDefaultStateChange()) scc.AddTxHashToCollectedStateChanges([]byte("txHash"+fmt.Sprintf("%d", i)), &transaction.Transaction{}) } err := scc.SetIndexToLastStateChange(numStateChanges) @@ -224,11 +233,11 @@ func TestStateChangesCollector_Reset(t *testing.T) { numStateChanges := 10 for i := 0; i < numStateChanges; i++ { - scc.AddStateChange(&StateChangeDTO{}) + scc.AddStateChange(getDefaultStateChange()) } scc.AddTxHashToCollectedStateChanges([]byte("txHash"), &transaction.Transaction{}) for i := numStateChanges; i < numStateChanges*2; i++ { - scc.AddStateChange(&StateChangeDTO{}) + scc.AddStateChange(getDefaultStateChange()) } assert.Equal(t, numStateChanges*2, len(scc.stateChanges)) From d88fa86a5657ed25d9ae6aafca52b6078ba76d8a Mon Sep 17 00:00:00 2001 From: ssd04 Date: Fri, 6 Sep 2024 19:07:00 +0300 Subject: [PATCH 29/55] fix linter issues --- state/accountsDB.go | 2 +- state/stateChanges/writeCollector_test.go | 8 +++++--- state/trackableDataTrie/trackableDataTrie_test.go | 12 ------------ 3 files changed, 6 insertions(+), 16 deletions(-) diff --git a/state/accountsDB.go b/state/accountsDB.go index 3b1cf114b3f..8969a079127 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -1151,7 +1151,7 @@ func (adb *AccountsDB) journalize(entry JournalEntry) { "entry type", fmt.Sprintf("%T", entry), ) - adb.stateChangesCollector.SetIndexToLastStateChange(len(adb.entries)) + _ = adb.stateChangesCollector.SetIndexToLastStateChange(len(adb.entries)) if len(adb.entries) == 1 { adb.stackDebug = debug.Stack() diff --git a/state/stateChanges/writeCollector_test.go b/state/stateChanges/writeCollector_test.go index 29b19326131..634fcf52ce3 100644 --- a/state/stateChanges/writeCollector_test.go +++ b/state/stateChanges/writeCollector_test.go @@ -152,7 +152,8 @@ func TestStateChangesCollector_RevertToIndex(t *testing.T) { numStateChanges := 10 for i := 0; i < numStateChanges; i++ { scc.AddStateChange(getDefaultStateChange()) - scc.SetIndexToLastStateChange(i) + err := scc.SetIndexToLastStateChange(i) + require.Nil(t, err) } scc.AddTxHashToCollectedStateChanges([]byte("txHash1"), &transaction.Transaction{}) @@ -160,11 +161,12 @@ func TestStateChangesCollector_RevertToIndex(t *testing.T) { scc.AddStateChange(getDefaultStateChange()) scc.AddTxHashToCollectedStateChanges([]byte("txHash"+fmt.Sprintf("%d", i)), &transaction.Transaction{}) } - scc.SetIndexToLastStateChange(numStateChanges) + err := scc.SetIndexToLastStateChange(numStateChanges) + require.Nil(t, err) assert.Equal(t, numStateChanges*2, len(scc.stateChanges)) - err := scc.RevertToIndex(numStateChanges) + err = scc.RevertToIndex(numStateChanges) require.Nil(t, err) assert.Equal(t, numStateChanges*2-1, len(scc.stateChanges)) diff --git a/state/trackableDataTrie/trackableDataTrie_test.go b/state/trackableDataTrie/trackableDataTrie_test.go index 684b7c6e74a..99e492f186c 100644 --- a/state/trackableDataTrie/trackableDataTrie_test.go +++ b/state/trackableDataTrie/trackableDataTrie_test.go @@ -22,18 +22,6 @@ import ( "github.com/stretchr/testify/assert" ) -func createDefaultTrackableDataTrie() state.DataTrieTracker { - tdt, _ := trackableDataTrie.NewTrackableDataTrie( - []byte("identifier"), - &hashingMocks.HasherMock{}, - &marshallerMock.MarshalizerMock{}, - enableEpochsHandlerMock.NewEnableEpochsHandlerStubWithNoFlagsDefined(), - stateChanges.NewStateChangesCollector(), - ) - - return tdt -} - func TestNewTrackableDataTrie(t *testing.T) { t.Parallel() From f492061fcec8f9fb8b716cafd96c1b871cb11edd Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Mon, 9 Sep 2024 10:18:29 +0300 Subject: [PATCH 30/55] commit go.sum. --- go.sum | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go.sum b/go.sum index 2e34d2a65ab..e11d135b2c0 100644 --- a/go.sum +++ b/go.sum @@ -387,8 +387,8 @@ github.com/multiversx/concurrent-map v0.1.4 h1:hdnbM8VE4b0KYJaGY5yJS2aNIW9TFFsUY github.com/multiversx/concurrent-map v0.1.4/go.mod h1:8cWFRJDOrWHOTNSqgYCUvwT7c7eFQ4U2vKMOp4A/9+o= github.com/multiversx/mx-chain-communication-go v1.0.13-0.20240126121117-627adccf10ad h1:izxTyKCxvT7z2mhXCWAZibSxwRVgLmq/kDovs4Nx/6Y= github.com/multiversx/mx-chain-communication-go v1.0.13-0.20240126121117-627adccf10ad/go.mod h1:n4E8BWIV0g3AcNGe1gf+vcjUC8A2QCJ4ARQSbiUDGrI= -github.com/multiversx/mx-chain-core-go v1.2.21-0.20240906142519-0afffde1cb6b h1:41RtfXzxUZ34lRhtoVT3kJo/oUVDFvHToucX3fKdFs0= -github.com/multiversx/mx-chain-core-go v1.2.21-0.20240906142519-0afffde1cb6b/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= +github.com/multiversx/mx-chain-core-go v1.2.21-0.20240906145048-894187a86b01 h1:x1ZZ+4bPpz+k7xmAAAtsKo0m97c1eVCaugJI9VX6W3c= +github.com/multiversx/mx-chain-core-go v1.2.21-0.20240906145048-894187a86b01/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= github.com/multiversx/mx-chain-crypto-go v1.2.10-0.20231206065052-38843c1f1479 h1:beVIhs5ysylwNplQ/bZ0h5DoDlqKNWgpWE/NMHHNmAw= github.com/multiversx/mx-chain-crypto-go v1.2.10-0.20231206065052-38843c1f1479/go.mod h1:Ap6p7QZFtwPlb++OvCG+85BfuZ+bLP/JtQp6EwjWJsI= github.com/multiversx/mx-chain-es-indexer-go v1.4.19-0.20240129150813-a772c480d33a h1:mOMUhbsjTq7n5oAv4KkVnL67ngS0+wkqmkiv1XJfBIY= From b4b49d6e69328a8785ddbaac56cd145b7f30d6b2 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Mon, 9 Sep 2024 11:51:09 +0300 Subject: [PATCH 31/55] update dependencies. --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index f94c99e7488..5f5b68c2c24 100644 --- a/go.mod +++ b/go.mod @@ -15,12 +15,12 @@ require ( github.com/klauspost/cpuid/v2 v2.2.5 github.com/mitchellh/mapstructure v1.5.0 github.com/multiversx/mx-chain-communication-go v1.0.13-0.20240126121117-627adccf10ad - github.com/multiversx/mx-chain-core-go v1.2.21-0.20240906145048-894187a86b01 + github.com/multiversx/mx-chain-core-go v1.2.23-0.20240909083346-b1e2eff1cc9c github.com/multiversx/mx-chain-crypto-go v1.2.10-0.20231206065052-38843c1f1479 github.com/multiversx/mx-chain-es-indexer-go v1.4.19-0.20240129150813-a772c480d33a - github.com/multiversx/mx-chain-logger-go v1.0.14-0.20240129144507-d00e967c890c + github.com/multiversx/mx-chain-logger-go v1.0.15 github.com/multiversx/mx-chain-scenario-go v1.4.3-0.20240212160120-cc32d1580157 - github.com/multiversx/mx-chain-storage-go v1.0.15-0.20240129144933-b1c0d642d7f8 + github.com/multiversx/mx-chain-storage-go v1.0.17-0.20240909083434-9ff458d5c374 github.com/multiversx/mx-chain-vm-common-go v1.5.12-0.20240305123516-2231c71162a2 github.com/multiversx/mx-chain-vm-go v1.5.28-0.20240307121727-b8d371971d9a github.com/multiversx/mx-chain-vm-v1_2-go v1.2.66-0.20240308085208-3b5a4ab4dd34 diff --git a/go.sum b/go.sum index e11d135b2c0..6712154f5af 100644 --- a/go.sum +++ b/go.sum @@ -387,18 +387,18 @@ github.com/multiversx/concurrent-map v0.1.4 h1:hdnbM8VE4b0KYJaGY5yJS2aNIW9TFFsUY github.com/multiversx/concurrent-map v0.1.4/go.mod h1:8cWFRJDOrWHOTNSqgYCUvwT7c7eFQ4U2vKMOp4A/9+o= github.com/multiversx/mx-chain-communication-go v1.0.13-0.20240126121117-627adccf10ad h1:izxTyKCxvT7z2mhXCWAZibSxwRVgLmq/kDovs4Nx/6Y= github.com/multiversx/mx-chain-communication-go v1.0.13-0.20240126121117-627adccf10ad/go.mod h1:n4E8BWIV0g3AcNGe1gf+vcjUC8A2QCJ4ARQSbiUDGrI= -github.com/multiversx/mx-chain-core-go v1.2.21-0.20240906145048-894187a86b01 h1:x1ZZ+4bPpz+k7xmAAAtsKo0m97c1eVCaugJI9VX6W3c= -github.com/multiversx/mx-chain-core-go v1.2.21-0.20240906145048-894187a86b01/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= +github.com/multiversx/mx-chain-core-go v1.2.23-0.20240909083346-b1e2eff1cc9c h1:ULbgO2tzBEZeu88gtjOcVWFSNmCPkYQkK8ngh9z0/p8= +github.com/multiversx/mx-chain-core-go v1.2.23-0.20240909083346-b1e2eff1cc9c/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= github.com/multiversx/mx-chain-crypto-go v1.2.10-0.20231206065052-38843c1f1479 h1:beVIhs5ysylwNplQ/bZ0h5DoDlqKNWgpWE/NMHHNmAw= github.com/multiversx/mx-chain-crypto-go v1.2.10-0.20231206065052-38843c1f1479/go.mod h1:Ap6p7QZFtwPlb++OvCG+85BfuZ+bLP/JtQp6EwjWJsI= github.com/multiversx/mx-chain-es-indexer-go v1.4.19-0.20240129150813-a772c480d33a h1:mOMUhbsjTq7n5oAv4KkVnL67ngS0+wkqmkiv1XJfBIY= github.com/multiversx/mx-chain-es-indexer-go v1.4.19-0.20240129150813-a772c480d33a/go.mod h1:3aSGRJNvfUuPQkZUGHWuF11rPPxphsKGuAuIB+eD3is= -github.com/multiversx/mx-chain-logger-go v1.0.14-0.20240129144507-d00e967c890c h1:QIUOn8FgNRa5cir4BCWHZi/Qcr6Gg0eGNhns4+jy6+k= -github.com/multiversx/mx-chain-logger-go v1.0.14-0.20240129144507-d00e967c890c/go.mod h1:fH/fR/GEBsDjPkBoZDVJMoYo2HhlA7++DP6QfITJ1N8= +github.com/multiversx/mx-chain-logger-go v1.0.15 h1:HlNdK8etyJyL9NQ+6mIXyKPEBo+wRqOwi3n+m2QIHXc= +github.com/multiversx/mx-chain-logger-go v1.0.15/go.mod h1:t3PRKaWB1M+i6gUfD27KXgzLJJC+mAQiN+FLlL1yoGQ= github.com/multiversx/mx-chain-scenario-go v1.4.3-0.20240212160120-cc32d1580157 h1:ydzN3f+Y7H0InXuxAcNUSyVc+omNYL8uYtLqVzqaaX4= github.com/multiversx/mx-chain-scenario-go v1.4.3-0.20240212160120-cc32d1580157/go.mod h1:ndk45i9J9McuCJpTcgiaK4ocd0yhnBBCPrlFwO6GRcs= -github.com/multiversx/mx-chain-storage-go v1.0.15-0.20240129144933-b1c0d642d7f8 h1:/EYv/HGX0OKbeNFt667J0yZRtuJiZH0lEK8YtobuH/c= -github.com/multiversx/mx-chain-storage-go v1.0.15-0.20240129144933-b1c0d642d7f8/go.mod h1:zl1A6teNe39T8yhdZlkX3ckm5aLYrMIJJZ6Ord1E71M= +github.com/multiversx/mx-chain-storage-go v1.0.17-0.20240909083434-9ff458d5c374 h1:Npzzj83YCZm8u2jEUWuQtZVzFiCSPAwVRXm2biB/6Xo= +github.com/multiversx/mx-chain-storage-go v1.0.17-0.20240909083434-9ff458d5c374/go.mod h1:ZQciW/dv/33j2pg7aFrtQG2RAsnitOyx6zgKXJ3Q3sA= github.com/multiversx/mx-chain-vm-common-go v1.5.12-0.20240305123516-2231c71162a2 h1:sBH1Zf5jdMqS+1LDfXBmsIdmol8CFloPzjDCtmBZGEc= github.com/multiversx/mx-chain-vm-common-go v1.5.12-0.20240305123516-2231c71162a2/go.mod h1:OUyhCFqZKqUk1uaPsenyPDwO1830SlHNDU7Q7b6CBVI= github.com/multiversx/mx-chain-vm-go v1.5.28-0.20240307121727-b8d371971d9a h1:QvIC6R5sf0koeSwAs+Ye8J+CjNkAdaosTMSNTVBB8sA= From 291f6c0dd462e166f081aa37636e87edcfeea4a1 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Mon, 9 Sep 2024 15:11:11 +0300 Subject: [PATCH 32/55] fix after merge. --- genesis/mock/userAccountMock.go | 5 +- go.mod | 18 ++-- go.sum | 36 ++++---- outport/outport.go | 2 +- process/transaction/shardProcess.go | 3 +- state/accountsDBApi.go | 15 ++-- state/interface.go | 6 +- state/stateChanges/writeCollector.go | 103 ++++++++++++++++------ state/stateChanges/writeCollector_test.go | 17 ++-- testscommon/p2pmocks/messengerStub.go | 1 + testscommon/state/accountWrapperMock.go | 7 +- testscommon/state/userAccountStub.go | 7 +- 12 files changed, 135 insertions(+), 85 deletions(-) diff --git a/genesis/mock/userAccountMock.go b/genesis/mock/userAccountMock.go index 6a9e31c2dd0..28ef9e7c966 100644 --- a/genesis/mock/userAccountMock.go +++ b/genesis/mock/userAccountMock.go @@ -6,8 +6,9 @@ import ( "math/big" "github.com/multiversx/mx-chain-core-go/core" + "github.com/multiversx/mx-chain-core-go/data/stateChange" + "github.com/multiversx/mx-chain-go/common" - "github.com/multiversx/mx-chain-go/state/stateChanges" ) // ErrNegativeValue - @@ -148,7 +149,7 @@ func (uam *UserAccountMock) GetUserName() []byte { } // SaveDirtyData - -func (uam *UserAccountMock) SaveDirtyData(_ common.Trie) ([]stateChanges.DataTrieChange, []core.TrieData, error) { +func (uam *UserAccountMock) SaveDirtyData(_ common.Trie) ([]*stateChange.DataTrieChange, []core.TrieData, error) { return nil, nil, nil } diff --git a/go.mod b/go.mod index 5f5b68c2c24..c95171c6435 100644 --- a/go.mod +++ b/go.mod @@ -15,17 +15,17 @@ require ( github.com/klauspost/cpuid/v2 v2.2.5 github.com/mitchellh/mapstructure v1.5.0 github.com/multiversx/mx-chain-communication-go v1.0.13-0.20240126121117-627adccf10ad - github.com/multiversx/mx-chain-core-go v1.2.23-0.20240909083346-b1e2eff1cc9c - github.com/multiversx/mx-chain-crypto-go v1.2.10-0.20231206065052-38843c1f1479 + github.com/multiversx/mx-chain-core-go v1.2.23-0.20240909105439-e1a7a06d4ac4 + github.com/multiversx/mx-chain-crypto-go v1.2.12 github.com/multiversx/mx-chain-es-indexer-go v1.4.19-0.20240129150813-a772c480d33a github.com/multiversx/mx-chain-logger-go v1.0.15 - github.com/multiversx/mx-chain-scenario-go v1.4.3-0.20240212160120-cc32d1580157 - github.com/multiversx/mx-chain-storage-go v1.0.17-0.20240909083434-9ff458d5c374 - github.com/multiversx/mx-chain-vm-common-go v1.5.12-0.20240305123516-2231c71162a2 - github.com/multiversx/mx-chain-vm-go v1.5.28-0.20240307121727-b8d371971d9a - github.com/multiversx/mx-chain-vm-v1_2-go v1.2.66-0.20240308085208-3b5a4ab4dd34 - github.com/multiversx/mx-chain-vm-v1_3-go v1.3.67-0.20240308082903-132f9002736b - github.com/multiversx/mx-chain-vm-v1_4-go v1.4.96-0.20240308082831-f05004a05b35 + github.com/multiversx/mx-chain-scenario-go v1.4.4 + github.com/multiversx/mx-chain-storage-go v1.0.17-0.20240909105906-39ad1daf2a4c + github.com/multiversx/mx-chain-vm-common-go v1.5.14-0.20240812082318-afa839968da3 + github.com/multiversx/mx-chain-vm-go v1.5.32-0.20240812082514-1f3c25b3171e + github.com/multiversx/mx-chain-vm-v1_2-go v1.2.68 + github.com/multiversx/mx-chain-vm-v1_3-go v1.3.69 + github.com/multiversx/mx-chain-vm-v1_4-go v1.4.98 github.com/pelletier/go-toml v1.9.3 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.14.0 diff --git a/go.sum b/go.sum index 6712154f5af..898f63d9dfd 100644 --- a/go.sum +++ b/go.sum @@ -387,28 +387,28 @@ github.com/multiversx/concurrent-map v0.1.4 h1:hdnbM8VE4b0KYJaGY5yJS2aNIW9TFFsUY github.com/multiversx/concurrent-map v0.1.4/go.mod h1:8cWFRJDOrWHOTNSqgYCUvwT7c7eFQ4U2vKMOp4A/9+o= github.com/multiversx/mx-chain-communication-go v1.0.13-0.20240126121117-627adccf10ad h1:izxTyKCxvT7z2mhXCWAZibSxwRVgLmq/kDovs4Nx/6Y= github.com/multiversx/mx-chain-communication-go v1.0.13-0.20240126121117-627adccf10ad/go.mod h1:n4E8BWIV0g3AcNGe1gf+vcjUC8A2QCJ4ARQSbiUDGrI= -github.com/multiversx/mx-chain-core-go v1.2.23-0.20240909083346-b1e2eff1cc9c h1:ULbgO2tzBEZeu88gtjOcVWFSNmCPkYQkK8ngh9z0/p8= -github.com/multiversx/mx-chain-core-go v1.2.23-0.20240909083346-b1e2eff1cc9c/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= -github.com/multiversx/mx-chain-crypto-go v1.2.10-0.20231206065052-38843c1f1479 h1:beVIhs5ysylwNplQ/bZ0h5DoDlqKNWgpWE/NMHHNmAw= -github.com/multiversx/mx-chain-crypto-go v1.2.10-0.20231206065052-38843c1f1479/go.mod h1:Ap6p7QZFtwPlb++OvCG+85BfuZ+bLP/JtQp6EwjWJsI= +github.com/multiversx/mx-chain-core-go v1.2.23-0.20240909105439-e1a7a06d4ac4 h1:omuzhvGYFRAE0UA9UMatgSLTBlANKea0hU/jskzj6pM= +github.com/multiversx/mx-chain-core-go v1.2.23-0.20240909105439-e1a7a06d4ac4/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= +github.com/multiversx/mx-chain-crypto-go v1.2.12 h1:zWip7rpUS4CGthJxfKn5MZfMfYPjVjIiCID6uX5BSOk= +github.com/multiversx/mx-chain-crypto-go v1.2.12/go.mod h1:HzcPpCm1zanNct/6h2rIh+MFrlXbjA5C8+uMyXj3LI4= github.com/multiversx/mx-chain-es-indexer-go v1.4.19-0.20240129150813-a772c480d33a h1:mOMUhbsjTq7n5oAv4KkVnL67ngS0+wkqmkiv1XJfBIY= github.com/multiversx/mx-chain-es-indexer-go v1.4.19-0.20240129150813-a772c480d33a/go.mod h1:3aSGRJNvfUuPQkZUGHWuF11rPPxphsKGuAuIB+eD3is= github.com/multiversx/mx-chain-logger-go v1.0.15 h1:HlNdK8etyJyL9NQ+6mIXyKPEBo+wRqOwi3n+m2QIHXc= github.com/multiversx/mx-chain-logger-go v1.0.15/go.mod h1:t3PRKaWB1M+i6gUfD27KXgzLJJC+mAQiN+FLlL1yoGQ= -github.com/multiversx/mx-chain-scenario-go v1.4.3-0.20240212160120-cc32d1580157 h1:ydzN3f+Y7H0InXuxAcNUSyVc+omNYL8uYtLqVzqaaX4= -github.com/multiversx/mx-chain-scenario-go v1.4.3-0.20240212160120-cc32d1580157/go.mod h1:ndk45i9J9McuCJpTcgiaK4ocd0yhnBBCPrlFwO6GRcs= -github.com/multiversx/mx-chain-storage-go v1.0.17-0.20240909083434-9ff458d5c374 h1:Npzzj83YCZm8u2jEUWuQtZVzFiCSPAwVRXm2biB/6Xo= -github.com/multiversx/mx-chain-storage-go v1.0.17-0.20240909083434-9ff458d5c374/go.mod h1:ZQciW/dv/33j2pg7aFrtQG2RAsnitOyx6zgKXJ3Q3sA= -github.com/multiversx/mx-chain-vm-common-go v1.5.12-0.20240305123516-2231c71162a2 h1:sBH1Zf5jdMqS+1LDfXBmsIdmol8CFloPzjDCtmBZGEc= -github.com/multiversx/mx-chain-vm-common-go v1.5.12-0.20240305123516-2231c71162a2/go.mod h1:OUyhCFqZKqUk1uaPsenyPDwO1830SlHNDU7Q7b6CBVI= -github.com/multiversx/mx-chain-vm-go v1.5.28-0.20240307121727-b8d371971d9a h1:QvIC6R5sf0koeSwAs+Ye8J+CjNkAdaosTMSNTVBB8sA= -github.com/multiversx/mx-chain-vm-go v1.5.28-0.20240307121727-b8d371971d9a/go.mod h1:Xs0xFsPv+c1p8pwurLV7VBS7bEpIN/0jZrCwXVU26zw= -github.com/multiversx/mx-chain-vm-v1_2-go v1.2.66-0.20240308085208-3b5a4ab4dd34 h1:aLJhYiDBtWW4yjizhvQgTU00KfkK3oL3GnEh7pVUPRs= -github.com/multiversx/mx-chain-vm-v1_2-go v1.2.66-0.20240308085208-3b5a4ab4dd34/go.mod h1:8uugq3HUeDiE6G4AS3F8/B3zA1Pabzbl7SSD6Cebwz8= -github.com/multiversx/mx-chain-vm-v1_3-go v1.3.67-0.20240308082903-132f9002736b h1:iDDarqnGFZBXxqpaPWp8ePOqhG5G3DeAoopGgRLteu0= -github.com/multiversx/mx-chain-vm-v1_3-go v1.3.67-0.20240308082903-132f9002736b/go.mod h1:4uezxguZiX42kUaYMK/x46LLbgpYqn/iQXbcGM7zdM0= -github.com/multiversx/mx-chain-vm-v1_4-go v1.4.96-0.20240308082831-f05004a05b35 h1:yRfY/Mj1CXPoGd21F3y84cqBIKsktSgPuxz/5a7FA3w= -github.com/multiversx/mx-chain-vm-v1_4-go v1.4.96-0.20240308082831-f05004a05b35/go.mod h1:Nvanb5BZVhqnFFlWUtn7PQ/GIsl72zPVcMEw/ZvYiQA= +github.com/multiversx/mx-chain-scenario-go v1.4.4 h1:DVE2V+FPeyD/yWoC+KEfPK3jsFzHeruelESfpTlf460= +github.com/multiversx/mx-chain-scenario-go v1.4.4/go.mod h1:kI+TWR3oIEgUkbwkHCPo2CQ3VjIge+ezGTibiSGwMxo= +github.com/multiversx/mx-chain-storage-go v1.0.17-0.20240909105906-39ad1daf2a4c h1:1Wmh5iblKtE6S5NBdyPcHcKmiR3C1oF+MNRBVOrmAcc= +github.com/multiversx/mx-chain-storage-go v1.0.17-0.20240909105906-39ad1daf2a4c/go.mod h1:2VWMriwcI0s7kTnKHqJxFQnb0aMswc5KTbHp8MKw3Uk= +github.com/multiversx/mx-chain-vm-common-go v1.5.14-0.20240812082318-afa839968da3 h1:RlHKl5enbGrleB0Aea9TinZLLymS4WvG0/xAt/iRb6E= +github.com/multiversx/mx-chain-vm-common-go v1.5.14-0.20240812082318-afa839968da3/go.mod h1:OSvFbzdWThfRbLZbUsEr7bikBSaLrPJQ2iUm9jw9nXQ= +github.com/multiversx/mx-chain-vm-go v1.5.32-0.20240812082514-1f3c25b3171e h1:BkZtPUAQ9JlATkENydCLxPZ819hjop6laZtmC7Wzqec= +github.com/multiversx/mx-chain-vm-go v1.5.32-0.20240812082514-1f3c25b3171e/go.mod h1:j9FBeftA/BKfn0BbndKV7bNFJAzwCnYZuebsM/sufK0= +github.com/multiversx/mx-chain-vm-v1_2-go v1.2.68 h1:L3GoAVFtLLzr9ya0rVv1YdTUzS3MyM7kQNBSAjCNO2g= +github.com/multiversx/mx-chain-vm-v1_2-go v1.2.68/go.mod h1:ixxwib+1pXwSDHG5Wa34v0SRScF+BwFzH4wFWY31saI= +github.com/multiversx/mx-chain-vm-v1_3-go v1.3.69 h1:G/PLsyfQV4bMLs2amGRvaLKZoW1DC7M+7ecVaLuaCNc= +github.com/multiversx/mx-chain-vm-v1_3-go v1.3.69/go.mod h1:msY3zaS+K+R10ypqQs/jke6xdNAJzS38PGIaeJj2zhg= +github.com/multiversx/mx-chain-vm-v1_4-go v1.4.98 h1:/fYx4ClVPU48pTKh2qk4QVlve0xjjDpvzOakjFUtXJ8= +github.com/multiversx/mx-chain-vm-v1_4-go v1.4.98/go.mod h1:4vqG8bSmufZx263DMrmr8OLbO6q6//VPC4W9PxZLB5Q= github.com/multiversx/mx-components-big-int v1.0.0 h1:Wkr8lSzK2nDqixOrrBa47VNuqdhV1m/aJhaP1EMaiS8= github.com/multiversx/mx-components-big-int v1.0.0/go.mod h1:maIEMgHlNE2u78JaDD0oLzri+ShgU4okHfzP3LWGdQM= github.com/multiversx/protobuf v1.3.2 h1:RaNkxvGTGbA0lMcnHAN24qE1G1i+Xs5yHA6MDvQ4mSM= diff --git a/outport/outport.go b/outport/outport.go index e3210361d56..edcecc0691a 100644 --- a/outport/outport.go +++ b/outport/outport.go @@ -279,7 +279,7 @@ func (o *outport) saveValidatorsRatingBlocking(validatorsRating *outportcore.Val } } -// SaveAccounts will save accounts for every driver +// SaveAccounts will save accounts for every driver func (o *outport) SaveAccounts(accounts *outportcore.Accounts) { o.mutex.RLock() defer o.mutex.RUnlock() diff --git a/process/transaction/shardProcess.go b/process/transaction/shardProcess.go index 415fc519c66..025418e68f1 100644 --- a/process/transaction/shardProcess.go +++ b/process/transaction/shardProcess.go @@ -15,6 +15,7 @@ import ( "github.com/multiversx/mx-chain-core-go/data/vm" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" + logger "github.com/multiversx/mx-chain-logger-go" vmcommon "github.com/multiversx/mx-chain-vm-common-go" @@ -22,8 +23,6 @@ import ( "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/state" - logger "github.com/multiversx/mx-chain-logger-go" - vmcommon "github.com/multiversx/mx-chain-vm-common-go" ) var log = logger.GetOrCreate("process/transaction") diff --git a/state/accountsDBApi.go b/state/accountsDBApi.go index 7b53e4ad85c..ef2f7e26e9f 100644 --- a/state/accountsDBApi.go +++ b/state/accountsDBApi.go @@ -65,7 +65,8 @@ func (accountsDB *accountsDBApi) doRecreateTrieWithBlockInfo(newBlockInfo common return currentBlockInfo, nil } - err := accountsDB.innerAccountsAdapter.RecreateTrie(newBlockInfo.GetRootHash()) + rootHashHolder := holders.NewDefaultRootHashesHolder(newBlockInfo.GetRootHash()) + err := accountsDB.innerAccountsAdapter.RecreateTrie(rootHashHolder) if err != nil { accountsDB.blockInfo = nil return nil, err @@ -166,14 +167,8 @@ func (accountsDB *accountsDBApi) RootHash() ([]byte, error) { return blockInfo.GetRootHash(), nil } -// RecreateTrie is used to reload the trie based on an existing rootHash -func (accountsDB *accountsDBApi) RecreateTrie(rootHash []byte) error { - _, err := accountsDB.doRecreateTrieWithBlockInfo(holders.NewBlockInfo([]byte{}, 0, rootHash)) - return err -} - -// RecreateTrieFromEpoch is a not permitted operation in this implementation and thus, will return an error -func (accountsDB *accountsDBApi) RecreateTrieFromEpoch(options common.RootHashHolder) error { +// RecreateTrie is a not permitted operation in this implementation and thus, will return an error +func (accountsDB *accountsDBApi) RecreateTrie(options common.RootHashHolder) error { accountsDB.mutRecreatedTrieBlockInfo.Lock() defer accountsDB.mutRecreatedTrieBlockInfo.Unlock() @@ -186,7 +181,7 @@ func (accountsDB *accountsDBApi) RecreateTrieFromEpoch(options common.RootHashHo return nil } - err := accountsDB.innerAccountsAdapter.RecreateTrieFromEpoch(options) + err := accountsDB.innerAccountsAdapter.RecreateTrie(options) if err != nil { accountsDB.blockInfo = nil return err diff --git a/state/interface.go b/state/interface.go index 76090b45a27..49be503dcc5 100644 --- a/state/interface.go +++ b/state/interface.go @@ -6,7 +6,6 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data/api" - "github.com/multiversx/mx-chain-core-go/data/stateChange" data "github.com/multiversx/mx-chain-core-go/data/stateChange" "github.com/multiversx/mx-chain-core-go/data/transaction" vmcommon "github.com/multiversx/mx-chain-vm-common-go" @@ -164,7 +163,7 @@ type baseAccountHandler interface { GetRootHash() []byte SetDataTrie(trie common.Trie) DataTrie() common.DataTrieHandler - SaveDirtyData(trie common.Trie) ([]*stateChange.DataTrieChange, []core.TrieData, error) + SaveDirtyData(trie common.Trie) ([]*data.DataTrieChange, []core.TrieData, error) IsInterfaceNil() bool } @@ -262,7 +261,7 @@ type DataTrieTracker interface { SaveKeyValue(key []byte, value []byte) error SetDataTrie(tr common.Trie) DataTrie() common.DataTrieHandler - SaveDirtyData(common.Trie) ([]*stateChange.DataTrieChange, []core.TrieData, error) + SaveDirtyData(common.Trie) ([]*data.DataTrieChange, []core.TrieData, error) MigrateDataTrieLeaves(args vmcommon.ArgsMigrateDataTrieLeaves) error IsInterfaceNil() bool } @@ -369,4 +368,5 @@ type StateChangesCollector interface { RevertToIndex(index int) error Publish() error IsInterfaceNil() bool + GetStateChangesForTxs() map[string]*data.StateChanges } diff --git a/state/stateChanges/writeCollector.go b/state/stateChanges/writeCollector.go index da58f2634a0..277b51b9e52 100644 --- a/state/stateChanges/writeCollector.go +++ b/state/stateChanges/writeCollector.go @@ -1,7 +1,10 @@ package stateChanges import ( + "bytes" + "encoding/hex" "errors" + "fmt" "sync" data "github.com/multiversx/mx-chain-core-go/data/stateChange" @@ -28,11 +31,11 @@ type StateChange interface { SetIndex(index int32) } -//// StateChangesForTx is used to collect state changes for a transaction hash -//type StateChangesForTx struct { -// TxHash []byte `json:"txHash"` -// StateChanges []StateChange `json:"stateChanges"` -//} +// StateChangesForTx is used to collect state changes for a transaction hash +type StateChangesForTx struct { + TxHash []byte `json:"txHash"` + StateChanges []StateChange `json:"stateChanges"` +} type stateChangesCollector struct { stateChanges []StateChange @@ -56,8 +59,48 @@ func (scc *stateChangesCollector) AddSaveAccountStateChange(_, _ vmcommon.Accoun // AddStateChange adds a new state change to the collector func (scc *stateChangesCollector) AddStateChange(stateChange StateChange) { scc.stateChangesMut.Lock() - scc.stateChanges = append(scc.stateChanges, stateChange) - scc.stateChangesMut.Unlock() + defer scc.stateChangesMut.Unlock() + + // TODO: add custom type for stateChange type + if stateChange.GetType() == "write" { + scc.stateChanges = append(scc.stateChanges, stateChange) + } +} + +func (scc *stateChangesCollector) getStateChangesForTxs() ([]StateChangesForTx, error) { + scc.stateChangesMut.Lock() + defer scc.stateChangesMut.Unlock() + + stateChangesForTxs := make([]StateChangesForTx, 0) + + for i := 0; i < len(scc.stateChanges); i++ { + txHash := scc.stateChanges[i].GetTxHash() + + if len(txHash) == 0 { + log.Warn("empty tx hash, state change event not associated to a transaction") + break + } + + innerStateChangesForTx := make([]StateChange, 0) + for j := i; j < len(scc.stateChanges); j++ { + txHash2 := scc.stateChanges[j].GetTxHash() + if !bytes.Equal(txHash, txHash2) { + i = j + break + } + + innerStateChangesForTx = append(innerStateChangesForTx, scc.stateChanges[j]) + i = j + } + + stateChangesForTx := StateChangesForTx{ + TxHash: txHash, + StateChanges: innerStateChangesForTx, + } + stateChangesForTxs = append(stateChangesForTxs, stateChangesForTx) + } + + return stateChangesForTxs, nil } func (scc *stateChangesCollector) GetStateChangesForTxs() map[string]*data.StateChanges { @@ -132,6 +175,10 @@ func (scc *stateChangesCollector) Reset() { scc.stateChangesMut.Lock() defer scc.stateChangesMut.Unlock() + scc.resetStateChangesUnprotected() +} + +func (scc *stateChangesCollector) resetStateChangesUnprotected() { scc.stateChanges = make([]StateChange, 0) } @@ -152,12 +199,16 @@ func (scc *stateChangesCollector) AddTxHashToCollectedStateChanges(txHash []byte // SetIndexToLastStateChange will set index to the last state change func (scc *stateChangesCollector) SetIndexToLastStateChange(index int) error { + scc.stateChangesMut.Lock() + defer scc.stateChangesMut.Unlock() + if index > len(scc.stateChanges) || index < 0 { return ErrStateChangesIndexOutOfBounds } - scc.stateChangesMut.Lock() - defer scc.stateChangesMut.Unlock() + if len(scc.stateChanges) == 0 { + return nil + } scc.stateChanges[len(scc.stateChanges)-1].SetIndex(int32(index)) @@ -190,28 +241,28 @@ func (scc *stateChangesCollector) RevertToIndex(index int) error { // Publish will export state changes func (scc *stateChangesCollector) Publish() error { - //stateChangesForTx, err := scc.GetStateChangesForTxs() - //if err != nil { - // return err - //} + stateChangesForTx, err := scc.getStateChangesForTxs() + if err != nil { + return err + } - //printStateChanges(stateChangesForTx) + printStateChanges(stateChangesForTx) return nil } -//func printStateChanges(stateChanges []StateChangesForTx) { -// for _, stateChange := range stateChanges { -// -// if stateChange.TxHash != nil { -// fmt.Println(hex.EncodeToString(stateChange.TxHash)) -// } -// -// for _, st := range stateChange.StateChanges { -// fmt.Println(st) -// } -// } -//} +func printStateChanges(stateChanges []StateChangesForTx) { + for _, stateChange := range stateChanges { + + if stateChange.TxHash != nil { + fmt.Println(hex.EncodeToString(stateChange.TxHash)) + } + + for _, st := range stateChange.StateChanges { + fmt.Println(st) + } + } +} // IsInterfaceNil returns true if there is no value under the interface func (scc *stateChangesCollector) IsInterfaceNil() bool { diff --git a/state/stateChanges/writeCollector_test.go b/state/stateChanges/writeCollector_test.go index 634fcf52ce3..6341ac724b1 100644 --- a/state/stateChanges/writeCollector_test.go +++ b/state/stateChanges/writeCollector_test.go @@ -5,14 +5,15 @@ import ( "strconv" "testing" + data "github.com/multiversx/mx-chain-core-go/data/stateChange" "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func getDefaultStateChange() *StateChangeDTO { - return &StateChangeDTO{ +func getDefaultStateChange() *data.StateChange { + return &data.StateChange{ Type: "write", } } @@ -49,7 +50,7 @@ func TestStateChangesCollector_GetStateChanges(t *testing.T) { numStateChanges := 10 for i := 0; i < numStateChanges; i++ { - scc.AddStateChange(&StateChangeDTO{ + scc.AddStateChange(&data.StateChange{ Type: "write", MainTrieKey: []byte(strconv.Itoa(i)), }) @@ -66,7 +67,7 @@ func TestStateChangesCollector_GetStateChanges(t *testing.T) { assert.Equal(t, 1, len(stateChangesForTx)) assert.Equal(t, []byte("txHash"), stateChangesForTx[0].TxHash) for i := 0; i < len(stateChangesForTx[0].StateChanges); i++ { - sc, ok := stateChangesForTx[0].StateChanges[i].(*StateChangeDTO) + sc, ok := stateChangesForTx[0].StateChanges[i].(*data.StateChange) require.True(t, ok) assert.Equal(t, []byte(strconv.Itoa(i)), sc.MainTrieKey) @@ -82,7 +83,7 @@ func TestStateChangesCollector_GetStateChanges(t *testing.T) { numStateChanges := 10 for i := 0; i < numStateChanges; i++ { - scc.AddStateChange(&StateChangeDTO{ + scc.AddStateChange(&data.StateChange{ Type: "write", MainTrieKey: []byte(strconv.Itoa(i)), }) @@ -104,11 +105,11 @@ func TestStateChangesCollector_AddTxHashToCollectedStateChanges(t *testing.T) { scc.AddTxHashToCollectedStateChanges([]byte("txHash0"), &transaction.Transaction{}) - stateChange := &StateChangeDTO{ + stateChange := &data.StateChange{ Type: "write", MainTrieKey: []byte("mainTrieKey"), MainTrieVal: []byte("mainTrieVal"), - DataTrieChanges: []DataTrieChange{{Key: []byte("dataTrieKey"), Val: []byte("dataTrieVal")}}, + DataTrieChanges: []*data.DataTrieChange{{Key: []byte("dataTrieKey"), Val: []byte("dataTrieVal")}}, } scc.AddStateChange(stateChange) @@ -123,7 +124,7 @@ func TestStateChangesCollector_AddTxHashToCollectedStateChanges(t *testing.T) { assert.Equal(t, []byte("txHash"), stateChangesForTx[0].TxHash) assert.Equal(t, 1, len(stateChangesForTx[0].StateChanges)) - sc, ok := stateChangesForTx[0].StateChanges[0].(*StateChangeDTO) + sc, ok := stateChangesForTx[0].StateChanges[0].(*data.StateChange) require.True(t, ok) assert.Equal(t, []byte("mainTrieKey"), sc.MainTrieKey) diff --git a/testscommon/p2pmocks/messengerStub.go b/testscommon/p2pmocks/messengerStub.go index c48c95b9868..d3ab2154034 100644 --- a/testscommon/p2pmocks/messengerStub.go +++ b/testscommon/p2pmocks/messengerStub.go @@ -4,6 +4,7 @@ import ( "time" "github.com/multiversx/mx-chain-core-go/core" + "github.com/multiversx/mx-chain-go/p2p" ) diff --git a/testscommon/state/accountWrapperMock.go b/testscommon/state/accountWrapperMock.go index 5eecb17393f..8d67dfad6aa 100644 --- a/testscommon/state/accountWrapperMock.go +++ b/testscommon/state/accountWrapperMock.go @@ -7,15 +7,16 @@ import ( "math/big" "github.com/multiversx/mx-chain-core-go/core" + "github.com/multiversx/mx-chain-core-go/data/stateChange" + vmcommon "github.com/multiversx/mx-chain-vm-common-go" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/state/disabled" - "github.com/multiversx/mx-chain-go/state/stateChanges" "github.com/multiversx/mx-chain-go/state/trackableDataTrie" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" "github.com/multiversx/mx-chain-go/testscommon/marshallerMock" - vmcommon "github.com/multiversx/mx-chain-vm-common-go" ) var _ state.UserAccountHandler = (*AccountWrapMock)(nil) @@ -199,7 +200,7 @@ func (awm *AccountWrapMock) DataTrie() common.DataTrieHandler { } // SaveDirtyData - -func (awm *AccountWrapMock) SaveDirtyData(trie common.Trie) ([]stateChanges.DataTrieChange, []core.TrieData, error) { +func (awm *AccountWrapMock) SaveDirtyData(trie common.Trie) ([]*stateChange.DataTrieChange, []core.TrieData, error) { return awm.trackableDataTrie.SaveDirtyData(trie) } diff --git a/testscommon/state/userAccountStub.go b/testscommon/state/userAccountStub.go index 10bc9fa6932..1487bb625d0 100644 --- a/testscommon/state/userAccountStub.go +++ b/testscommon/state/userAccountStub.go @@ -6,10 +6,11 @@ import ( "math/big" "github.com/multiversx/mx-chain-core-go/core" + "github.com/multiversx/mx-chain-core-go/data/stateChange" + vmcommon "github.com/multiversx/mx-chain-vm-common-go" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/state" - "github.com/multiversx/mx-chain-go/state/stateChanges" - vmcommon "github.com/multiversx/mx-chain-vm-common-go" ) var _ state.UserAccountHandler = (*UserAccountStub)(nil) @@ -190,7 +191,7 @@ func (u *UserAccountStub) IsGuarded() bool { } // SaveDirtyData - -func (u *UserAccountStub) SaveDirtyData(_ common.Trie) ([]stateChanges.DataTrieChange, []core.TrieData, error) { +func (u *UserAccountStub) SaveDirtyData(_ common.Trie) ([]stateChange.DataTrieChange, []core.TrieData, error) { return nil, nil, nil } From 225035446fe8de4c502554f0bea42ac06bfc27d9 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Mon, 9 Sep 2024 15:45:32 +0300 Subject: [PATCH 33/55] fix version and imports. --- go.mod | 2 +- go.sum | 4 ++-- p2p/disabled/networkMessenger.go | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c95171c6435..86fd8eb1ef6 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/gorilla/websocket v1.5.0 github.com/klauspost/cpuid/v2 v2.2.5 github.com/mitchellh/mapstructure v1.5.0 - github.com/multiversx/mx-chain-communication-go v1.0.13-0.20240126121117-627adccf10ad + github.com/multiversx/mx-chain-communication-go v1.1.0 github.com/multiversx/mx-chain-core-go v1.2.23-0.20240909105439-e1a7a06d4ac4 github.com/multiversx/mx-chain-crypto-go v1.2.12 github.com/multiversx/mx-chain-es-indexer-go v1.4.19-0.20240129150813-a772c480d33a diff --git a/go.sum b/go.sum index 898f63d9dfd..f587754dd0a 100644 --- a/go.sum +++ b/go.sum @@ -385,8 +385,8 @@ github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/n github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/multiversx/concurrent-map v0.1.4 h1:hdnbM8VE4b0KYJaGY5yJS2aNIW9TFFsUYwbO0993uPI= github.com/multiversx/concurrent-map v0.1.4/go.mod h1:8cWFRJDOrWHOTNSqgYCUvwT7c7eFQ4U2vKMOp4A/9+o= -github.com/multiversx/mx-chain-communication-go v1.0.13-0.20240126121117-627adccf10ad h1:izxTyKCxvT7z2mhXCWAZibSxwRVgLmq/kDovs4Nx/6Y= -github.com/multiversx/mx-chain-communication-go v1.0.13-0.20240126121117-627adccf10ad/go.mod h1:n4E8BWIV0g3AcNGe1gf+vcjUC8A2QCJ4ARQSbiUDGrI= +github.com/multiversx/mx-chain-communication-go v1.1.0 h1:J7bX6HoN3HiHY7cUeEjG8AJWgQDDPcY+OPDOsSUOkRE= +github.com/multiversx/mx-chain-communication-go v1.1.0/go.mod h1:WK6bP4pGEHGDDna/AYRIMtl6G9OA0NByI1Lw8PmOnRM= github.com/multiversx/mx-chain-core-go v1.2.23-0.20240909105439-e1a7a06d4ac4 h1:omuzhvGYFRAE0UA9UMatgSLTBlANKea0hU/jskzj6pM= github.com/multiversx/mx-chain-core-go v1.2.23-0.20240909105439-e1a7a06d4ac4/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= github.com/multiversx/mx-chain-crypto-go v1.2.12 h1:zWip7rpUS4CGthJxfKn5MZfMfYPjVjIiCID6uX5BSOk= diff --git a/p2p/disabled/networkMessenger.go b/p2p/disabled/networkMessenger.go index 4f854d976bc..f76228462f9 100644 --- a/p2p/disabled/networkMessenger.go +++ b/p2p/disabled/networkMessenger.go @@ -4,6 +4,7 @@ import ( "time" "github.com/multiversx/mx-chain-core-go/core" + "github.com/multiversx/mx-chain-go/p2p" ) From 62d23268f23aabc8836d8a226865cbb3b1c182b7 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Mon, 9 Sep 2024 16:34:04 +0300 Subject: [PATCH 34/55] fix some tests. --- outport/process/outportDataProvider_test.go | 5 ++++- process/transaction/shardProcess.go | 6 ++++++ state/accountsDB_test.go | 14 ++++++++------ 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/outport/process/outportDataProvider_test.go b/outport/process/outportDataProvider_test.go index ef1422d230a..0f801968ac6 100644 --- a/outport/process/outportDataProvider_test.go +++ b/outport/process/outportDataProvider_test.go @@ -12,8 +12,11 @@ import ( "github.com/multiversx/mx-chain-core-go/data/rewardTx" "github.com/multiversx/mx-chain-core-go/data/smartContractResult" "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-go/outport/mock" "github.com/multiversx/mx-chain-go/outport/process/transactionsfee" + "github.com/multiversx/mx-chain-go/state/stateChanges" "github.com/multiversx/mx-chain-go/testscommon" commonMocks "github.com/multiversx/mx-chain-go/testscommon/common" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" @@ -21,7 +24,6 @@ import ( "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" "github.com/multiversx/mx-chain-go/testscommon/marshallerMock" "github.com/multiversx/mx-chain-go/testscommon/shardingMocks" - "github.com/stretchr/testify/require" ) func createArgOutportDataProvider() ArgOutportDataProvider { @@ -45,6 +47,7 @@ func createArgOutportDataProvider() ArgOutportDataProvider { ExecutionOrderHandler: &commonMocks.TxExecutionOrderHandlerStub{}, Marshaller: &marshallerMock.MarshalizerMock{}, Hasher: &hashingMocks.HasherMock{}, + StateChangesCollector: stateChanges.NewStateChangesCollector(), } } diff --git a/process/transaction/shardProcess.go b/process/transaction/shardProcess.go index 025418e68f1..9014f05949b 100644 --- a/process/transaction/shardProcess.go +++ b/process/transaction/shardProcess.go @@ -148,6 +148,12 @@ func NewTxProcessor(args ArgsNewTxProcessor) (*txProcessor, error) { if check.IfNil(args.TxLogsProcessor) { return nil, process.ErrNilTxLogsProcessor } + if check.IfNil(args.RelayedTxV3Processor) { + return nil, process.ErrNilRelayedTxV3Processor + } + if check.IfNil(args.FailedTxLogsAccumulator) { + return nil, process.ErrNilFailedTxLogsAccumulator + } baseTxProcess := &baseTxProcessor{ accounts: args.Accounts, diff --git a/state/accountsDB_test.go b/state/accountsDB_test.go index f6f91242cf4..085d1bec004 100644 --- a/state/accountsDB_test.go +++ b/state/accountsDB_test.go @@ -16,7 +16,13 @@ import ( "github.com/multiversx/mx-chain-core-go/core/atomic" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/core/keyValStorage" + data "github.com/multiversx/mx-chain-core-go/data/stateChange" "github.com/multiversx/mx-chain-core-go/marshal" + vmcommon "github.com/multiversx/mx-chain-vm-common-go" + "github.com/multiversx/mx-chain-vm-common-go/dataTrieMigrator" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/errChan" "github.com/multiversx/mx-chain-go/common/holders" @@ -42,10 +48,6 @@ import ( "github.com/multiversx/mx-chain-go/testscommon/storageManager" trieMock "github.com/multiversx/mx-chain-go/testscommon/trie" "github.com/multiversx/mx-chain-go/trie" - vmcommon "github.com/multiversx/mx-chain-vm-common-go" - "github.com/multiversx/mx-chain-vm-common-go/dataTrieMigrator" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) const trieDbOperationDelay = time.Second @@ -372,8 +374,8 @@ func TestAccountsDB_SaveAccountSavesCodeAndDataTrieForUserAccount(t *testing.T) }) dtt := &trieMock.DataTrieTrackerStub{ - SaveDirtyDataCalled: func(_ common.Trie) ([]stateChanges.DataTrieChange, []core.TrieData, error) { - var stateChanges []stateChanges.DataTrieChange + SaveDirtyDataCalled: func(_ common.Trie) ([]*data.DataTrieChange, []core.TrieData, error) { + var stateChanges []*data.DataTrieChange oldVal := []core.TrieData{ { Key: []byte("key"), From c86068433c903a6f126db14907031d2542b091e5 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Mon, 9 Sep 2024 17:05:35 +0300 Subject: [PATCH 35/55] fix broken import. --- testscommon/state/userAccountStub.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testscommon/state/userAccountStub.go b/testscommon/state/userAccountStub.go index 1487bb625d0..d4af407abba 100644 --- a/testscommon/state/userAccountStub.go +++ b/testscommon/state/userAccountStub.go @@ -191,7 +191,7 @@ func (u *UserAccountStub) IsGuarded() bool { } // SaveDirtyData - -func (u *UserAccountStub) SaveDirtyData(_ common.Trie) ([]stateChange.DataTrieChange, []core.TrieData, error) { +func (u *UserAccountStub) SaveDirtyData(_ common.Trie) ([]*stateChange.DataTrieChange, []core.TrieData, error) { return nil, nil, nil } From c48252341415d9ec4ff17151fa3c4680fe5eef25 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Mon, 9 Sep 2024 17:19:47 +0300 Subject: [PATCH 36/55] another round of tests fixed. --- state/stateChanges/export_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/state/stateChanges/export_test.go b/state/stateChanges/export_test.go index 3e52d0985b2..86350f45a5c 100644 --- a/state/stateChanges/export_test.go +++ b/state/stateChanges/export_test.go @@ -2,6 +2,6 @@ package stateChanges // GetStateChanges - func (scc *stateChangesCollector) GetStateChanges() []StateChangesForTx { - scs, _ := scc.GetStateChangesForTxs() + scs, _ := scc.getStateChangesForTxs() return scs } From 30d5dd97088dd1b54fedeebd8206a06a269371e7 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Mon, 9 Sep 2024 17:29:48 +0300 Subject: [PATCH 37/55] fix linter. --- outport/disabled/disabledOutport.go | 1 - 1 file changed, 1 deletion(-) diff --git a/outport/disabled/disabledOutport.go b/outport/disabled/disabledOutport.go index c4e861ec210..9000f800cce 100644 --- a/outport/disabled/disabledOutport.go +++ b/outport/disabled/disabledOutport.go @@ -41,7 +41,6 @@ func (n *disabledOutport) SaveAccounts(_ *outportcore.Accounts) { // SaveStateChanges does nothing func (n *disabledOutport) SaveStateChanges() { - return } // FinalizedBlock does nothing From 9d983f1e24839d105e908136bb4453ea4d938211 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Tue, 10 Sep 2024 15:42:15 +0300 Subject: [PATCH 38/55] fixes after review. --- outport/disabled/disabledOutport.go | 4 --- outport/factory/hostDriverFactory.go | 10 +++---- outport/factory/outportFactory.go | 6 ++-- outport/mock/driverStub.go | 10 ------- outport/notifier/eventNotifier.go | 5 ---- scripts/testnet/variables.sh | 10 +++---- .../disabled/disabledStateChangesCollector.go | 1 + state/stateChanges/dataAnalysisCollector.go | 4 --- state/stateChanges/writeCollector.go | 28 +------------------ 9 files changed, 13 insertions(+), 65 deletions(-) diff --git a/outport/disabled/disabledOutport.go b/outport/disabled/disabledOutport.go index 9000f800cce..24a53eaca18 100644 --- a/outport/disabled/disabledOutport.go +++ b/outport/disabled/disabledOutport.go @@ -39,10 +39,6 @@ func (n *disabledOutport) SaveValidatorsRating(_ *outportcore.ValidatorsRating) func (n *disabledOutport) SaveAccounts(_ *outportcore.Accounts) { } -// SaveStateChanges does nothing -func (n *disabledOutport) SaveStateChanges() { -} - // FinalizedBlock does nothing func (n *disabledOutport) FinalizedBlock(_ *outportcore.FinalizedBlock) { } diff --git a/outport/factory/hostDriverFactory.go b/outport/factory/hostDriverFactory.go index 90b5b562dac..efcf6b6c159 100644 --- a/outport/factory/hostDriverFactory.go +++ b/outport/factory/hostDriverFactory.go @@ -5,18 +5,16 @@ import ( "github.com/multiversx/mx-chain-communication-go/websocket/factory" "github.com/multiversx/mx-chain-core-go/marshal" + logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-go/config" "github.com/multiversx/mx-chain-go/outport" "github.com/multiversx/mx-chain-go/outport/host" - "github.com/multiversx/mx-chain-go/state" - - logger "github.com/multiversx/mx-chain-logger-go" ) type ArgsHostDriverFactory struct { - HostConfig config.HostDriversConfig - Marshaller marshal.Marshalizer - StateChangesCollector state.StateChangesCollector + HostConfig config.HostDriversConfig + Marshaller marshal.Marshalizer } var log = logger.GetOrCreate("outport/factory/hostdriver") diff --git a/outport/factory/outportFactory.go b/outport/factory/outportFactory.go index 84d963131ae..a137175053e 100644 --- a/outport/factory/outportFactory.go +++ b/outport/factory/outportFactory.go @@ -74,14 +74,12 @@ func createAndSubscribeElasticDriverIfNeeded( return nil } - //TODO: this needs discussed. - _, err := indexerFactory.NewIndexer(args) + elasticDriver, err := indexerFactory.NewIndexer(args) if err != nil { return err } - return nil - //return outport.SubscribeDriver(elasticDriver) + return outport.SubscribeDriver(elasticDriver) } func createAndSubscribeEventNotifierIfNeeded( diff --git a/outport/mock/driverStub.go b/outport/mock/driverStub.go index 2b7cfe35159..144e09a0ab6 100644 --- a/outport/mock/driverStub.go +++ b/outport/mock/driverStub.go @@ -19,7 +19,6 @@ type DriverStub struct { CloseCalled func() error RegisterHandlerCalled func(handlerFunction func() error, topic string) error SetCurrentSettingsCalled func(config outportcore.OutportConfig) error - SaveStateChangesCalled func() error } // SaveBlock - @@ -108,15 +107,6 @@ func (d *DriverStub) RegisterHandler(handlerFunction func() error, topic string) return nil } -// SaveStateChanges - -func (d *DriverStub) SaveStateChanges() error { - if d.SaveStateChangesCalled != nil { - return d.SaveStateChangesCalled() - } - - return nil -} - // Close - func (d *DriverStub) Close() error { if d.CloseCalled != nil { diff --git a/outport/notifier/eventNotifier.go b/outport/notifier/eventNotifier.go index 247fd7e3d23..828b027c88e 100644 --- a/outport/notifier/eventNotifier.go +++ b/outport/notifier/eventNotifier.go @@ -115,11 +115,6 @@ func (en *eventNotifier) SaveAccounts(_ *outport.Accounts) error { return nil } -// SaveStateChanges does nothing -func (en *eventNotifier) SaveStateChanges() error { - return nil -} - // GetMarshaller returns internal marshaller func (en *eventNotifier) GetMarshaller() marshal.Marshalizer { return en.marshalizer diff --git a/scripts/testnet/variables.sh b/scripts/testnet/variables.sh index 3812adbc5df..c5a5b013523 100644 --- a/scripts/testnet/variables.sh +++ b/scripts/testnet/variables.sh @@ -12,7 +12,7 @@ export USE_PROXY=1 # Enable the MultiversX Transaction Generator. Note that this is a private # repository (mx-chain-txgen-go). -export USE_TXGEN=1 +export USE_TXGEN=0 # Path where the testnet will be instantiated. This folder is assumed to not # exist, but it doesn't matter if it already does. It will be created if not, @@ -52,13 +52,13 @@ export GENESIS_STAKE_TYPE="direct" #'delegated' or 'direct' as in direct stake export OBSERVERS_ANTIFLOOD_DISABLE=0 # Shard structure -export SHARDCOUNT=1 -export SHARD_VALIDATORCOUNT=1 +export SHARDCOUNT=2 +export SHARD_VALIDATORCOUNT=3 export SHARD_OBSERVERCOUNT=1 -export SHARD_CONSENSUS_SIZE=1 +export SHARD_CONSENSUS_SIZE=3 # Metashard structure -export META_VALIDATORCOUNT=1 +export META_VALIDATORCOUNT=3 export META_OBSERVERCOUNT=1 export META_CONSENSUS_SIZE=$META_VALIDATORCOUNT diff --git a/state/disabled/disabledStateChangesCollector.go b/state/disabled/disabledStateChangesCollector.go index e6e0f23990f..629deeb293a 100644 --- a/state/disabled/disabledStateChangesCollector.go +++ b/state/disabled/disabledStateChangesCollector.go @@ -49,6 +49,7 @@ func (d *disabledStateChangesCollector) Publish() error { return nil } +// GetStateChangesForTxs - func (d *disabledStateChangesCollector) GetStateChangesForTxs() map[string]*stateChange.StateChanges { return nil } diff --git a/state/stateChanges/dataAnalysisCollector.go b/state/stateChanges/dataAnalysisCollector.go index a58bcdd7e9e..0fe627b799f 100644 --- a/state/stateChanges/dataAnalysisCollector.go +++ b/state/stateChanges/dataAnalysisCollector.go @@ -187,10 +187,6 @@ func (scc *dataAnalysisCollector) Publish() error { return nil } -func (scc *dataAnalysisCollector) GetStateChangesForTx() []StateChange { - return scc.stateChanges -} - // IsInterfaceNil returns true if there is no value under the interface func (scc *dataAnalysisCollector) IsInterfaceNil() bool { return scc == nil diff --git a/state/stateChanges/writeCollector.go b/state/stateChanges/writeCollector.go index 277b51b9e52..8cebce4dc54 100644 --- a/state/stateChanges/writeCollector.go +++ b/state/stateChanges/writeCollector.go @@ -103,6 +103,7 @@ func (scc *stateChangesCollector) getStateChangesForTxs() ([]StateChangesForTx, return stateChangesForTxs, nil } +// GetStateChangesForTxs will retrieve the state changes linked with the tx hash. func (scc *stateChangesCollector) GetStateChangesForTxs() map[string]*data.StateChanges { scc.stateChangesMut.Lock() defer scc.stateChangesMut.Unlock() @@ -140,33 +141,6 @@ func (scc *stateChangesCollector) GetStateChangesForTxs() map[string]*data.State } } - //for i := 0; i < len(scc.stateChanges); i++ { - // txHash := scc.stateChanges[i].GetTxHash() - // - // if len(txHash) == 0 { - // log.Warn("empty tx hash, state change event not associated to a transaction") - // break - // } - // - // innerStateChangesForTx := make([]StateChange, 0) - // for j := i; j < len(scc.stateChanges); j++ { - // txHash2 := scc.stateChanges[j].GetTxHash() - // if !bytes.Equal(txHash, txHash2) { - // i = j - // break - // } - // - // innerStateChangesForTx = append(innerStateChangesForTx, scc.stateChanges[j]) - // i = j - // } - // - // stateChangesForTx := StateChangesForTx{ - // TxHash: txHash, - // StateChanges: innerStateChangesForTx, - // } - // stateChangesForTxs = append(stateChangesForTxs, stateChangesForTx) - //} - return stateChangesForTxs } From 776d2b109a9ec660c7a4ca67d96fd1a46a51b1af Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Tue, 10 Sep 2024 15:46:34 +0300 Subject: [PATCH 39/55] fix missing argument. --- factory/status/statusComponents.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/factory/status/statusComponents.go b/factory/status/statusComponents.go index cea175fdfaf..73ecb5e2777 100644 --- a/factory/status/statusComponents.go +++ b/factory/status/statusComponents.go @@ -287,9 +287,8 @@ func (scf *statusComponentsFactory) makeHostDriversArgs() ([]outportDriverFactor } argsHostDriverFactorySlice = append(argsHostDriverFactorySlice, outportDriverFactory.ArgsHostDriverFactory{ - Marshaller: marshaller, - HostConfig: hostConfig, - StateChangesCollector: scf.stateComponents.StateChangesCollector(), + Marshaller: marshaller, + HostConfig: hostConfig, }) } From 1f90b5d740816058dd6a98ac4d87a3405680fde3 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Thu, 12 Sep 2024 13:43:45 +0300 Subject: [PATCH 40/55] fixes after review. --- go.mod | 2 +- go.sum | 4 ++-- outport/process/outportDataProvider.go | 1 - process/block/metablock.go | 1 + process/block/shardblock.go | 4 +++- scripts/testnet/variables.sh | 10 +++++----- state/accountsDB.go | 3 ++- 7 files changed, 14 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 86fd8eb1ef6..a0bd43bc59f 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/multiversx/mx-chain-communication-go v1.1.0 github.com/multiversx/mx-chain-core-go v1.2.23-0.20240909105439-e1a7a06d4ac4 github.com/multiversx/mx-chain-crypto-go v1.2.12 - github.com/multiversx/mx-chain-es-indexer-go v1.4.19-0.20240129150813-a772c480d33a + github.com/multiversx/mx-chain-es-indexer-go v1.4.21 github.com/multiversx/mx-chain-logger-go v1.0.15 github.com/multiversx/mx-chain-scenario-go v1.4.4 github.com/multiversx/mx-chain-storage-go v1.0.17-0.20240909105906-39ad1daf2a4c diff --git a/go.sum b/go.sum index f587754dd0a..3b4364c1cbc 100644 --- a/go.sum +++ b/go.sum @@ -391,8 +391,8 @@ github.com/multiversx/mx-chain-core-go v1.2.23-0.20240909105439-e1a7a06d4ac4 h1: github.com/multiversx/mx-chain-core-go v1.2.23-0.20240909105439-e1a7a06d4ac4/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= github.com/multiversx/mx-chain-crypto-go v1.2.12 h1:zWip7rpUS4CGthJxfKn5MZfMfYPjVjIiCID6uX5BSOk= github.com/multiversx/mx-chain-crypto-go v1.2.12/go.mod h1:HzcPpCm1zanNct/6h2rIh+MFrlXbjA5C8+uMyXj3LI4= -github.com/multiversx/mx-chain-es-indexer-go v1.4.19-0.20240129150813-a772c480d33a h1:mOMUhbsjTq7n5oAv4KkVnL67ngS0+wkqmkiv1XJfBIY= -github.com/multiversx/mx-chain-es-indexer-go v1.4.19-0.20240129150813-a772c480d33a/go.mod h1:3aSGRJNvfUuPQkZUGHWuF11rPPxphsKGuAuIB+eD3is= +github.com/multiversx/mx-chain-es-indexer-go v1.4.21 h1:rzxXCkgOsqj67GRYtqzKuf9XgHwnZLTZhU90Ck3VbrE= +github.com/multiversx/mx-chain-es-indexer-go v1.4.21/go.mod h1:V9xxOBkfV7GjN4K5SODaOetoGVpQm4snibMVPCjL0Kk= github.com/multiversx/mx-chain-logger-go v1.0.15 h1:HlNdK8etyJyL9NQ+6mIXyKPEBo+wRqOwi3n+m2QIHXc= github.com/multiversx/mx-chain-logger-go v1.0.15/go.mod h1:t3PRKaWB1M+i6gUfD27KXgzLJJC+mAQiN+FLlL1yoGQ= github.com/multiversx/mx-chain-scenario-go v1.4.4 h1:DVE2V+FPeyD/yWoC+KEfPK3jsFzHeruelESfpTlf460= diff --git a/outport/process/outportDataProvider.go b/outport/process/outportDataProvider.go index cd3028c600f..0528dd6b06d 100644 --- a/outport/process/outportDataProvider.go +++ b/outport/process/outportDataProvider.go @@ -140,7 +140,6 @@ func (odp *outportDataProvider) PrepareOutportSaveBlockData(arg ArgPrepareOutpor } stateChanges := odp.stateChangesCollector.GetStateChangesForTxs() - odp.stateChangesCollector.Reset() return &outportcore.OutportBlockWithHeaderAndBody{ OutportBlock: &outportcore.OutportBlock{ diff --git a/process/block/metablock.go b/process/block/metablock.go index ebae848ae4f..2343f17e78d 100644 --- a/process/block/metablock.go +++ b/process/block/metablock.go @@ -1307,6 +1307,7 @@ func (mp *metaProcessor) CommitBlock( // TODO: Should be sent also validatorInfoTxs alongside rewardsTxs -> mp.validatorInfoCreator.GetValidatorInfoTxs(body) ? mp.indexBlock(header, headerHash, body, lastMetaBlock, notarizedHeadersHashes, rewardsTxs) + mp.stateChangesCollector.Reset() mp.recordBlockInHistory(headerHash, headerHandler, bodyHandler) highestFinalBlockNonce := mp.forkDetector.GetHighestFinalBlockNonce() diff --git a/process/block/shardblock.go b/process/block/shardblock.go index 0e562e276e4..b0683947a09 100644 --- a/process/block/shardblock.go +++ b/process/block/shardblock.go @@ -11,6 +11,8 @@ import ( "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/headerVersionData" + logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/holders" "github.com/multiversx/mx-chain-go/dataRetriever" @@ -20,7 +22,6 @@ import ( "github.com/multiversx/mx-chain-go/process/block/helpers" "github.com/multiversx/mx-chain-go/process/block/processedMb" "github.com/multiversx/mx-chain-go/state" - logger "github.com/multiversx/mx-chain-logger-go" ) var _ process.BlockProcessor = (*shardProcessor)(nil) @@ -1041,6 +1042,7 @@ func (sp *shardProcessor) CommitBlock( sp.blockChain.SetCurrentBlockHeaderHash(headerHash) sp.indexBlockIfNeeded(bodyHandler, headerHash, headerHandler, lastBlockHeader) + sp.stateChangesCollector.Reset() sp.recordBlockInHistory(headerHash, headerHandler, bodyHandler) lastCrossNotarizedHeader, _, err := sp.blockTracker.GetLastCrossNotarizedHeader(core.MetachainShardId) diff --git a/scripts/testnet/variables.sh b/scripts/testnet/variables.sh index c5a5b013523..3812adbc5df 100644 --- a/scripts/testnet/variables.sh +++ b/scripts/testnet/variables.sh @@ -12,7 +12,7 @@ export USE_PROXY=1 # Enable the MultiversX Transaction Generator. Note that this is a private # repository (mx-chain-txgen-go). -export USE_TXGEN=0 +export USE_TXGEN=1 # Path where the testnet will be instantiated. This folder is assumed to not # exist, but it doesn't matter if it already does. It will be created if not, @@ -52,13 +52,13 @@ export GENESIS_STAKE_TYPE="direct" #'delegated' or 'direct' as in direct stake export OBSERVERS_ANTIFLOOD_DISABLE=0 # Shard structure -export SHARDCOUNT=2 -export SHARD_VALIDATORCOUNT=3 +export SHARDCOUNT=1 +export SHARD_VALIDATORCOUNT=1 export SHARD_OBSERVERCOUNT=1 -export SHARD_CONSENSUS_SIZE=3 +export SHARD_CONSENSUS_SIZE=1 # Metashard structure -export META_VALIDATORCOUNT=3 +export META_VALIDATORCOUNT=1 export META_OBSERVERCOUNT=1 export META_CONSENSUS_SIZE=$META_VALIDATORCOUNT diff --git a/state/accountsDB.go b/state/accountsDB.go index 039ee9c21d0..f2641b5af58 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -303,7 +303,8 @@ func (adb *AccountsDB) saveCodeAndDataTrie(oldAcc, newAcc vmcommon.AccountHandle baseOldAccount, _ := oldAcc.(baseAccountHandler) if !newAccOk { - return make([]*stateChange.DataTrieChange, 0), nil + //return make([]*stateChange.DataTrieChange, 0), nil + return nil, nil } newValues, err := adb.saveDataTrie(baseNewAcc) From e774eeab370416326716957361ccad4001ffdd96 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Thu, 12 Sep 2024 14:05:59 +0300 Subject: [PATCH 41/55] revert due to failing test. --- state/accountsDB.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/state/accountsDB.go b/state/accountsDB.go index f2641b5af58..039ee9c21d0 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -303,8 +303,7 @@ func (adb *AccountsDB) saveCodeAndDataTrie(oldAcc, newAcc vmcommon.AccountHandle baseOldAccount, _ := oldAcc.(baseAccountHandler) if !newAccOk { - //return make([]*stateChange.DataTrieChange, 0), nil - return nil, nil + return make([]*stateChange.DataTrieChange, 0), nil } newValues, err := adb.saveDataTrie(baseNewAcc) From 494a7b3e4cd57d65d2bfa096fdc8dc97fef30882 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Thu, 12 Sep 2024 14:40:01 +0300 Subject: [PATCH 42/55] fix failing test. --- process/block/metablock_test.go | 8 +- state/accountsDB.go | 2 +- .../state/stateChangesCollectorStub.go | 94 +++++++++++++++++++ 3 files changed, 101 insertions(+), 3 deletions(-) create mode 100644 testscommon/state/stateChangesCollectorStub.go diff --git a/process/block/metablock_test.go b/process/block/metablock_test.go index c78f2c5b039..a08301b13c2 100644 --- a/process/block/metablock_test.go +++ b/process/block/metablock_test.go @@ -13,6 +13,9 @@ import ( "github.com/multiversx/mx-chain-core-go/core/atomic" "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/dataRetriever/blockchain" @@ -36,8 +39,6 @@ import ( stateMock "github.com/multiversx/mx-chain-go/testscommon/state" statusHandlerMock "github.com/multiversx/mx-chain-go/testscommon/statusHandler" storageStubs "github.com/multiversx/mx-chain-go/testscommon/storage" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func createMockComponentHolders() ( @@ -1050,6 +1051,9 @@ func TestMetaProcessor_CommitBlockOkValsShouldWork(t *testing.T) { resetCountersForManagedBlockSignerCalled = true }, } + arguments.StateChangesCollector = &stateMock.StateChangesCollectorStub{ + ResetCalled: func() {}, + } mp, _ := blproc.NewMetaProcessor(arguments) diff --git a/state/accountsDB.go b/state/accountsDB.go index 039ee9c21d0..2ad674fbf1a 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -303,7 +303,7 @@ func (adb *AccountsDB) saveCodeAndDataTrie(oldAcc, newAcc vmcommon.AccountHandle baseOldAccount, _ := oldAcc.(baseAccountHandler) if !newAccOk { - return make([]*stateChange.DataTrieChange, 0), nil + return nil, nil } newValues, err := adb.saveDataTrie(baseNewAcc) diff --git a/testscommon/state/stateChangesCollectorStub.go b/testscommon/state/stateChangesCollectorStub.go new file mode 100644 index 00000000000..895eb2be1d3 --- /dev/null +++ b/testscommon/state/stateChangesCollectorStub.go @@ -0,0 +1,94 @@ +package state + +import ( + "github.com/multiversx/mx-chain-core-go/data/stateChange" + "github.com/multiversx/mx-chain-core-go/data/transaction" + vmcommon "github.com/multiversx/mx-chain-vm-common-go" + + "github.com/multiversx/mx-chain-go/state/stateChanges" +) + +type StateChangesCollectorStub struct { + AddStateChangeCalled func(stateChange stateChanges.StateChange) + AddSaveAccountStateChangeCalled func(oldAccount, account vmcommon.AccountHandler, stateChange stateChanges.StateChange) + ResetCalled func() + AddTxHashToCollectedStateChangesCalled func(txHash []byte, tx *transaction.Transaction) + SetIndexToLastStateChangeCalled func(index int) error + RevertToIndexCalled func(index int) error + PublishCalled func() error + IsInterfaceNilCalled func() bool + GetStateChangesForTxsCalled func() map[string]*stateChange.StateChanges +} + +// AddStateChange - +func (s *StateChangesCollectorStub) AddStateChange(stateChange stateChanges.StateChange) { + if s.AddStateChangeCalled != nil { + s.AddStateChangeCalled(stateChange) + } +} + +// AddSaveAccountStateChange - +func (s *StateChangesCollectorStub) AddSaveAccountStateChange(oldAccount, account vmcommon.AccountHandler, stateChange stateChanges.StateChange) { + if s.AddSaveAccountStateChangeCalled != nil { + s.AddSaveAccountStateChangeCalled(oldAccount, account, stateChange) + } +} + +// Reset - +func (s *StateChangesCollectorStub) Reset() { + if s.ResetCalled != nil { + s.ResetCalled() + } +} + +// AddTxHashToCollectedStateChanges - +func (s *StateChangesCollectorStub) AddTxHashToCollectedStateChanges(txHash []byte, tx *transaction.Transaction) { + if s.AddTxHashToCollectedStateChangesCalled != nil { + s.AddTxHashToCollectedStateChangesCalled(txHash, tx) + } +} + +// SetIndexToLastStateChange - +func (s *StateChangesCollectorStub) SetIndexToLastStateChange(index int) error { + if s.SetIndexToLastStateChangeCalled != nil { + return s.SetIndexToLastStateChangeCalled(index) + } + + return nil +} + +// RevertToIndex - +func (s *StateChangesCollectorStub) RevertToIndex(index int) error { + if s.RevertToIndexCalled != nil { + return s.RevertToIndexCalled(index) + } + + return nil +} + +// Publish - +func (s *StateChangesCollectorStub) Publish() error { + if s.PublishCalled != nil { + return s.PublishCalled() + } + + return nil +} + +// IsInterfaceNil - +func (s *StateChangesCollectorStub) IsInterfaceNil() bool { + if s.IsInterfaceNilCalled != nil { + return s.IsInterfaceNilCalled() + } + + return false +} + +// GetStateChangesForTxs - +func (s *StateChangesCollectorStub) GetStateChangesForTxs() map[string]*stateChange.StateChanges { + if s.GetStateChangesForTxsCalled != nil { + return s.GetStateChangesForTxsCalled() + } + + return nil +} From 4b971e203d8f669d535cd7804554ea81d8f449f6 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Thu, 12 Sep 2024 17:16:16 +0300 Subject: [PATCH 43/55] fix more tests. --- process/block/metablock_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/process/block/metablock_test.go b/process/block/metablock_test.go index a08301b13c2..124320a21ee 100644 --- a/process/block/metablock_test.go +++ b/process/block/metablock_test.go @@ -920,6 +920,9 @@ func TestMetaProcessor_CommitBlockStorageFailsForHeaderShouldNotReturnError(t *t return &block.Header{}, []byte("hash"), nil } arguments.BlockTracker = blockTrackerMock + arguments.StateChangesCollector = &stateMock.StateChangesCollectorStub{ + ResetCalled: func() {}, + } mp, _ := blproc.NewMetaProcessor(arguments) processHandler := arguments.CoreComponents.ProcessStatusHandler() From 676c5a5e4c9124254adc244d3ecf5fb33e7a1160 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Fri, 13 Sep 2024 11:14:52 +0300 Subject: [PATCH 44/55] cosmetic changes. --- state/accountsDB.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/state/accountsDB.go b/state/accountsDB.go index 2ad674fbf1a..5bb0d1a13aa 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -160,9 +160,7 @@ func checkArgsAccountsDB(args ArgsAccountsDB) error { if check.IfNil(args.SnapshotsManager) { return ErrNilSnapshotsManager } - if check.IfNil(args.StateChangesCollector) { - return ErrNilStateChangesCollector - } + if check.IfNil(args.StateChangesCollector) { return ErrNilStateChangesCollector } return nil } @@ -918,9 +916,6 @@ func (adb *AccountsDB) commit() ([]byte, error) { return nil, err } - //TODO: discuss the workflow. If reset here, the outport driver won't be able to pick up the changes. - //adb.stateChangesCollector.Reset() - oldHashes := make(common.ModifiedHashes) newHashes := make(common.ModifiedHashes) // Step 1. commit all data tries From d892b5e9fdc8077af7922c8941cb01b751bf6d75 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Fri, 13 Sep 2024 12:10:48 +0300 Subject: [PATCH 45/55] added test for new method and minor fixes. --- state/stateChanges/writeCollector.go | 4 +- state/stateChanges/writeCollector_test.go | 43 +++++++++++++++++++ .../state/stateChangesCollectorStub.go | 1 + 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/state/stateChanges/writeCollector.go b/state/stateChanges/writeCollector.go index 8cebce4dc54..0dde2c890f4 100644 --- a/state/stateChanges/writeCollector.go +++ b/state/stateChanges/writeCollector.go @@ -44,8 +44,6 @@ type stateChangesCollector struct { // NewStateChangesCollector creates a new StateChangesCollector func NewStateChangesCollector() *stateChangesCollector { - // TODO: add outport driver - return &stateChangesCollector{ stateChanges: make([]StateChange, 0), } @@ -127,7 +125,7 @@ func (scc *stateChangesCollector) GetStateChangesForTxs() map[string]*data.State }, } } else { - stateChangesForTxs[txHash].StateChanges = append(sc.StateChanges, + sc.StateChanges = append(sc.StateChanges, &data.StateChange{ Type: stateChange.GetType(), Index: stateChange.GetIndex(), diff --git a/state/stateChanges/writeCollector_test.go b/state/stateChanges/writeCollector_test.go index 6341ac724b1..6aec61b7ff4 100644 --- a/state/stateChanges/writeCollector_test.go +++ b/state/stateChanges/writeCollector_test.go @@ -251,3 +251,46 @@ func TestStateChangesCollector_Reset(t *testing.T) { assert.Equal(t, 0, len(scc.GetStateChanges())) } + +func TestStateChangesCollector_GetStateChangesForTx(t *testing.T) { + t.Parallel() + + scc := NewStateChangesCollector() + assert.Equal(t, 0, len(scc.stateChanges)) + + numStateChanges := 10 + for i := 0; i < numStateChanges; i++ { + scc.AddStateChange(&data.StateChange{ + Type: "write", + // distribute evenly based on parity of the index + TxHash: []byte(fmt.Sprintf("hash%d", i%2)), + }) + } + + stateChangesForTx := scc.GetStateChangesForTxs() + + require.Len(t, stateChangesForTx, 2) + require.Len(t, stateChangesForTx["hash0"].StateChanges, 5) + require.Len(t, stateChangesForTx["hash1"].StateChanges, 5) + + require.Equal(t, stateChangesForTx, map[string]*data.StateChanges{ + "hash0" : { + []*data.StateChange{ + {Type: "write", TxHash: []byte("hash0")}, + {Type: "write", TxHash: []byte("hash0")}, + {Type: "write", TxHash: []byte("hash0")}, + {Type: "write", TxHash: []byte("hash0")}, + {Type: "write", TxHash: []byte("hash0")}, + }, + }, + "hash1" : { + []*data.StateChange{ + {Type: "write", TxHash: []byte("hash1")}, + {Type: "write", TxHash: []byte("hash1")}, + {Type: "write", TxHash: []byte("hash1")}, + {Type: "write", TxHash: []byte("hash1")}, + {Type: "write", TxHash: []byte("hash1")}, + }, + }, + }) +} diff --git a/testscommon/state/stateChangesCollectorStub.go b/testscommon/state/stateChangesCollectorStub.go index 895eb2be1d3..b748ce2b99d 100644 --- a/testscommon/state/stateChangesCollectorStub.go +++ b/testscommon/state/stateChangesCollectorStub.go @@ -8,6 +8,7 @@ import ( "github.com/multiversx/mx-chain-go/state/stateChanges" ) +// StateChangesCollectorStub represents a mock for the StateChangesCollector interface type StateChangesCollectorStub struct { AddStateChangeCalled func(stateChange stateChanges.StateChange) AddSaveAccountStateChangeCalled func(oldAccount, account vmcommon.AccountHandler, stateChange stateChanges.StateChange) From 8bd428bbdbbf6a0bbfbc8ee7b915c2ea7acc3661 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Fri, 13 Sep 2024 12:16:20 +0300 Subject: [PATCH 46/55] update mx-chain-es-indexer --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a0bd43bc59f..9266911f467 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/multiversx/mx-chain-communication-go v1.1.0 github.com/multiversx/mx-chain-core-go v1.2.23-0.20240909105439-e1a7a06d4ac4 github.com/multiversx/mx-chain-crypto-go v1.2.12 - github.com/multiversx/mx-chain-es-indexer-go v1.4.21 + github.com/multiversx/mx-chain-es-indexer-go v1.7.5-0.20240807095116-4f2f595e52d9 github.com/multiversx/mx-chain-logger-go v1.0.15 github.com/multiversx/mx-chain-scenario-go v1.4.4 github.com/multiversx/mx-chain-storage-go v1.0.17-0.20240909105906-39ad1daf2a4c diff --git a/go.sum b/go.sum index 3b4364c1cbc..74f15a2455f 100644 --- a/go.sum +++ b/go.sum @@ -391,8 +391,8 @@ github.com/multiversx/mx-chain-core-go v1.2.23-0.20240909105439-e1a7a06d4ac4 h1: github.com/multiversx/mx-chain-core-go v1.2.23-0.20240909105439-e1a7a06d4ac4/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= github.com/multiversx/mx-chain-crypto-go v1.2.12 h1:zWip7rpUS4CGthJxfKn5MZfMfYPjVjIiCID6uX5BSOk= github.com/multiversx/mx-chain-crypto-go v1.2.12/go.mod h1:HzcPpCm1zanNct/6h2rIh+MFrlXbjA5C8+uMyXj3LI4= -github.com/multiversx/mx-chain-es-indexer-go v1.4.21 h1:rzxXCkgOsqj67GRYtqzKuf9XgHwnZLTZhU90Ck3VbrE= -github.com/multiversx/mx-chain-es-indexer-go v1.4.21/go.mod h1:V9xxOBkfV7GjN4K5SODaOetoGVpQm4snibMVPCjL0Kk= +github.com/multiversx/mx-chain-es-indexer-go v1.7.5-0.20240807095116-4f2f595e52d9 h1:VJOigTM9JbjFdy9ICVhsDfM9YQkFqMigAaQCHaM0iwY= +github.com/multiversx/mx-chain-es-indexer-go v1.7.5-0.20240807095116-4f2f595e52d9/go.mod h1:oGcRK2E3Syv6vRTszWrrb/TqD8akq0yeoMr1wPPiTO4= github.com/multiversx/mx-chain-logger-go v1.0.15 h1:HlNdK8etyJyL9NQ+6mIXyKPEBo+wRqOwi3n+m2QIHXc= github.com/multiversx/mx-chain-logger-go v1.0.15/go.mod h1:t3PRKaWB1M+i6gUfD27KXgzLJJC+mAQiN+FLlL1yoGQ= github.com/multiversx/mx-chain-scenario-go v1.4.4 h1:DVE2V+FPeyD/yWoC+KEfPK3jsFzHeruelESfpTlf460= From 6a8dc4133fac9ebc5b503813a1eed2809b33fa6c Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Fri, 13 Sep 2024 12:17:32 +0300 Subject: [PATCH 47/55] revert variables.sh --- scripts/testnet/variables.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/testnet/variables.sh b/scripts/testnet/variables.sh index 3812adbc5df..c5a5b013523 100644 --- a/scripts/testnet/variables.sh +++ b/scripts/testnet/variables.sh @@ -12,7 +12,7 @@ export USE_PROXY=1 # Enable the MultiversX Transaction Generator. Note that this is a private # repository (mx-chain-txgen-go). -export USE_TXGEN=1 +export USE_TXGEN=0 # Path where the testnet will be instantiated. This folder is assumed to not # exist, but it doesn't matter if it already does. It will be created if not, @@ -52,13 +52,13 @@ export GENESIS_STAKE_TYPE="direct" #'delegated' or 'direct' as in direct stake export OBSERVERS_ANTIFLOOD_DISABLE=0 # Shard structure -export SHARDCOUNT=1 -export SHARD_VALIDATORCOUNT=1 +export SHARDCOUNT=2 +export SHARD_VALIDATORCOUNT=3 export SHARD_OBSERVERCOUNT=1 -export SHARD_CONSENSUS_SIZE=1 +export SHARD_CONSENSUS_SIZE=3 # Metashard structure -export META_VALIDATORCOUNT=1 +export META_VALIDATORCOUNT=3 export META_OBSERVERCOUNT=1 export META_CONSENSUS_SIZE=$META_VALIDATORCOUNT From 0ffca72659f603ad05c8d2d923961f07298041ca Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Fri, 13 Sep 2024 13:46:40 +0300 Subject: [PATCH 48/55] improved readability. --- state/stateChanges/writeCollector.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/state/stateChanges/writeCollector.go b/state/stateChanges/writeCollector.go index 0dde2c890f4..87eed266252 100644 --- a/state/stateChanges/writeCollector.go +++ b/state/stateChanges/writeCollector.go @@ -103,15 +103,15 @@ func (scc *stateChangesCollector) getStateChangesForTxs() ([]StateChangesForTx, // GetStateChangesForTxs will retrieve the state changes linked with the tx hash. func (scc *stateChangesCollector) GetStateChangesForTxs() map[string]*data.StateChanges { - scc.stateChangesMut.Lock() - defer scc.stateChangesMut.Unlock() + scc.stateChangesMut.RLock() + defer scc.stateChangesMut.RUnlock() stateChangesForTxs := make(map[string]*data.StateChanges) for _, stateChange := range scc.stateChanges { txHash := string(stateChange.GetTxHash()) - if sc, ok := stateChangesForTxs[txHash]; !ok { + if _, ok := stateChangesForTxs[txHash]; !ok { stateChangesForTxs[txHash] = &data.StateChanges{StateChanges: []*data.StateChange{ { Type: stateChange.GetType(), @@ -125,7 +125,7 @@ func (scc *stateChangesCollector) GetStateChangesForTxs() map[string]*data.State }, } } else { - sc.StateChanges = append(sc.StateChanges, + stateChangesForTxs[txHash].StateChanges = append(stateChangesForTxs[txHash].StateChanges, &data.StateChange{ Type: stateChange.GetType(), Index: stateChange.GetIndex(), From 43e7cb86f8f99208487f3fa27469c267f7064bdc Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Tue, 17 Sep 2024 14:18:59 +0300 Subject: [PATCH 49/55] fixes after review. --- process/block/metablock_test.go | 8 ++------ state/accountsDB.go | 4 +++- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/process/block/metablock_test.go b/process/block/metablock_test.go index 124320a21ee..a5665e7d8cb 100644 --- a/process/block/metablock_test.go +++ b/process/block/metablock_test.go @@ -920,9 +920,7 @@ func TestMetaProcessor_CommitBlockStorageFailsForHeaderShouldNotReturnError(t *t return &block.Header{}, []byte("hash"), nil } arguments.BlockTracker = blockTrackerMock - arguments.StateChangesCollector = &stateMock.StateChangesCollectorStub{ - ResetCalled: func() {}, - } + arguments.StateChangesCollector = &stateMock.StateChangesCollectorStub{} mp, _ := blproc.NewMetaProcessor(arguments) processHandler := arguments.CoreComponents.ProcessStatusHandler() @@ -1054,9 +1052,7 @@ func TestMetaProcessor_CommitBlockOkValsShouldWork(t *testing.T) { resetCountersForManagedBlockSignerCalled = true }, } - arguments.StateChangesCollector = &stateMock.StateChangesCollectorStub{ - ResetCalled: func() {}, - } + arguments.StateChangesCollector = &stateMock.StateChangesCollectorStub{} mp, _ := blproc.NewMetaProcessor(arguments) diff --git a/state/accountsDB.go b/state/accountsDB.go index 5bb0d1a13aa..a090c4cb358 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -160,7 +160,9 @@ func checkArgsAccountsDB(args ArgsAccountsDB) error { if check.IfNil(args.SnapshotsManager) { return ErrNilSnapshotsManager } - if check.IfNil(args.StateChangesCollector) { return ErrNilStateChangesCollector } + if check.IfNil(args.StateChangesCollector) { + return ErrNilStateChangesCollector + } return nil } From 260b7b4c7a893aac654690e983d5c3a248831ebd Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Tue, 17 Sep 2024 17:18:04 +0300 Subject: [PATCH 50/55] bumped mx-chain-core-go version. --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9266911f467..9782d4fea95 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/klauspost/cpuid/v2 v2.2.5 github.com/mitchellh/mapstructure v1.5.0 github.com/multiversx/mx-chain-communication-go v1.1.0 - github.com/multiversx/mx-chain-core-go v1.2.23-0.20240909105439-e1a7a06d4ac4 + github.com/multiversx/mx-chain-core-go v1.2.23-0.20240917141620-f95a4e7bfd42 github.com/multiversx/mx-chain-crypto-go v1.2.12 github.com/multiversx/mx-chain-es-indexer-go v1.7.5-0.20240807095116-4f2f595e52d9 github.com/multiversx/mx-chain-logger-go v1.0.15 diff --git a/go.sum b/go.sum index 74f15a2455f..6693c3c494a 100644 --- a/go.sum +++ b/go.sum @@ -387,8 +387,8 @@ github.com/multiversx/concurrent-map v0.1.4 h1:hdnbM8VE4b0KYJaGY5yJS2aNIW9TFFsUY github.com/multiversx/concurrent-map v0.1.4/go.mod h1:8cWFRJDOrWHOTNSqgYCUvwT7c7eFQ4U2vKMOp4A/9+o= github.com/multiversx/mx-chain-communication-go v1.1.0 h1:J7bX6HoN3HiHY7cUeEjG8AJWgQDDPcY+OPDOsSUOkRE= github.com/multiversx/mx-chain-communication-go v1.1.0/go.mod h1:WK6bP4pGEHGDDna/AYRIMtl6G9OA0NByI1Lw8PmOnRM= -github.com/multiversx/mx-chain-core-go v1.2.23-0.20240909105439-e1a7a06d4ac4 h1:omuzhvGYFRAE0UA9UMatgSLTBlANKea0hU/jskzj6pM= -github.com/multiversx/mx-chain-core-go v1.2.23-0.20240909105439-e1a7a06d4ac4/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= +github.com/multiversx/mx-chain-core-go v1.2.23-0.20240917141620-f95a4e7bfd42 h1:mxZCpz0qF0f9LzeZVAFgmuDKpKraHBOmMlfqzPmEASs= +github.com/multiversx/mx-chain-core-go v1.2.23-0.20240917141620-f95a4e7bfd42/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= github.com/multiversx/mx-chain-crypto-go v1.2.12 h1:zWip7rpUS4CGthJxfKn5MZfMfYPjVjIiCID6uX5BSOk= github.com/multiversx/mx-chain-crypto-go v1.2.12/go.mod h1:HzcPpCm1zanNct/6h2rIh+MFrlXbjA5C8+uMyXj3LI4= github.com/multiversx/mx-chain-es-indexer-go v1.7.5-0.20240807095116-4f2f595e52d9 h1:VJOigTM9JbjFdy9ICVhsDfM9YQkFqMigAaQCHaM0iwY= From ef20419897d17e655140942b60d9342af19ed4cc Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Wed, 18 Sep 2024 13:08:47 +0300 Subject: [PATCH 51/55] bump mx-chain-core go version. --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9782d4fea95..d879c1cf45b 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/klauspost/cpuid/v2 v2.2.5 github.com/mitchellh/mapstructure v1.5.0 github.com/multiversx/mx-chain-communication-go v1.1.0 - github.com/multiversx/mx-chain-core-go v1.2.23-0.20240917141620-f95a4e7bfd42 + github.com/multiversx/mx-chain-core-go v1.2.23-0.20240918093335-b9e28fbed67c github.com/multiversx/mx-chain-crypto-go v1.2.12 github.com/multiversx/mx-chain-es-indexer-go v1.7.5-0.20240807095116-4f2f595e52d9 github.com/multiversx/mx-chain-logger-go v1.0.15 diff --git a/go.sum b/go.sum index 6693c3c494a..3a6a35eb248 100644 --- a/go.sum +++ b/go.sum @@ -387,8 +387,8 @@ github.com/multiversx/concurrent-map v0.1.4 h1:hdnbM8VE4b0KYJaGY5yJS2aNIW9TFFsUY github.com/multiversx/concurrent-map v0.1.4/go.mod h1:8cWFRJDOrWHOTNSqgYCUvwT7c7eFQ4U2vKMOp4A/9+o= github.com/multiversx/mx-chain-communication-go v1.1.0 h1:J7bX6HoN3HiHY7cUeEjG8AJWgQDDPcY+OPDOsSUOkRE= github.com/multiversx/mx-chain-communication-go v1.1.0/go.mod h1:WK6bP4pGEHGDDna/AYRIMtl6G9OA0NByI1Lw8PmOnRM= -github.com/multiversx/mx-chain-core-go v1.2.23-0.20240917141620-f95a4e7bfd42 h1:mxZCpz0qF0f9LzeZVAFgmuDKpKraHBOmMlfqzPmEASs= -github.com/multiversx/mx-chain-core-go v1.2.23-0.20240917141620-f95a4e7bfd42/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= +github.com/multiversx/mx-chain-core-go v1.2.23-0.20240918093335-b9e28fbed67c h1:wPqkaTaiSnMXXGmnqJNtn+xMZUoJEAw16QgtrlUGSgk= +github.com/multiversx/mx-chain-core-go v1.2.23-0.20240918093335-b9e28fbed67c/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= github.com/multiversx/mx-chain-crypto-go v1.2.12 h1:zWip7rpUS4CGthJxfKn5MZfMfYPjVjIiCID6uX5BSOk= github.com/multiversx/mx-chain-crypto-go v1.2.12/go.mod h1:HzcPpCm1zanNct/6h2rIh+MFrlXbjA5C8+uMyXj3LI4= github.com/multiversx/mx-chain-es-indexer-go v1.7.5-0.20240807095116-4f2f595e52d9 h1:VJOigTM9JbjFdy9ICVhsDfM9YQkFqMigAaQCHaM0iwY= From d5a81a5cb64ed1d9820f448f3978265e386aed6b Mon Sep 17 00:00:00 2001 From: ssd04 Date: Thu, 19 Sep 2024 23:13:13 +0300 Subject: [PATCH 52/55] fixes after review --- epochStart/metachain/systemSCs_test.go | 3 +- factory/state/stateComponents.go | 10 +- outport/host/errors.go | 3 - state/accountsDB.go | 11 +- state/accountsDB_test.go | 256 +++++++++--------- .../disabled/disabledStateChangesCollector.go | 5 +- state/errors.go | 3 + state/export_test.go | 10 + state/interface.go | 19 +- state/stateChanges/dataAnalysisCollector.go | 36 +-- .../dataAnalysisCollector_test.go | 2 + state/stateChanges/writeCollector.go | 70 ++--- state/stateChanges/writeCollector_test.go | 13 +- .../state/stateChangesCollectorStub.go | 10 +- 14 files changed, 233 insertions(+), 218 deletions(-) diff --git a/epochStart/metachain/systemSCs_test.go b/epochStart/metachain/systemSCs_test.go index d526e26608f..8d410aeb017 100644 --- a/epochStart/metachain/systemSCs_test.go +++ b/epochStart/metachain/systemSCs_test.go @@ -38,7 +38,6 @@ import ( "github.com/multiversx/mx-chain-go/state" disabledState "github.com/multiversx/mx-chain-go/state/disabled" "github.com/multiversx/mx-chain-go/state/factory" - "github.com/multiversx/mx-chain-go/state/stateChanges" "github.com/multiversx/mx-chain-go/state/storagePruningManager" "github.com/multiversx/mx-chain-go/state/storagePruningManager/evictionWaitingList" "github.com/multiversx/mx-chain-go/storage" @@ -783,7 +782,7 @@ func createFullArgumentsForSystemSCProcessing(enableEpochsConfig config.EnableEp Hasher: hasher, Marshaller: marshalizer, EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, - StateChangesCollector: stateChanges.NewStateChangesCollector(), + StateChangesCollector: disabledState.NewDisabledStateChangesCollector(), } accCreator, _ := factory.NewAccountCreator(argsAccCreator) peerAccCreator := factory.NewPeerAccountCreator() diff --git a/factory/state/stateComponents.go b/factory/state/stateComponents.go index e9ce1270e49..dce52d14547 100644 --- a/factory/state/stateComponents.go +++ b/factory/state/stateComponents.go @@ -272,11 +272,6 @@ func (scf *stateComponentsFactory) createPeerAdapter(triesContainer common.Tries return nil, err } - stateChangesCollector := disabled.NewDisabledStateChangesCollector() - // if scf.config.StateTriesConfig.CollectStateChangesEnabled { - // stateChangesCollector = state.NewStateChangesCollector() - // } - argStateMetrics := stateMetrics.ArgsStateMetrics{ SnapshotInProgressKey: common.MetricPeersSnapshotInProgress, LastSnapshotDurationKey: common.MetricLastPeersSnapshotDurationSec, @@ -292,6 +287,11 @@ func (scf *stateComponentsFactory) createPeerAdapter(triesContainer common.Tries return nil, err } + stateChangesCollector, err := scf.createStateChangesCollector() + if err != nil { + return nil, err + } + argsProcessingPeerAccountsDB := state.ArgsAccountsDB{ Trie: merkleTrie, Hasher: scf.core.Hasher(), diff --git a/outport/host/errors.go b/outport/host/errors.go index 8b092a1a102..de45f08acef 100644 --- a/outport/host/errors.go +++ b/outport/host/errors.go @@ -7,6 +7,3 @@ var ErrHostIsClosed = errors.New("server is closed") // ErrNilHost signals that a nil host has been provided var ErrNilHost = errors.New("nil host provided") - -// ErrNilStateChangesCollector signals that a nil state change collector has been provided. -var ErrNilStateChangesCollector = errors.New("nil state changes collector provided") diff --git a/state/accountsDB.go b/state/accountsDB.go index a090c4cb358..52cb1b87b51 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -719,6 +719,15 @@ func (adb *AccountsDB) getAccount(address []byte, mainTrie common.Trie) (vmcommo return nil, nil } + stateChange := &stateChange.StateChange{ + Type: "read", + MainTrieKey: address, + MainTrieVal: val, + Operation: "getAccount", + DataTrieChanges: nil, + } + adb.stateChangesCollector.AddStateChange(stateChange) + acnt, err := adb.accountFactory.CreateAccount(address) if err != nil { return nil, err @@ -912,9 +921,9 @@ func (adb *AccountsDB) commit() ([]byte, error) { log.Trace("accountsDB.Commit started") adb.entries = make([]JournalEntry, 0) + // TODO: evaluate moving this to procesing on CommitBlock err := adb.stateChangesCollector.Publish() if err != nil { - log.Warn("failed to dump state changes to json file", "error", err) return nil, err } diff --git a/state/accountsDB_test.go b/state/accountsDB_test.go index 085d1bec004..97cfdf5724e 100644 --- a/state/accountsDB_test.go +++ b/state/accountsDB_test.go @@ -17,6 +17,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/core/keyValStorage" data "github.com/multiversx/mx-chain-core-go/data/stateChange" + "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-core-go/marshal" vmcommon "github.com/multiversx/mx-chain-vm-common-go" "github.com/multiversx/mx-chain-vm-common-go/dataTrieMigrator" @@ -31,6 +32,8 @@ import ( "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/state/accounts" + "github.com/multiversx/mx-chain-go/state/dataTrieValue" + stateDisabled "github.com/multiversx/mx-chain-go/state/disabled" "github.com/multiversx/mx-chain-go/state/factory" "github.com/multiversx/mx-chain-go/state/iteratorChannelsProvider" "github.com/multiversx/mx-chain-go/state/lastSnapshotMarker" @@ -154,7 +157,7 @@ func getDefaultStateComponents( Hasher: hasher, Marshaller: marshaller, EnableEpochsHandler: enableEpochsHandler, - StateChangesCollector: stateChanges.NewStateChangesCollector(), + StateChangesCollector: stateDisabled.NewDisabledStateChangesCollector(), } accCreator, _ := factory.NewAccountCreator(argsAccCreator) @@ -426,127 +429,134 @@ func TestAccountsDB_SaveAccountMalfunctionMarshallerShouldErr(t *testing.T) { assert.NotNil(t, err) } -// func TestAccountsDB_SaveAccountCollectsAllStateChanges(t *testing.T) { -// t.Parallel() - -// autoBalanceFlagEnabled := false -// enableEpochs := &enableEpochsHandlerMock.EnableEpochsHandlerStub{ -// IsFlagEnabledCalled: func(flag core.EnableEpochFlag) bool { -// return autoBalanceFlagEnabled -// }, -// } -// _, adb := getDefaultStateComponentsWithCustomEnableEpochs(enableEpochs) -// address := generateRandomByteArray(32) - -// stepCreateAccountWithDataTrieAndCode(t, adb, address) -// autoBalanceFlagEnabled = true -// stepMigrateDataTrieValAndChangeCode(t, adb, address) -// } - -// func stepCreateAccountWithDataTrieAndCode( -// t *testing.T, -// adb *state.AccountsDB, -// address []byte, -// ) { -// marshaller := &marshallerMock.MarshalizerMock{} - -// acc, _ := adb.LoadAccount(address) -// userAcc := acc.(state.UserAccountHandler) -// code := []byte("smart contract code") -// key1 := []byte("key1") -// key2 := []byte("key2") -// userAcc.SetCode(code) -// _ = userAcc.SaveKeyValue(key1, []byte("value")) -// _ = userAcc.SaveKeyValue(key2, []byte("value")) -// _ = adb.SaveAccount(userAcc) -// adb.SetTxHashForLatestStateChanges([]byte("accountCreationTxHash"), &transaction.Transaction{}) -// serializedAcc, _ := marshaller.Marshal(userAcc) -// codeHash := userAcc.GetCodeHash() - -// stateChangesForTx := adb.ResetStateChangesCollector() -// assert.Equal(t, 1, len(stateChangesForTx)) - -// stateChanges := stateChangesForTx[0].StateChanges -// assert.Equal(t, 2, len(stateChanges)) -// assert.Equal(t, []byte("accountCreationTxHash"), stateChangesForTx[0].TxHash) - -// codeStateChange := stateChanges[0] -// assert.Equal(t, codeHash, codeStateChange.MainTrieKey) -// codeEntry := &state.CodeEntry{ -// Code: code, -// NumReferences: 1, -// } -// serializedCodeEntry, _ := marshaller.Marshal(codeEntry) -// assert.Equal(t, serializedCodeEntry, codeStateChange.MainTrieVal) -// assert.Equal(t, 0, len(codeStateChange.DataTrieChanges)) - -// accountStateChange := stateChanges[1] -// assert.Equal(t, address, accountStateChange.MainTrieKey) -// assert.Equal(t, serializedAcc, accountStateChange.MainTrieVal) -// assert.Equal(t, 2, len(accountStateChange.DataTrieChanges)) -// assert.Equal(t, key1, accountStateChange.DataTrieChanges[0].Key) -// valWithMetadata1 := append([]byte("value"), key1...) -// valWithMetadata1 = append(valWithMetadata1, address...) -// assert.Equal(t, valWithMetadata1, accountStateChange.DataTrieChanges[0].Val) -// valWithMetadata2 := append([]byte("value"), key2...) -// valWithMetadata2 = append(valWithMetadata2, address...) -// assert.Equal(t, valWithMetadata2, accountStateChange.DataTrieChanges[1].Val) -// } - -// func stepMigrateDataTrieValAndChangeCode( -// t *testing.T, -// adb *state.AccountsDB, -// address []byte, -// ) { -// marshaller := &marshallerMock.MarshalizerMock{} -// hasher := &hashingMocks.HasherMock{} - -// acc, _ := adb.LoadAccount(address) -// userAcc := acc.(state.UserAccountHandler) -// oldCodeHash := userAcc.GetCodeHash() -// code := []byte("new smart contract code") -// userAcc.SetCode(code) -// _ = userAcc.SaveKeyValue([]byte("key1"), []byte("value1")) -// _ = adb.SaveAccount(userAcc) -// adb.SetTxHashForLatestStateChanges([]byte("accountUpdateTxHash"), &transaction.Transaction{}) - -// stateChangesForTx := adb.ResetStateChangesCollector() -// assert.Equal(t, 1, len(stateChangesForTx)) -// assert.Equal(t, 3, len(stateChangesForTx[0].StateChanges)) -// assert.Equal(t, []byte("accountUpdateTxHash"), stateChangesForTx[0].TxHash) - -// stateChanges := stateChangesForTx[0].StateChanges -// oldCodeEntryChange := stateChanges[0] -// assert.Equal(t, oldCodeHash, oldCodeEntryChange.MainTrieKey) -// assert.Equal(t, []byte(nil), oldCodeEntryChange.MainTrieVal) -// assert.Equal(t, 0, len(oldCodeEntryChange.DataTrieChanges)) - -// newCodeEntryChange := stateChanges[1] -// codeEntry := &state.CodeEntry{ -// Code: code, -// NumReferences: 1, -// } -// serializedCodeEntry, _ := marshaller.Marshal(codeEntry) -// assert.Equal(t, userAcc.GetCodeHash(), newCodeEntryChange.MainTrieKey) -// assert.Equal(t, serializedCodeEntry, newCodeEntryChange.MainTrieVal) -// assert.Equal(t, 0, len(newCodeEntryChange.DataTrieChanges)) - -// accountStateChange := stateChanges[2] -// serializedAcc, _ := marshaller.Marshal(userAcc) -// assert.Equal(t, address, accountStateChange.MainTrieKey) -// assert.Equal(t, serializedAcc, accountStateChange.MainTrieVal) -// assert.Equal(t, 2, len(accountStateChange.DataTrieChanges)) -// trieVal := &dataTrieValue.TrieLeafData{ -// Value: []byte("value1"), -// Key: []byte("key1"), -// Address: address, -// } -// serializedTrieVal, _ := marshaller.Marshal(trieVal) -// assert.Equal(t, hasher.Compute("key1"), accountStateChange.DataTrieChanges[0].Key) -// assert.Equal(t, serializedTrieVal, accountStateChange.DataTrieChanges[0].Val) -// assert.Equal(t, []byte("key1"), accountStateChange.DataTrieChanges[1].Key) -// assert.Equal(t, []byte(nil), accountStateChange.DataTrieChanges[1].Val) -// } +func TestAccountsDB_SaveAccountCollectsAllStateChanges(t *testing.T) { + t.Parallel() + + autoBalanceFlagEnabled := false + enableEpochs := &enableEpochsHandlerMock.EnableEpochsHandlerStub{ + IsFlagEnabledCalled: func(flag core.EnableEpochFlag) bool { + return autoBalanceFlagEnabled + }, + } + + _, adb := getDefaultStateComponentsWithCustomEnableEpochs(enableEpochs) + address := generateRandomByteArray(32) + + stepCreateAccountWithDataTrieAndCode(t, adb, address) + autoBalanceFlagEnabled = true + stepMigrateDataTrieValAndChangeCode(t, adb, address) +} + +func stepCreateAccountWithDataTrieAndCode( + t *testing.T, + adb *state.AccountsDB, + address []byte, +) { + marshaller := &marshallerMock.MarshalizerMock{} + + acc, _ := adb.LoadAccount(address) + userAcc := acc.(state.UserAccountHandler) + code := []byte("smart contract code") + key1 := []byte("key1") + key2 := []byte("key2") + userAcc.SetCode(code) + _ = userAcc.SaveKeyValue(key1, []byte("value")) + _ = userAcc.SaveKeyValue(key2, []byte("value")) + _ = adb.SaveAccount(userAcc) + + txHash := []byte("accountCreationTxHash") + + adb.SetTxHashForLatestStateChanges(txHash, &transaction.Transaction{}) + serializedAcc, _ := marshaller.Marshal(userAcc) + codeHash := userAcc.GetCodeHash() + + stateChangesForTx := adb.ResetStateChangesCollector() + assert.Equal(t, 1, len(stateChangesForTx)) + + stateChanges := stateChangesForTx[string(txHash)].StateChanges + assert.Equal(t, 2, len(stateChanges)) + assert.Equal(t, txHash, stateChangesForTx[string(txHash)].StateChanges[0].GetTxHash()) + + codeStateChange := stateChanges[0] + assert.Equal(t, codeHash, codeStateChange.MainTrieKey) + codeEntry := &state.CodeEntry{ + Code: code, + NumReferences: 1, + } + serializedCodeEntry, _ := marshaller.Marshal(codeEntry) + assert.Equal(t, serializedCodeEntry, codeStateChange.MainTrieVal) + assert.Equal(t, 0, len(codeStateChange.DataTrieChanges)) + + accountStateChange := stateChanges[1] + assert.Equal(t, address, accountStateChange.MainTrieKey) + assert.Equal(t, serializedAcc, accountStateChange.MainTrieVal) + assert.Equal(t, 2, len(accountStateChange.DataTrieChanges)) + assert.Equal(t, key1, accountStateChange.DataTrieChanges[0].Key) + valWithMetadata1 := append([]byte("value"), key1...) + valWithMetadata1 = append(valWithMetadata1, address...) + assert.Equal(t, valWithMetadata1, accountStateChange.DataTrieChanges[0].Val) + valWithMetadata2 := append([]byte("value"), key2...) + valWithMetadata2 = append(valWithMetadata2, address...) + assert.Equal(t, valWithMetadata2, accountStateChange.DataTrieChanges[1].Val) +} + +func stepMigrateDataTrieValAndChangeCode( + t *testing.T, + adb *state.AccountsDB, + address []byte, +) { + marshaller := &marshallerMock.MarshalizerMock{} + hasher := &hashingMocks.HasherMock{} + + acc, _ := adb.LoadAccount(address) + userAcc := acc.(state.UserAccountHandler) + oldCodeHash := userAcc.GetCodeHash() + code := []byte("new smart contract code") + userAcc.SetCode(code) + _ = userAcc.SaveKeyValue([]byte("key1"), []byte("value1")) + _ = adb.SaveAccount(userAcc) + + txHash := []byte("accountCreationTxHash") + + adb.SetTxHashForLatestStateChanges(txHash, &transaction.Transaction{}) + + stateChangesForTx := adb.ResetStateChangesCollector() + assert.Equal(t, 1, len(stateChangesForTx)) + assert.Equal(t, 3, len(stateChangesForTx[string(txHash)].StateChanges)) + assert.Equal(t, txHash, stateChangesForTx[string(txHash)].StateChanges[0].GetTxHash()) + + stateChanges := stateChangesForTx[string(txHash)].StateChanges + oldCodeEntryChange := stateChanges[0] + assert.Equal(t, oldCodeHash, oldCodeEntryChange.MainTrieKey) + assert.Equal(t, []byte(nil), oldCodeEntryChange.MainTrieVal) + assert.Equal(t, 0, len(oldCodeEntryChange.DataTrieChanges)) + + newCodeEntryChange := stateChanges[1] + codeEntry := &state.CodeEntry{ + Code: code, + NumReferences: 1, + } + serializedCodeEntry, _ := marshaller.Marshal(codeEntry) + assert.Equal(t, userAcc.GetCodeHash(), newCodeEntryChange.MainTrieKey) + assert.Equal(t, serializedCodeEntry, newCodeEntryChange.MainTrieVal) + assert.Equal(t, 0, len(newCodeEntryChange.DataTrieChanges)) + + accountStateChange := stateChanges[2] + serializedAcc, _ := marshaller.Marshal(userAcc) + assert.Equal(t, address, accountStateChange.MainTrieKey) + assert.Equal(t, serializedAcc, accountStateChange.MainTrieVal) + assert.Equal(t, 2, len(accountStateChange.DataTrieChanges)) + trieVal := &dataTrieValue.TrieLeafData{ + Value: []byte("value1"), + Key: []byte("key1"), + Address: address, + } + serializedTrieVal, _ := marshaller.Marshal(trieVal) + assert.Equal(t, hasher.Compute("key1"), accountStateChange.DataTrieChanges[0].Key) + assert.Equal(t, serializedTrieVal, accountStateChange.DataTrieChanges[0].Val) + assert.Equal(t, []byte("key1"), accountStateChange.DataTrieChanges[1].Key) + assert.Equal(t, []byte(nil), accountStateChange.DataTrieChanges[1].Val) +} func TestAccountsDB_SaveAccountWithSomeValuesShouldWork(t *testing.T) { t.Parallel() @@ -2937,6 +2947,8 @@ func TestAccountsDb_Concurrent(t *testing.T) { numAccounts := 100000 accountsAddresses := generateAccounts(t, numAccounts, adb) + + adb.SetTxHashForLatestStateChanges([]byte("txHash"), &transaction.Transaction{}) rootHash, _ := adb.Commit() testAccountMethodsConcurrency(t, adb, accountsAddresses, rootHash) diff --git a/state/disabled/disabledStateChangesCollector.go b/state/disabled/disabledStateChangesCollector.go index 629deeb293a..297d89ce5dd 100644 --- a/state/disabled/disabledStateChangesCollector.go +++ b/state/disabled/disabledStateChangesCollector.go @@ -6,7 +6,6 @@ import ( vmcommon "github.com/multiversx/mx-chain-vm-common-go" "github.com/multiversx/mx-chain-go/state" - "github.com/multiversx/mx-chain-go/state/stateChanges" ) // disabledStateChangesCollector is a state changes collector that does nothing @@ -19,11 +18,11 @@ func NewDisabledStateChangesCollector() state.StateChangesCollector { } // AddSaveAccountStateChange - -func (d *disabledStateChangesCollector) AddSaveAccountStateChange(oldAccount, account vmcommon.AccountHandler, stateChange stateChanges.StateChange) { +func (d *disabledStateChangesCollector) AddSaveAccountStateChange(oldAccount, account vmcommon.AccountHandler, stateChange state.StateChange) { } // AddStateChange does nothing -func (d *disabledStateChangesCollector) AddStateChange(_ stateChanges.StateChange) { +func (d *disabledStateChangesCollector) AddStateChange(_ state.StateChange) { } // Reset does nothing diff --git a/state/errors.go b/state/errors.go index 8da20ed523d..c057562b4a6 100644 --- a/state/errors.go +++ b/state/errors.go @@ -168,3 +168,6 @@ var ErrValidatorNotFound = errors.New("validator not found") // ErrNilStateChangesCollector signals that a nil state changes collector has been given var ErrNilStateChangesCollector = errors.New("nil state changes collector") + +// ErrStateChangesIndexOutOfBounds signals that the state changes index is out of bounds +var ErrStateChangesIndexOutOfBounds = errors.New("state changes index out of bounds") diff --git a/state/export_test.go b/state/export_test.go index 4398d616dd3..8353abd92d4 100644 --- a/state/export_test.go +++ b/state/export_test.go @@ -1,6 +1,7 @@ package state import ( + data "github.com/multiversx/mx-chain-core-go/data/stateChange" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" vmcommon "github.com/multiversx/mx-chain-vm-common-go" @@ -26,6 +27,15 @@ func (adb *AccountsDB) GetObsoleteHashes() map[string][][]byte { return adb.obsoleteDataTrieHashes } +// ResetStateChangesCollector - +func (adb *AccountsDB) ResetStateChangesCollector() map[string]*data.StateChanges { + stateChanges := adb.stateChangesCollector.GetStateChangesForTxs() + + adb.stateChangesCollector.Reset() + + return stateChanges +} + // GetCode - func GetCode(account baseAccountHandler) []byte { return account.GetCodeHash() diff --git a/state/interface.go b/state/interface.go index 49be503dcc5..73ac6c9d50a 100644 --- a/state/interface.go +++ b/state/interface.go @@ -11,7 +11,6 @@ import ( vmcommon "github.com/multiversx/mx-chain-vm-common-go" "github.com/multiversx/mx-chain-go/common" - "github.com/multiversx/mx-chain-go/state/stateChanges" ) // AccountFactory creates an account of different types @@ -360,8 +359,8 @@ type ValidatorInfoHandler interface { // StateChangesCollector defines the methods needed for an StateChangesCollector implementation type StateChangesCollector interface { - AddStateChange(stateChange stateChanges.StateChange) - AddSaveAccountStateChange(oldAccount, account vmcommon.AccountHandler, stateChange stateChanges.StateChange) + AddStateChange(stateChange StateChange) + AddSaveAccountStateChange(oldAccount, account vmcommon.AccountHandler, stateChange StateChange) Reset() AddTxHashToCollectedStateChanges(txHash []byte, tx *transaction.Transaction) SetIndexToLastStateChange(index int) error @@ -370,3 +369,17 @@ type StateChangesCollector interface { IsInterfaceNil() bool GetStateChangesForTxs() map[string]*data.StateChanges } + +// StateChange defines the behaviour of a state change holder +type StateChange interface { + GetType() string + GetIndex() int32 + GetTxHash() []byte + GetMainTrieKey() []byte + GetMainTrieVal() []byte + GetOperation() string + GetDataTrieChanges() []*data.DataTrieChange + + SetTxHash(txHash []byte) + SetIndex(index int32) +} diff --git a/state/stateChanges/dataAnalysisCollector.go b/state/stateChanges/dataAnalysisCollector.go index 0fe627b799f..52aae5ab0cc 100644 --- a/state/stateChanges/dataAnalysisCollector.go +++ b/state/stateChanges/dataAnalysisCollector.go @@ -10,12 +10,13 @@ import ( "github.com/multiversx/mx-chain-core-go/data/transaction" vmcommon "github.com/multiversx/mx-chain-vm-common-go" + "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/storage" ) // TODO: use proto stucts -type DataAnalysisStateChangeDTO struct { - StateChange +type dataAnalysisStateChangeDTO struct { + state.StateChange Operation string `json:"operation"` Nonce bool `json:"nonceChanged"` Balance bool `json:"balanceChanged"` @@ -27,7 +28,7 @@ type DataAnalysisStateChangeDTO struct { CodeMetadata bool `json:"codeMetadataChanged"` } -type DataAnalysisStateChangesForTx struct { +type dataAnalysisStateChangesForTx struct { StateChangesForTx Tx *transaction.Transaction `json:"tx"` } @@ -57,17 +58,15 @@ func NewDataAnalysisStateChangesCollector(storer storage.Persister) (*dataAnalys } return &dataAnalysisCollector{ - stateChangesCollector: &stateChangesCollector{ - stateChanges: make([]StateChange, 0), - }, - cachedTxs: make(map[string]*transaction.Transaction), - storer: storer, + stateChangesCollector: NewStateChangesCollector(), + cachedTxs: make(map[string]*transaction.Transaction), + storer: storer, }, nil } // AddSaveAccountStateChange adds a new state change for the save account operation -func (scc *dataAnalysisCollector) AddSaveAccountStateChange(oldAccount, account vmcommon.AccountHandler, stateChange StateChange) { - dataAnalysisStateChange := &DataAnalysisStateChangeDTO{ +func (scc *dataAnalysisCollector) AddSaveAccountStateChange(oldAccount, account vmcommon.AccountHandler, stateChange state.StateChange) { + dataAnalysisStateChange := &dataAnalysisStateChangeDTO{ StateChange: stateChange, } @@ -76,7 +75,7 @@ func (scc *dataAnalysisCollector) AddSaveAccountStateChange(oldAccount, account scc.AddStateChange(stateChange) } -func checkAccountChanges(oldAcc, newAcc vmcommon.AccountHandler, stateChange *DataAnalysisStateChangeDTO) { +func checkAccountChanges(oldAcc, newAcc vmcommon.AccountHandler, stateChange *dataAnalysisStateChangeDTO) { baseNewAcc, newAccOk := newAcc.(userAccountHandler) if !newAccOk { return @@ -120,14 +119,14 @@ func checkAccountChanges(oldAcc, newAcc vmcommon.AccountHandler, stateChange *Da } // AddStateChange adds a new state change to the collector -func (scc *dataAnalysisCollector) AddStateChange(stateChange StateChange) { +func (scc *dataAnalysisCollector) AddStateChange(stateChange state.StateChange) { scc.stateChangesMut.Lock() defer scc.stateChangesMut.Unlock() scc.stateChanges = append(scc.stateChanges, stateChange) } -func (scc *dataAnalysisCollector) getDataAnalysisStateChangesForTxs() ([]DataAnalysisStateChangesForTx, error) { +func (scc *dataAnalysisCollector) getDataAnalysisStateChangesForTxs() ([]dataAnalysisStateChangesForTx, error) { scc.stateChangesMut.Lock() defer scc.stateChangesMut.Unlock() @@ -136,17 +135,17 @@ func (scc *dataAnalysisCollector) getDataAnalysisStateChangesForTxs() ([]DataAna return nil, err } - dataAnalysisStateChangesForTxs := make([]DataAnalysisStateChangesForTx, len(stateChangesForTxs)) + dataAnalysisStateChangesForTxs := make([]dataAnalysisStateChangesForTx, len(stateChangesForTxs)) for _, stateChangeForTx := range stateChangesForTxs { txHash := string(stateChangeForTx.TxHash) - cachedTx, txOk := scc.cachedTxs[string(txHash)] + cachedTx, txOk := scc.cachedTxs[txHash] if !txOk { return nil, fmt.Errorf("did not find tx in cache") } - stateChangesForTx := DataAnalysisStateChangesForTx{ + stateChangesForTx := dataAnalysisStateChangesForTx{ StateChangesForTx: stateChangeForTx, Tx: cachedTx, } @@ -156,6 +155,11 @@ func (scc *dataAnalysisCollector) getDataAnalysisStateChangesForTxs() ([]DataAna return dataAnalysisStateChangesForTxs, nil } +func (scc *dataAnalysisCollector) AddTxHashToCollectedStateChanges(txHash []byte, tx *transaction.Transaction) { + scc.cachedTxs[string(txHash)] = tx + scc.addTxHashToCollectedStateChanges(txHash) +} + // Reset resets the state changes collector func (scc *dataAnalysisCollector) Reset() { scc.stateChangesMut.Lock() diff --git a/state/stateChanges/dataAnalysisCollector_test.go b/state/stateChanges/dataAnalysisCollector_test.go index e45e002a9c1..82e25cad9d4 100644 --- a/state/stateChanges/dataAnalysisCollector_test.go +++ b/state/stateChanges/dataAnalysisCollector_test.go @@ -1 +1,3 @@ package stateChanges + +// TODO: add unit tests diff --git a/state/stateChanges/writeCollector.go b/state/stateChanges/writeCollector.go index 87eed266252..80e6f65069e 100644 --- a/state/stateChanges/writeCollector.go +++ b/state/stateChanges/writeCollector.go @@ -2,60 +2,42 @@ package stateChanges import ( "bytes" - "encoding/hex" - "errors" - "fmt" "sync" data "github.com/multiversx/mx-chain-core-go/data/stateChange" "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/multiversx/mx-chain-go/state" logger "github.com/multiversx/mx-chain-logger-go" vmcommon "github.com/multiversx/mx-chain-vm-common-go" ) var log = logger.GetOrCreate("state/stateChanges") -// ErrStateChangesIndexOutOfBounds signals that the state changes index is out of bounds -var ErrStateChangesIndexOutOfBounds = errors.New("state changes index out of bounds") - -type StateChange interface { - GetType() string - GetIndex() int32 - GetTxHash() []byte - GetMainTrieKey() []byte - GetMainTrieVal() []byte - GetOperation() string - GetDataTrieChanges() []*data.DataTrieChange - - SetTxHash(txHash []byte) - SetIndex(index int32) -} - // StateChangesForTx is used to collect state changes for a transaction hash type StateChangesForTx struct { - TxHash []byte `json:"txHash"` - StateChanges []StateChange `json:"stateChanges"` + TxHash []byte `json:"txHash"` + StateChanges []state.StateChange `json:"stateChanges"` } type stateChangesCollector struct { - stateChanges []StateChange + stateChanges []state.StateChange stateChangesMut sync.RWMutex } // NewStateChangesCollector creates a new StateChangesCollector func NewStateChangesCollector() *stateChangesCollector { return &stateChangesCollector{ - stateChanges: make([]StateChange, 0), + stateChanges: make([]state.StateChange, 0), } } // AddSaveAccountStateChange adds a new state change for the save account operation -func (scc *stateChangesCollector) AddSaveAccountStateChange(_, _ vmcommon.AccountHandler, stateChange StateChange) { +func (scc *stateChangesCollector) AddSaveAccountStateChange(_, _ vmcommon.AccountHandler, stateChange state.StateChange) { scc.AddStateChange(stateChange) } // AddStateChange adds a new state change to the collector -func (scc *stateChangesCollector) AddStateChange(stateChange StateChange) { +func (scc *stateChangesCollector) AddStateChange(stateChange state.StateChange) { scc.stateChangesMut.Lock() defer scc.stateChangesMut.Unlock() @@ -76,10 +58,10 @@ func (scc *stateChangesCollector) getStateChangesForTxs() ([]StateChangesForTx, if len(txHash) == 0 { log.Warn("empty tx hash, state change event not associated to a transaction") - break + continue } - innerStateChangesForTx := make([]StateChange, 0) + innerStateChangesForTx := make([]state.StateChange, 0) for j := i; j < len(scc.stateChanges); j++ { txHash2 := scc.stateChanges[j].GetTxHash() if !bytes.Equal(txHash, txHash2) { @@ -151,12 +133,16 @@ func (scc *stateChangesCollector) Reset() { } func (scc *stateChangesCollector) resetStateChangesUnprotected() { - scc.stateChanges = make([]StateChange, 0) + scc.stateChanges = make([]state.StateChange, 0) } // AddTxHashToCollectedStateChanges will try to set txHash field to each state change // if the field is not already set -func (scc *stateChangesCollector) AddTxHashToCollectedStateChanges(txHash []byte, tx *transaction.Transaction) { +func (scc *stateChangesCollector) AddTxHashToCollectedStateChanges(txHash []byte, _ *transaction.Transaction) { + scc.addTxHashToCollectedStateChanges(txHash) +} + +func (scc *stateChangesCollector) addTxHashToCollectedStateChanges(txHash []byte) { scc.stateChangesMut.Lock() defer scc.stateChangesMut.Unlock() @@ -175,7 +161,7 @@ func (scc *stateChangesCollector) SetIndexToLastStateChange(index int) error { defer scc.stateChangesMut.Unlock() if index > len(scc.stateChanges) || index < 0 { - return ErrStateChangesIndexOutOfBounds + return state.ErrStateChangesIndexOutOfBounds } if len(scc.stateChanges) == 0 { @@ -190,7 +176,7 @@ func (scc *stateChangesCollector) SetIndexToLastStateChange(index int) error { // RevertToIndex will revert to index func (scc *stateChangesCollector) RevertToIndex(index int) error { if index > len(scc.stateChanges) || index < 0 { - return ErrStateChangesIndexOutOfBounds + return state.ErrStateChangesIndexOutOfBounds } if index == 0 { @@ -211,31 +197,11 @@ func (scc *stateChangesCollector) RevertToIndex(index int) error { return nil } -// Publish will export state changes +// Publish returns nil func (scc *stateChangesCollector) Publish() error { - stateChangesForTx, err := scc.getStateChangesForTxs() - if err != nil { - return err - } - - printStateChanges(stateChangesForTx) - return nil } -func printStateChanges(stateChanges []StateChangesForTx) { - for _, stateChange := range stateChanges { - - if stateChange.TxHash != nil { - fmt.Println(hex.EncodeToString(stateChange.TxHash)) - } - - for _, st := range stateChange.StateChanges { - fmt.Println(st) - } - } -} - // IsInterfaceNil returns true if there is no value under the interface func (scc *stateChangesCollector) IsInterfaceNil() bool { return scc == nil diff --git a/state/stateChanges/writeCollector_test.go b/state/stateChanges/writeCollector_test.go index 6aec61b7ff4..4414ca2f911 100644 --- a/state/stateChanges/writeCollector_test.go +++ b/state/stateChanges/writeCollector_test.go @@ -7,6 +7,7 @@ import ( data "github.com/multiversx/mx-chain-core-go/data/stateChange" "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/multiversx/mx-chain-go/state" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -139,10 +140,10 @@ func TestStateChangesCollector_RevertToIndex_FailIfWrongIndex(t *testing.T) { numStateChanges := len(scc.stateChanges) err := scc.RevertToIndex(-1) - require.Equal(t, ErrStateChangesIndexOutOfBounds, err) + require.Equal(t, state.ErrStateChangesIndexOutOfBounds, err) err = scc.RevertToIndex(numStateChanges + 1) - require.Equal(t, ErrStateChangesIndexOutOfBounds, err) + require.Equal(t, state.ErrStateChangesIndexOutOfBounds, err) } func TestStateChangesCollector_RevertToIndex(t *testing.T) { @@ -197,11 +198,11 @@ func TestStateChangesCollector_SetIndexToLastStateChange(t *testing.T) { scc := NewStateChangesCollector() err := scc.SetIndexToLastStateChange(-1) - require.Equal(t, ErrStateChangesIndexOutOfBounds, err) + require.Equal(t, state.ErrStateChangesIndexOutOfBounds, err) numStateChanges := len(scc.stateChanges) err = scc.SetIndexToLastStateChange(numStateChanges + 1) - require.Equal(t, ErrStateChangesIndexOutOfBounds, err) + require.Equal(t, state.ErrStateChangesIndexOutOfBounds, err) }) t.Run("should work", func(t *testing.T) { @@ -274,7 +275,7 @@ func TestStateChangesCollector_GetStateChangesForTx(t *testing.T) { require.Len(t, stateChangesForTx["hash1"].StateChanges, 5) require.Equal(t, stateChangesForTx, map[string]*data.StateChanges{ - "hash0" : { + "hash0": { []*data.StateChange{ {Type: "write", TxHash: []byte("hash0")}, {Type: "write", TxHash: []byte("hash0")}, @@ -283,7 +284,7 @@ func TestStateChangesCollector_GetStateChangesForTx(t *testing.T) { {Type: "write", TxHash: []byte("hash0")}, }, }, - "hash1" : { + "hash1": { []*data.StateChange{ {Type: "write", TxHash: []byte("hash1")}, {Type: "write", TxHash: []byte("hash1")}, diff --git a/testscommon/state/stateChangesCollectorStub.go b/testscommon/state/stateChangesCollectorStub.go index b748ce2b99d..c8fc099879f 100644 --- a/testscommon/state/stateChangesCollectorStub.go +++ b/testscommon/state/stateChangesCollectorStub.go @@ -5,13 +5,13 @@ import ( "github.com/multiversx/mx-chain-core-go/data/transaction" vmcommon "github.com/multiversx/mx-chain-vm-common-go" - "github.com/multiversx/mx-chain-go/state/stateChanges" + "github.com/multiversx/mx-chain-go/state" ) // StateChangesCollectorStub represents a mock for the StateChangesCollector interface type StateChangesCollectorStub struct { - AddStateChangeCalled func(stateChange stateChanges.StateChange) - AddSaveAccountStateChangeCalled func(oldAccount, account vmcommon.AccountHandler, stateChange stateChanges.StateChange) + AddStateChangeCalled func(stateChange state.StateChange) + AddSaveAccountStateChangeCalled func(oldAccount, account vmcommon.AccountHandler, stateChange state.StateChange) ResetCalled func() AddTxHashToCollectedStateChangesCalled func(txHash []byte, tx *transaction.Transaction) SetIndexToLastStateChangeCalled func(index int) error @@ -22,14 +22,14 @@ type StateChangesCollectorStub struct { } // AddStateChange - -func (s *StateChangesCollectorStub) AddStateChange(stateChange stateChanges.StateChange) { +func (s *StateChangesCollectorStub) AddStateChange(stateChange state.StateChange) { if s.AddStateChangeCalled != nil { s.AddStateChangeCalled(stateChange) } } // AddSaveAccountStateChange - -func (s *StateChangesCollectorStub) AddSaveAccountStateChange(oldAccount, account vmcommon.AccountHandler, stateChange stateChanges.StateChange) { +func (s *StateChangesCollectorStub) AddSaveAccountStateChange(oldAccount, account vmcommon.AccountHandler, stateChange state.StateChange) { if s.AddSaveAccountStateChangeCalled != nil { s.AddSaveAccountStateChangeCalled(oldAccount, account, stateChange) } From 9f68130e7667c628ba5afd47d942df60ea728aad Mon Sep 17 00:00:00 2001 From: ssd04 Date: Fri, 20 Sep 2024 15:49:21 +0300 Subject: [PATCH 53/55] add data analysis collector unit tests --- state/stateChanges/dataAnalysisCollector.go | 10 +- .../dataAnalysisCollector_test.go | 205 +++++++++++++++++- state/stateChanges/export_test.go | 6 + state/stateChanges/writeCollector.go | 33 +-- storage/errors.go | 3 + testscommon/state/userAccountStub.go | 3 +- 6 files changed, 228 insertions(+), 32 deletions(-) diff --git a/state/stateChanges/dataAnalysisCollector.go b/state/stateChanges/dataAnalysisCollector.go index 52aae5ab0cc..a81633d462e 100644 --- a/state/stateChanges/dataAnalysisCollector.go +++ b/state/stateChanges/dataAnalysisCollector.go @@ -14,7 +14,6 @@ import ( "github.com/multiversx/mx-chain-go/storage" ) -// TODO: use proto stucts type dataAnalysisStateChangeDTO struct { state.StateChange Operation string `json:"operation"` @@ -54,7 +53,7 @@ type dataAnalysisCollector struct { // NewDataAnalysisStateChangesCollector will create a new instance of data analysis collector func NewDataAnalysisStateChangesCollector(storer storage.Persister) (*dataAnalysisCollector, error) { if check.IfNil(storer) { - return nil, storage.ErrNilPersisterFactory + return nil, storage.ErrNilPersister } return &dataAnalysisCollector{ @@ -72,7 +71,7 @@ func (scc *dataAnalysisCollector) AddSaveAccountStateChange(oldAccount, account checkAccountChanges(oldAccount, account, dataAnalysisStateChange) - scc.AddStateChange(stateChange) + scc.AddStateChange(dataAnalysisStateChange) } func checkAccountChanges(oldAcc, newAcc vmcommon.AccountHandler, stateChange *dataAnalysisStateChangeDTO) { @@ -127,15 +126,12 @@ func (scc *dataAnalysisCollector) AddStateChange(stateChange state.StateChange) } func (scc *dataAnalysisCollector) getDataAnalysisStateChangesForTxs() ([]dataAnalysisStateChangesForTx, error) { - scc.stateChangesMut.Lock() - defer scc.stateChangesMut.Unlock() - stateChangesForTxs, err := scc.getStateChangesForTxs() if err != nil { return nil, err } - dataAnalysisStateChangesForTxs := make([]dataAnalysisStateChangesForTx, len(stateChangesForTxs)) + dataAnalysisStateChangesForTxs := make([]dataAnalysisStateChangesForTx, 0) for _, stateChangeForTx := range stateChangesForTxs { txHash := string(stateChangeForTx.TxHash) diff --git a/state/stateChanges/dataAnalysisCollector_test.go b/state/stateChanges/dataAnalysisCollector_test.go index 82e25cad9d4..1d33d0e79d4 100644 --- a/state/stateChanges/dataAnalysisCollector_test.go +++ b/state/stateChanges/dataAnalysisCollector_test.go @@ -1,3 +1,206 @@ package stateChanges -// TODO: add unit tests +import ( + "math/big" + "testing" + + data "github.com/multiversx/mx-chain-core-go/data/stateChange" + "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/multiversx/mx-chain-go/storage" + "github.com/multiversx/mx-chain-go/storage/mock" + "github.com/multiversx/mx-chain-go/testscommon/state" + "github.com/stretchr/testify/require" +) + +func TestNewDataAnalysisCollector(t *testing.T) { + t.Parallel() + + t.Run("nil storer", func(t *testing.T) { + t.Parallel() + + dsc, err := NewDataAnalysisStateChangesCollector(nil) + require.Nil(t, dsc) + require.Equal(t, storage.ErrNilPersister, err) + }) + + t.Run("should work", func(t *testing.T) { + t.Parallel() + + dsc, err := NewDataAnalysisStateChangesCollector(&mock.PersisterStub{}) + require.Nil(t, err) + require.False(t, dsc.IsInterfaceNil()) + }) +} + +func TestDataAnalysisStateChangesCollector_AddStateChange(t *testing.T) { + t.Parallel() + + dsc, err := NewDataAnalysisStateChangesCollector(&mock.PersisterStub{}) + require.Nil(t, err) + + require.Equal(t, 0, len(dsc.stateChanges)) + + dsc.AddStateChange(&data.StateChange{ + Type: "write", + }) + dsc.AddStateChange(&data.StateChange{ + Type: "read", + }) + dsc.AddStateChange(&data.StateChange{ + Type: "write", + }) + + require.Equal(t, 3, len(dsc.stateChanges)) +} + +func TestDataAnalysisStateChangesCollector_AddSaveAccountStateChange(t *testing.T) { + t.Parallel() + + t.Run("nil old account should return early", func(t *testing.T) { + t.Parallel() + + dsc, err := NewDataAnalysisStateChangesCollector(&mock.PersisterStub{}) + require.Nil(t, err) + + dsc.AddSaveAccountStateChange( + nil, + &state.UserAccountStub{}, + &data.StateChange{ + Type: "saveAccount", + Index: 2, + TxHash: []byte("txHash1"), + MainTrieKey: []byte("key1"), + }, + ) + + dsc.AddTxHashToCollectedStateChanges([]byte("txHash1"), &transaction.Transaction{}) + + stateChangesForTx := dsc.GetStateChanges() + require.Equal(t, 1, len(stateChangesForTx)) + + sc := stateChangesForTx[0].StateChanges[0] + dasc, ok := sc.(*dataAnalysisStateChangeDTO) + require.True(t, ok) + + require.False(t, dasc.Nonce) + require.False(t, dasc.Balance) + require.False(t, dasc.CodeHash) + require.False(t, dasc.RootHash) + require.False(t, dasc.DeveloperReward) + require.False(t, dasc.OwnerAddress) + require.False(t, dasc.UserName) + require.False(t, dasc.CodeMetadata) + }) + + t.Run("nil new account should return early", func(t *testing.T) { + t.Parallel() + + dsc, err := NewDataAnalysisStateChangesCollector(&mock.PersisterStub{}) + require.Nil(t, err) + + dsc.AddSaveAccountStateChange( + &state.UserAccountStub{}, + nil, + &data.StateChange{ + Type: "saveAccount", + Index: 2, + TxHash: []byte("txHash1"), + MainTrieKey: []byte("key1"), + }, + ) + + dsc.AddTxHashToCollectedStateChanges([]byte("txHash1"), &transaction.Transaction{}) + + stateChangesForTx := dsc.GetStateChanges() + require.Equal(t, 1, len(stateChangesForTx)) + + sc := stateChangesForTx[0].StateChanges[0] + dasc, ok := sc.(*dataAnalysisStateChangeDTO) + require.True(t, ok) + + require.False(t, dasc.Nonce) + require.False(t, dasc.Balance) + require.False(t, dasc.CodeHash) + require.False(t, dasc.RootHash) + require.False(t, dasc.DeveloperReward) + require.False(t, dasc.OwnerAddress) + require.False(t, dasc.UserName) + require.False(t, dasc.CodeMetadata) + }) + + t.Run("should work", func(t *testing.T) { + t.Parallel() + + dsc, err := NewDataAnalysisStateChangesCollector(&mock.PersisterStub{}) + require.Nil(t, err) + + dsc.AddSaveAccountStateChange( + &state.UserAccountStub{ + Nonce: 0, + Balance: big.NewInt(0), + DeveloperRewards: big.NewInt(0), + UserName: []byte{0}, + Owner: []byte{0}, + Address: []byte{0}, + CodeMetadata: []byte{0}, + CodeHash: []byte{0}, + GetRootHashCalled: func() []byte { + return []byte{0} + }, + }, + &state.UserAccountStub{ + Nonce: 1, + Balance: big.NewInt(1), + DeveloperRewards: big.NewInt(1), + UserName: []byte{1}, + Owner: []byte{1}, + Address: []byte{1}, + CodeMetadata: []byte{1}, + CodeHash: []byte{1}, + GetRootHashCalled: func() []byte { + return []byte{1} + }, + }, + &data.StateChange{ + Type: "saveAccount", + Index: 2, + TxHash: []byte("txHash1"), + MainTrieKey: []byte("key1"), + }, + ) + + dsc.AddTxHashToCollectedStateChanges([]byte("txHash1"), &transaction.Transaction{}) + + stateChangesForTx := dsc.GetStateChanges() + require.Equal(t, 1, len(stateChangesForTx)) + + sc := stateChangesForTx[0].StateChanges[0] + dasc, ok := sc.(*dataAnalysisStateChangeDTO) + require.True(t, ok) + + require.True(t, dasc.Nonce) + require.True(t, dasc.Balance) + require.True(t, dasc.CodeHash) + require.True(t, dasc.RootHash) + require.True(t, dasc.DeveloperReward) + require.True(t, dasc.OwnerAddress) + require.True(t, dasc.UserName) + require.True(t, dasc.CodeMetadata) + }) +} + +func TestDataAnalysisStateChangesCollector_Reset(t *testing.T) { + t.Parallel() + + dsc, err := NewDataAnalysisStateChangesCollector(&mock.PersisterStub{}) + require.Nil(t, err) + + numStateChanges := 10 + for i := 0; i < numStateChanges; i++ { + dsc.AddStateChange(getDefaultStateChange()) + } + require.Equal(t, numStateChanges, len(dsc.stateChanges)) + + dsc.Reset() + require.Equal(t, 0, len(dsc.GetStateChanges())) +} diff --git a/state/stateChanges/export_test.go b/state/stateChanges/export_test.go index 86350f45a5c..ed4b541ee20 100644 --- a/state/stateChanges/export_test.go +++ b/state/stateChanges/export_test.go @@ -5,3 +5,9 @@ func (scc *stateChangesCollector) GetStateChanges() []StateChangesForTx { scs, _ := scc.getStateChangesForTxs() return scs } + +// GetStateChanges - +func (dsc *dataAnalysisCollector) GetStateChanges() []dataAnalysisStateChangesForTx { + scs, _ := dsc.getDataAnalysisStateChangesForTxs() + return scs +} diff --git a/state/stateChanges/writeCollector.go b/state/stateChanges/writeCollector.go index 80e6f65069e..1691ca0fedb 100644 --- a/state/stateChanges/writeCollector.go +++ b/state/stateChanges/writeCollector.go @@ -93,31 +93,18 @@ func (scc *stateChangesCollector) GetStateChangesForTxs() map[string]*data.State for _, stateChange := range scc.stateChanges { txHash := string(stateChange.GetTxHash()) - if _, ok := stateChangesForTxs[txHash]; !ok { - stateChangesForTxs[txHash] = &data.StateChanges{StateChanges: []*data.StateChange{ - { - Type: stateChange.GetType(), - Index: stateChange.GetIndex(), - TxHash: stateChange.GetTxHash(), - MainTrieKey: stateChange.GetMainTrieKey(), - MainTrieVal: stateChange.GetMainTrieVal(), - Operation: stateChange.GetOperation(), - DataTrieChanges: stateChange.GetDataTrieChanges(), - }, - }, + st, ok := stateChange.(*data.StateChange) + if !ok { + continue + } + + _, ok = stateChangesForTxs[txHash] + if !ok { + stateChangesForTxs[txHash] = &data.StateChanges{ + StateChanges: []*data.StateChange{st}, } } else { - stateChangesForTxs[txHash].StateChanges = append(stateChangesForTxs[txHash].StateChanges, - &data.StateChange{ - Type: stateChange.GetType(), - Index: stateChange.GetIndex(), - TxHash: stateChange.GetTxHash(), - MainTrieKey: stateChange.GetMainTrieKey(), - MainTrieVal: stateChange.GetMainTrieVal(), - Operation: stateChange.GetOperation(), - DataTrieChanges: stateChange.GetDataTrieChanges(), - }, - ) + stateChangesForTxs[txHash].StateChanges = append(stateChangesForTxs[txHash].StateChanges, st) } } diff --git a/storage/errors.go b/storage/errors.go index 4cf2716bfab..6eb89672527 100644 --- a/storage/errors.go +++ b/storage/errors.go @@ -82,6 +82,9 @@ var ErrNotSupportedCacheType = storageErrors.ErrNotSupportedCacheType // ErrDBIsClosed is raised when the DB is closed var ErrDBIsClosed = storageErrors.ErrDBIsClosed +// ErrNilPersister is raised when a nil persister is provided +var ErrNilPersister = storageErrors.ErrNilPersister + // ErrEpochKeepIsLowerThanNumActive signals that num epochs to keep is lower than num active epochs var ErrEpochKeepIsLowerThanNumActive = errors.New("num epochs to keep is lower than num active epochs") diff --git a/testscommon/state/userAccountStub.go b/testscommon/state/userAccountStub.go index d4af407abba..22f28958c6a 100644 --- a/testscommon/state/userAccountStub.go +++ b/testscommon/state/userAccountStub.go @@ -17,6 +17,7 @@ var _ state.UserAccountHandler = (*UserAccountStub)(nil) // UserAccountStub - type UserAccountStub struct { + Nonce uint64 Balance *big.Int DeveloperRewards *big.Int UserName []byte @@ -108,7 +109,7 @@ func (u *UserAccountStub) IncreaseNonce(_ uint64) { // GetNonce - func (u *UserAccountStub) GetNonce() uint64 { - return 0 + return u.Nonce } // SetCode - From 038dde05dbda36e7eedaec297a53d361ee5ba33d Mon Sep 17 00:00:00 2001 From: ssd04 Date: Thu, 26 Sep 2024 16:51:14 +0300 Subject: [PATCH 54/55] update to use mock for state changes collector --- genesis/process/genesisBlockCreator_test.go | 3 +-- .../state/stateTrie/stateTrie_test.go | 9 ++++----- integrationTests/testProcessorNode.go | 3 +-- outport/process/outportDataProvider_test.go | 4 ++-- process/rewardTransaction/process_test.go | 3 +-- process/scToProtocol/stakingToPeer_test.go | 3 +-- .../smartContract/processorV2/process_test.go | 3 +-- state/accountsDB_test.go | 17 ++++++++--------- 8 files changed, 19 insertions(+), 26 deletions(-) diff --git a/genesis/process/genesisBlockCreator_test.go b/genesis/process/genesisBlockCreator_test.go index e44a56bc377..13ca44fc785 100644 --- a/genesis/process/genesisBlockCreator_test.go +++ b/genesis/process/genesisBlockCreator_test.go @@ -25,7 +25,6 @@ import ( "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/state/accounts" factoryState "github.com/multiversx/mx-chain-go/state/factory" - "github.com/multiversx/mx-chain-go/state/stateChanges" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/testscommon" commonMocks "github.com/multiversx/mx-chain-go/testscommon/common" @@ -204,7 +203,7 @@ func createMockArgument( Hasher: &hashingMocks.HasherMock{}, Marshaller: &mock.MarshalizerMock{}, EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, - StateChangesCollector: stateChanges.NewStateChangesCollector(), + StateChangesCollector: &stateMock.StateChangesCollectorStub{}, } accCreator, err := factoryState.NewAccountCreator(argsAccCreator) require.Nil(t, err) diff --git a/integrationTests/state/stateTrie/stateTrie_test.go b/integrationTests/state/stateTrie/stateTrie_test.go index 1c78c97eb9d..7110d09613d 100644 --- a/integrationTests/state/stateTrie/stateTrie_test.go +++ b/integrationTests/state/stateTrie/stateTrie_test.go @@ -39,7 +39,6 @@ import ( "github.com/multiversx/mx-chain-go/state/factory" "github.com/multiversx/mx-chain-go/state/iteratorChannelsProvider" "github.com/multiversx/mx-chain-go/state/lastSnapshotMarker" - "github.com/multiversx/mx-chain-go/state/stateChanges" "github.com/multiversx/mx-chain-go/state/storagePruningManager" "github.com/multiversx/mx-chain-go/state/storagePruningManager/evictionWaitingList" "github.com/multiversx/mx-chain-go/storage" @@ -1063,7 +1062,7 @@ func createAccounts( Hasher: integrationTests.TestHasher, Marshaller: integrationTests.TestMarshalizer, EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, - StateChangesCollector: stateChanges.NewStateChangesCollector(), + StateChangesCollector: &stateMock.StateChangesCollectorStub{}, } accCreator, _ := factory.NewAccountCreator(argsAccCreator) snapshotsManager, _ := state.NewSnapshotsManager(state.ArgsNewSnapshotsManager{ @@ -1085,7 +1084,7 @@ func createAccounts( StoragePruningManager: spm, AddressConverter: &testscommon.PubkeyConverterMock{}, SnapshotsManager: snapshotsManager, - StateChangesCollector: stateChanges.NewStateChangesCollector(), + StateChangesCollector: &stateMock.StateChangesCollectorStub{}, } adb, _ := state.NewAccountsDB(argsAccountsDB) @@ -2737,7 +2736,7 @@ func createAccountsDBTestSetup() *state.AccountsDB { Hasher: integrationTests.TestHasher, Marshaller: integrationTests.TestMarshalizer, EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, - StateChangesCollector: stateChanges.NewStateChangesCollector(), + StateChangesCollector: &stateMock.StateChangesCollectorStub{}, } accCreator, _ := factory.NewAccountCreator(argsAccCreator) @@ -2761,7 +2760,7 @@ func createAccountsDBTestSetup() *state.AccountsDB { StoragePruningManager: spm, AddressConverter: &testscommon.PubkeyConverterMock{}, SnapshotsManager: snapshotsManager, - StateChangesCollector: stateChanges.NewStateChangesCollector(), + StateChangesCollector: &stateMock.StateChangesCollectorStub{}, } adb, _ := state.NewAccountsDB(argsAccountsDB) diff --git a/integrationTests/testProcessorNode.go b/integrationTests/testProcessorNode.go index f2a37cd60d3..bed1bef8177 100644 --- a/integrationTests/testProcessorNode.go +++ b/integrationTests/testProcessorNode.go @@ -98,7 +98,6 @@ import ( "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/state/blockInfoProviders" - stateDisabled "github.com/multiversx/mx-chain-go/state/disabled" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/cache" "github.com/multiversx/mx-chain-go/storage/storageunit" @@ -2235,7 +2234,7 @@ func (tpn *TestProcessorNode) initBlockProcessor() { BlockProcessingCutoffHandler: &testscommon.BlockProcessingCutoffStub{}, ManagedPeersHolder: &testscommon.ManagedPeersHolderStub{}, SentSignaturesTracker: &testscommon.SentSignatureTrackerStub{}, - StateChangesCollector: stateDisabled.NewDisabledStateChangesCollector(), + StateChangesCollector: &stateMock.StateChangesCollectorStub{}, } if check.IfNil(tpn.EpochStartNotifier) { diff --git a/outport/process/outportDataProvider_test.go b/outport/process/outportDataProvider_test.go index 0f801968ac6..43c2fc6b57b 100644 --- a/outport/process/outportDataProvider_test.go +++ b/outport/process/outportDataProvider_test.go @@ -16,7 +16,6 @@ import ( "github.com/multiversx/mx-chain-go/outport/mock" "github.com/multiversx/mx-chain-go/outport/process/transactionsfee" - "github.com/multiversx/mx-chain-go/state/stateChanges" "github.com/multiversx/mx-chain-go/testscommon" commonMocks "github.com/multiversx/mx-chain-go/testscommon/common" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" @@ -24,6 +23,7 @@ import ( "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" "github.com/multiversx/mx-chain-go/testscommon/marshallerMock" "github.com/multiversx/mx-chain-go/testscommon/shardingMocks" + "github.com/multiversx/mx-chain-go/testscommon/state" ) func createArgOutportDataProvider() ArgOutportDataProvider { @@ -47,7 +47,7 @@ func createArgOutportDataProvider() ArgOutportDataProvider { ExecutionOrderHandler: &commonMocks.TxExecutionOrderHandlerStub{}, Marshaller: &marshallerMock.MarshalizerMock{}, Hasher: &hashingMocks.HasherMock{}, - StateChangesCollector: stateChanges.NewStateChangesCollector(), + StateChangesCollector: &state.StateChangesCollectorStub{}, } } diff --git a/process/rewardTransaction/process_test.go b/process/rewardTransaction/process_test.go index 85ec1cf6373..13b546213ab 100644 --- a/process/rewardTransaction/process_test.go +++ b/process/rewardTransaction/process_test.go @@ -12,7 +12,6 @@ import ( "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/process/rewardTransaction" "github.com/multiversx/mx-chain-go/state/accounts" - "github.com/multiversx/mx-chain-go/state/disabled" "github.com/multiversx/mx-chain-go/state/trackableDataTrie" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" @@ -266,7 +265,7 @@ func TestRewardTxProcessor_ProcessRewardTransactionToASmartContractShouldWork(t &hashingMocks.HasherMock{}, &marshallerMock.MarshalizerMock{}, enableEpochsHandlerMock.NewEnableEpochsHandlerStub(), - disabled.NewDisabledStateChangesCollector(), + &stateMock.StateChangesCollectorStub{}, ) userAccount, _ := accounts.NewUserAccount(address, dtt, &trie.TrieLeafParserStub{}) accountsDb := &stateMock.AccountsStub{ diff --git a/process/scToProtocol/stakingToPeer_test.go b/process/scToProtocol/stakingToPeer_test.go index d0ec8c6eeba..fdc97c84908 100644 --- a/process/scToProtocol/stakingToPeer_test.go +++ b/process/scToProtocol/stakingToPeer_test.go @@ -20,7 +20,6 @@ import ( "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/state/accounts" - "github.com/multiversx/mx-chain-go/state/stateChanges" "github.com/multiversx/mx-chain-go/state/trackableDataTrie" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" @@ -67,7 +66,7 @@ func createStakingScAccount() state.UserAccountHandler { &hashingMocks.HasherMock{}, &marshallerMock.MarshalizerMock{}, enableEpochsHandlerMock.NewEnableEpochsHandlerStub(), - stateChanges.NewStateChangesCollector(), + &stateMock.StateChangesCollectorStub{}, ) userAcc, _ := accounts.NewUserAccount(vm.StakingSCAddress, dtt, &trie.TrieLeafParserStub{}) diff --git a/process/smartContract/processorV2/process_test.go b/process/smartContract/processorV2/process_test.go index be934b41b07..0a1bbdd77e1 100644 --- a/process/smartContract/processorV2/process_test.go +++ b/process/smartContract/processorV2/process_test.go @@ -35,7 +35,6 @@ import ( "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/state" stateFactory "github.com/multiversx/mx-chain-go/state/factory" - "github.com/multiversx/mx-chain-go/state/stateChanges" "github.com/multiversx/mx-chain-go/storage/storageunit" "github.com/multiversx/mx-chain-go/storage/txcache" "github.com/multiversx/mx-chain-go/testscommon" @@ -67,7 +66,7 @@ func createAccount(address []byte) state.UserAccountHandler { Hasher: &hashingMocks.HasherMock{}, Marshaller: &marshallerMock.MarshalizerMock{}, EnableEpochsHandler: enableEpochsHandlerMock.NewEnableEpochsHandlerStub(), - StateChangesCollector: stateChanges.NewStateChangesCollector(), + StateChangesCollector: &stateMock.StateChangesCollectorStub{}, } accountFactory, _ := stateFactory.NewAccountCreator(argsAccCreation) account, _ := accountFactory.CreateAccount(address) diff --git a/state/accountsDB_test.go b/state/accountsDB_test.go index 97cfdf5724e..b316899cb97 100644 --- a/state/accountsDB_test.go +++ b/state/accountsDB_test.go @@ -33,7 +33,6 @@ import ( "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/state/accounts" "github.com/multiversx/mx-chain-go/state/dataTrieValue" - stateDisabled "github.com/multiversx/mx-chain-go/state/disabled" "github.com/multiversx/mx-chain-go/state/factory" "github.com/multiversx/mx-chain-go/state/iteratorChannelsProvider" "github.com/multiversx/mx-chain-go/state/lastSnapshotMarker" @@ -86,7 +85,7 @@ func createMockAccountsDBArgs() state.ArgsAccountsDB { StoragePruningManager: disabled.NewDisabledStoragePruningManager(), AddressConverter: &testscommon.PubkeyConverterMock{}, SnapshotsManager: snapshotsManager, - StateChangesCollector: stateChanges.NewStateChangesCollector(), + StateChangesCollector: &stateMock.StateChangesCollectorStub{}, } } @@ -157,7 +156,7 @@ func getDefaultStateComponents( Hasher: hasher, Marshaller: marshaller, EnableEpochsHandler: enableEpochsHandler, - StateChangesCollector: stateDisabled.NewDisabledStateChangesCollector(), + StateChangesCollector: &stateMock.StateChangesCollectorStub{}, } accCreator, _ := factory.NewAccountCreator(argsAccCreator) @@ -1935,7 +1934,7 @@ func TestAccountsDB_MainTrieAutomaticallyMarksCodeUpdatesForEviction(t *testing. Hasher: hasher, Marshaller: marshaller, EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, - StateChangesCollector: stateChanges.NewStateChangesCollector(), + StateChangesCollector: &stateMock.StateChangesCollectorStub{}, } argsAccountsDB.AccountFactory, _ = factory.NewAccountCreator(argsAccCreator) argsAccountsDB.StoragePruningManager = spm @@ -2017,7 +2016,7 @@ func TestAccountsDB_RemoveAccountMarksObsoleteHashesForEviction(t *testing.T) { Hasher: hasher, Marshaller: marshaller, EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, - StateChangesCollector: stateChanges.NewStateChangesCollector(), + StateChangesCollector: &stateMock.StateChangesCollectorStub{}, } argsAccountsDB.AccountFactory, _ = factory.NewAccountCreator(argsAccCreator) argsAccountsDB.StoragePruningManager = spm @@ -2220,7 +2219,7 @@ func TestAccountsDB_GetCode(t *testing.T) { Hasher: hasher, Marshaller: marshaller, EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, - StateChangesCollector: stateChanges.NewStateChangesCollector(), + StateChangesCollector: &stateMock.StateChangesCollectorStub{}, } argsAccountsDB.AccountFactory, _ = factory.NewAccountCreator(argsAccCreator) argsAccountsDB.StoragePruningManager = spm @@ -2388,7 +2387,7 @@ func TestAccountsDB_Close(t *testing.T) { Hasher: hasher, Marshaller: marshaller, EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, - StateChangesCollector: stateChanges.NewStateChangesCollector(), + StateChangesCollector: &stateMock.StateChangesCollectorStub{}, } argsAccountsDB.AccountFactory, _ = factory.NewAccountCreator(argsAccCreator) argsAccountsDB.StoragePruningManager = spm @@ -2677,7 +2676,7 @@ func BenchmarkAccountsDb_GetCodeEntry(b *testing.B) { Hasher: hasher, Marshaller: marshaller, EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, - StateChangesCollector: stateChanges.NewStateChangesCollector(), + StateChangesCollector: &stateMock.StateChangesCollectorStub{}, } argsAccountsDB.AccountFactory, _ = factory.NewAccountCreator(argsAccCreator) argsAccountsDB.StoragePruningManager = spm @@ -3000,7 +2999,7 @@ func TestAccountsDB_RevertTxWhichMigratesDataRemovesMigratedData(t *testing.T) { Hasher: hasher, Marshaller: marshaller, EnableEpochsHandler: enableEpochsHandler, - StateChangesCollector: stateChanges.NewStateChangesCollector(), + StateChangesCollector: &stateMock.StateChangesCollectorStub{}, } argsAccountsDB.AccountFactory, _ = factory.NewAccountCreator(argsAccCreator) argsAccountsDB.StoragePruningManager = spm From 004d008e2bd996e5b22b0ec07318786603386887 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Fri, 27 Sep 2024 07:38:16 +0300 Subject: [PATCH 55/55] fix linter issues. --- state/stateChanges/dataAnalysisCollector.go | 4 +--- state/stateChanges/writeCollector_test.go | 7 ++++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/state/stateChanges/dataAnalysisCollector.go b/state/stateChanges/dataAnalysisCollector.go index a81633d462e..481eb8e9fc6 100644 --- a/state/stateChanges/dataAnalysisCollector.go +++ b/state/stateChanges/dataAnalysisCollector.go @@ -134,9 +134,7 @@ func (scc *dataAnalysisCollector) getDataAnalysisStateChangesForTxs() ([]dataAna dataAnalysisStateChangesForTxs := make([]dataAnalysisStateChangesForTx, 0) for _, stateChangeForTx := range stateChangesForTxs { - txHash := string(stateChangeForTx.TxHash) - - cachedTx, txOk := scc.cachedTxs[txHash] + cachedTx, txOk := scc.cachedTxs[string(stateChangeForTx.TxHash)] if !txOk { return nil, fmt.Errorf("did not find tx in cache") } diff --git a/state/stateChanges/writeCollector_test.go b/state/stateChanges/writeCollector_test.go index 4414ca2f911..1dc76a124c7 100644 --- a/state/stateChanges/writeCollector_test.go +++ b/state/stateChanges/writeCollector_test.go @@ -7,9 +7,10 @@ import ( data "github.com/multiversx/mx-chain-core-go/data/stateChange" "github.com/multiversx/mx-chain-core-go/data/transaction" - "github.com/multiversx/mx-chain-go/state" "github.com/stretchr/testify/assert" + "github.com/multiversx/mx-chain-go/state" + "github.com/stretchr/testify/require" ) @@ -276,7 +277,7 @@ func TestStateChangesCollector_GetStateChangesForTx(t *testing.T) { require.Equal(t, stateChangesForTx, map[string]*data.StateChanges{ "hash0": { - []*data.StateChange{ + StateChanges: []*data.StateChange{ {Type: "write", TxHash: []byte("hash0")}, {Type: "write", TxHash: []byte("hash0")}, {Type: "write", TxHash: []byte("hash0")}, @@ -285,7 +286,7 @@ func TestStateChangesCollector_GetStateChangesForTx(t *testing.T) { }, }, "hash1": { - []*data.StateChange{ + StateChanges: []*data.StateChange{ {Type: "write", TxHash: []byte("hash1")}, {Type: "write", TxHash: []byte("hash1")}, {Type: "write", TxHash: []byte("hash1")},