From 1059e89c388a00220a2cf52dd32ff57c27c2f50d Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Wed, 26 Jun 2024 13:55:16 +0200 Subject: [PATCH 001/489] system_tests: check that state is freed in StateDB finalizer --- system_tests/recreatestate_rpc_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/system_tests/recreatestate_rpc_test.go b/system_tests/recreatestate_rpc_test.go index bf321808de..dcd936259c 100644 --- a/system_tests/recreatestate_rpc_test.go +++ b/system_tests/recreatestate_rpc_test.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "math/big" + "runtime" "strings" "sync" "testing" @@ -521,12 +522,23 @@ func TestGettingStateForRPCFullNode(t *testing.T) { blockCountRequiredToFlushDirties := builder.execConfig.Caching.BlockCount makeSomeTransfers(t, ctx, builder, blockCountRequiredToFlushDirties) + // force garbage callection to check if it won't break anything + runtime.GC() + exists = state.Exist(addr) err = state.Error() Require(t, err) if !exists { Fatal(t, "User2 address does not exist in the state") } + + // force garbage collection of StateDB object, what should cause the state finalizer to run + state = nil + runtime.GC() + _, err = bc.StateAt(header.Root) + if err == nil { + Fatal(t, "StateAndHeaderByNumber didn't failed as expected") + } } func TestGettingStateForRPCHybridArchiveNode(t *testing.T) { From d7a50e7a60ee2dad03464f5ba1ed6d415d2fa42f Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Wed, 26 Jun 2024 14:23:10 +0200 Subject: [PATCH 002/489] system_tests: merge tests for getting state in full node and sparse archive node --- system_tests/recreatestate_rpc_test.go | 57 +++++++------------------- 1 file changed, 15 insertions(+), 42 deletions(-) diff --git a/system_tests/recreatestate_rpc_test.go b/system_tests/recreatestate_rpc_test.go index dcd936259c..1aa98588f6 100644 --- a/system_tests/recreatestate_rpc_test.go +++ b/system_tests/recreatestate_rpc_test.go @@ -488,16 +488,11 @@ func TestSkippingSavingStateAndRecreatingAfterRestart(t *testing.T) { } } -func TestGettingStateForRPCFullNode(t *testing.T) { +func testGettingState(t *testing.T, execConfig *gethexec.Config) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := gethexec.ConfigDefaultTest() - execConfig.Caching.SnapshotCache = 0 // disable snapshots - execConfig.Caching.BlockAge = 0 // use only Caching.BlockCount to keep only last N blocks in dirties cache, no matter how new they are - execConfig.Sequencer.MaxBlockSpeed = 0 - execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 builder, cancelNode := prepareNodeWithHistory(t, ctx, execConfig, 16) - execNode, _ := builder.L2.ExecNode, builder.L2.Client + execNode := builder.L2.ExecNode defer cancelNode() bc := execNode.Backend.ArbInterface().BlockChain() api := execNode.Backend.APIBackend() @@ -541,10 +536,17 @@ func TestGettingStateForRPCFullNode(t *testing.T) { } } -func TestGettingStateForRPCHybridArchiveNode(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() +func TestGettingState(t *testing.T) { execConfig := gethexec.ConfigDefaultTest() + execConfig.Caching.SnapshotCache = 0 // disable snapshots + execConfig.Caching.BlockAge = 0 // use only Caching.BlockCount to keep only last N blocks in dirties cache, no matter how new they are + execConfig.Sequencer.MaxBlockSpeed = 0 + execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 + t.Run("TestGettingStateForRPCFullNode", func(t *testing.T) { + testGettingState(t, execConfig) + }) + + execConfig = gethexec.ConfigDefaultTest() execConfig.Caching.Archive = true execConfig.Caching.MaxNumberOfBlocksToSkipStateSaving = 128 execConfig.Caching.BlockCount = 128 @@ -552,38 +554,9 @@ func TestGettingStateForRPCHybridArchiveNode(t *testing.T) { execConfig.Caching.BlockAge = 0 // use only Caching.BlockCount to keep only last N blocks in dirties cache, no matter how new they are execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 - builder, cancelNode := prepareNodeWithHistory(t, ctx, execConfig, 16) - execNode, _ := builder.L2.ExecNode, builder.L2.Client - defer cancelNode() - bc := execNode.Backend.ArbInterface().BlockChain() - api := execNode.Backend.APIBackend() - - header := bc.CurrentBlock() - if header == nil { - Fatal(t, "failed to get current block header") - } - state, _, err := api.StateAndHeaderByNumber(ctx, rpc.BlockNumber(header.Number.Uint64())) - Require(t, err) - addr := builder.L2Info.GetAddress("User2") - exists := state.Exist(addr) - err = state.Error() - Require(t, err) - if !exists { - Fatal(t, "User2 address does not exist in the state") - } - // Get the state again to avoid caching - state, _, err = api.StateAndHeaderByNumber(ctx, rpc.BlockNumber(header.Number.Uint64())) - Require(t, err) - - blockCountRequiredToFlushDirties := builder.execConfig.Caching.BlockCount - makeSomeTransfers(t, ctx, builder, blockCountRequiredToFlushDirties) - - exists = state.Exist(addr) - err = state.Error() - Require(t, err) - if !exists { - Fatal(t, "User2 address does not exist in the state") - } + t.Run("TestGettingStateForRPCSparseArchiveNode", func(t *testing.T) { + testGettingState(t, execConfig) + }) } func TestStateAndHeaderForRecentBlock(t *testing.T) { From 97622b8a9393c63d136ff580df31b6d6d9938e0a Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Thu, 27 Jun 2024 12:35:37 +0200 Subject: [PATCH 003/489] add comments to TestStateAndHeaderForRecentBlock --- system_tests/recreatestate_rpc_test.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/system_tests/recreatestate_rpc_test.go b/system_tests/recreatestate_rpc_test.go index 1aa98588f6..65938abcc5 100644 --- a/system_tests/recreatestate_rpc_test.go +++ b/system_tests/recreatestate_rpc_test.go @@ -559,6 +559,8 @@ func TestGettingState(t *testing.T) { }) } +// regression test for issue caused by accessing block state that has just been committed to TrieDB but not yet referenced in core.BlockChain.writeBlockWithState (here called state of "recent" block) +// before the corresponding fix, access to the recent block state caused premature garbage collection of the head block state func TestStateAndHeaderForRecentBlock(t *testing.T) { threads := 32 ctx, cancel := context.WithCancel(context.Background()) @@ -597,15 +599,22 @@ func TestStateAndHeaderForRecentBlock(t *testing.T) { }() api := builder.L2.ExecNode.Backend.APIBackend() db := builder.L2.ExecNode.Backend.ChainDb() - i := 1 + + recentBlock := 1 var mtx sync.RWMutex var wgCallers sync.WaitGroup for j := 0; j < threads && ctx.Err() == nil; j++ { wgCallers.Add(1) + // each thread attempts to get state for a block that is just being created (here called recent): + // 1. Before state trie node is referenced in core.BlockChain.writeBlockWithState, block body is written to database with key prefix `b` followed by block number and then block hash (see: rawdb.blockBodyKey) + // 2. Each thread tries to read the block body entry to: a. extract recent block hash b. congest resource usage to slow down execution of core.BlockChain.writeBlockWithState + // 3. After extracting the hash from block body entry key, StateAndHeaderByNumberOfHash is called for the hash. It is expected that it will: + // a. either fail with "ahead of current block" if we made it before StateDB commit + // b. or it will succeed if state was already commited - then the recentBlock is advanced go func() { defer wgCallers.Done() mtx.RLock() - blockNumber := i + blockNumber := recentBlock mtx.RUnlock() for blockNumber < 300 && ctx.Err() == nil { prefix := make([]byte, 8) @@ -624,8 +633,8 @@ func TestStateAndHeaderForRecentBlock(t *testing.T) { _, _, err := api.StateAndHeaderByNumberOrHash(ctx, rpc.BlockNumberOrHash{BlockHash: &blockHash}) if err == nil { mtx.Lock() - if blockNumber == i { - i++ + if blockNumber == recentBlock { + recentBlock++ } mtx.Unlock() break @@ -645,7 +654,7 @@ func TestStateAndHeaderForRecentBlock(t *testing.T) { } it.Release() mtx.RLock() - blockNumber = i + blockNumber = recentBlock mtx.RUnlock() } }() From ef65cc7115a93e08995bed43a1ef51d2ac59a3a5 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 28 Jun 2024 11:59:03 +0200 Subject: [PATCH 004/489] use subtests in TestSkippingSavingStateAndRecreatingAfterRestart --- system_tests/recreatestate_rpc_test.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/system_tests/recreatestate_rpc_test.go b/system_tests/recreatestate_rpc_test.go index 65938abcc5..adff4a3822 100644 --- a/system_tests/recreatestate_rpc_test.go +++ b/system_tests/recreatestate_rpc_test.go @@ -327,6 +327,7 @@ func TestRecreateStateForRPCBlockNotFoundWhileRecreating(t *testing.T) { } func testSkippingSavingStateAndRecreatingAfterRestart(t *testing.T, cacheConfig *gethexec.CachingConfig, txCount int) { + t.Parallel() maxRecreateStateDepth := int64(30 * 1000 * 1000) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -455,20 +456,26 @@ func TestSkippingSavingStateAndRecreatingAfterRestart(t *testing.T) { cacheConfig.SnapshotCache = 0 // disable snapshots cacheConfig.BlockAge = 0 // use only Caching.BlockCount to keep only last N blocks in dirties cache, no matter how new they are + runTestCase := func(t *testing.T, cacheConfig gethexec.CachingConfig, txes int) { + t.Run(fmt.Sprintf("TestSkippingSavingStateAndRecreatingAfterRestart-skip-blocks-%d-skip-gas-%d-txes-%d", cacheConfig.MaxNumberOfBlocksToSkipStateSaving, cacheConfig.MaxAmountOfGasToSkipStateSaving, txes), func(t *testing.T) { + testSkippingSavingStateAndRecreatingAfterRestart(t, &cacheConfig, txes) + }) + } + // test defaults - testSkippingSavingStateAndRecreatingAfterRestart(t, &cacheConfig, 512) + runTestCase(t, cacheConfig, 512) cacheConfig.MaxNumberOfBlocksToSkipStateSaving = 127 cacheConfig.MaxAmountOfGasToSkipStateSaving = 0 - testSkippingSavingStateAndRecreatingAfterRestart(t, &cacheConfig, 512) + runTestCase(t, cacheConfig, 512) cacheConfig.MaxNumberOfBlocksToSkipStateSaving = 0 cacheConfig.MaxAmountOfGasToSkipStateSaving = 15 * 1000 * 1000 - testSkippingSavingStateAndRecreatingAfterRestart(t, &cacheConfig, 512) + runTestCase(t, cacheConfig, 512) cacheConfig.MaxNumberOfBlocksToSkipStateSaving = 127 cacheConfig.MaxAmountOfGasToSkipStateSaving = 15 * 1000 * 1000 - testSkippingSavingStateAndRecreatingAfterRestart(t, &cacheConfig, 512) + runTestCase(t, cacheConfig, 512) // lower number of blocks in triegc below 100 blocks, to be able to check for nonexistence in testSkippingSavingStateAndRecreatingAfterRestart (it doesn't check last BlockCount blocks as some of them may be persisted on node shutdown) cacheConfig.BlockCount = 16 @@ -483,7 +490,7 @@ func TestSkippingSavingStateAndRecreatingAfterRestart(t *testing.T) { for _, skipBlocks := range skipBlockValues[:len(skipBlockValues)-2] { cacheConfig.MaxAmountOfGasToSkipStateSaving = skipGas cacheConfig.MaxNumberOfBlocksToSkipStateSaving = uint32(skipBlocks) - testSkippingSavingStateAndRecreatingAfterRestart(t, &cacheConfig, 100) + runTestCase(t, cacheConfig, 100) } } } From 35b9054bbee2f5440747629d5b17672e8ca4d8c7 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 28 Jun 2024 12:26:51 +0200 Subject: [PATCH 005/489] update the recent block test comment --- system_tests/recreatestate_rpc_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/system_tests/recreatestate_rpc_test.go b/system_tests/recreatestate_rpc_test.go index adff4a3822..1a2e00d7f4 100644 --- a/system_tests/recreatestate_rpc_test.go +++ b/system_tests/recreatestate_rpc_test.go @@ -616,8 +616,8 @@ func TestStateAndHeaderForRecentBlock(t *testing.T) { // 1. Before state trie node is referenced in core.BlockChain.writeBlockWithState, block body is written to database with key prefix `b` followed by block number and then block hash (see: rawdb.blockBodyKey) // 2. Each thread tries to read the block body entry to: a. extract recent block hash b. congest resource usage to slow down execution of core.BlockChain.writeBlockWithState // 3. After extracting the hash from block body entry key, StateAndHeaderByNumberOfHash is called for the hash. It is expected that it will: - // a. either fail with "ahead of current block" if we made it before StateDB commit - // b. or it will succeed if state was already commited - then the recentBlock is advanced + // a. either fail with "ahead of current block" if we made it before rawdb.WriteCanonicalHash is called in core.BlockChain.writeHeadBlock, what is called after writeBlockWithState finishes, + // b. or it will succeed if the canonical hash was written for the block meaning that writeBlockWithState was fully executed (i.a. state root trie node correctly referenced) - then the recentBlock is advanced go func() { defer wgCallers.Done() mtx.RLock() From 6acb87ad531f37163fc22beaf98904b865f92baa Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Tue, 16 Jul 2024 19:00:32 +0530 Subject: [PATCH 006/489] Merge v1.13.14 --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 7c6e05f753..227dc3fa1e 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 7c6e05f75337d59562a6e2b8e1bdf1e445ad14fb +Subproject commit 227dc3fa1e31889a0ac2126dc407a46ec5f8e409 From 826b230f19567cfdd96254786187d17e4678bea5 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Tue, 16 Jul 2024 19:42:10 +0530 Subject: [PATCH 007/489] Merge v1.13.15 --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 227dc3fa1e..41caa90e92 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 227dc3fa1e31889a0ac2126dc407a46ec5f8e409 +Subproject commit 41caa90e923312bf7b39f8de4b8e9534724cd4cd From b042b2a078c25fbcf4193aab5ac34fb85dca9357 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Thu, 18 Jul 2024 17:25:07 +0530 Subject: [PATCH 008/489] Merge v1.14.0 --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 41caa90e92..30a31796f4 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 41caa90e923312bf7b39f8de4b8e9534724cd4cd +Subproject commit 30a31796f41fb14fb52b123b0896f14f35208ee6 From 5dc1096b2993175bf73bb2dbd0b69bc976baf8e3 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Thu, 18 Jul 2024 17:29:04 +0530 Subject: [PATCH 009/489] fix ci --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 30a31796f4..112bf69280 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 30a31796f41fb14fb52b123b0896f14f35208ee6 +Subproject commit 112bf69280b5a6954eb1ae80447a86cc71ab519a From cedf55bfe82ff15acb5cf98f177f6c8bcdd91b3a Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Thu, 18 Jul 2024 19:07:09 +0530 Subject: [PATCH 010/489] Fix --- arbos/arbosState/initialize.go | 7 +- arbos/engine.go | 11 +- arbos/l1pricing_test.go | 3 +- arbos/programs/api.go | 8 +- arbos/tx_processor.go | 4 +- arbos/util/tracing.go | 19 +- arbos/util/transfer.go | 9 +- blsSignatures/blsSignatures.go | 22 +-- cmd/genericconf/config.go | 2 +- cmd/genericconf/loglevel.go | 2 +- execution/nodeInterface/NodeInterface.go | 8 +- go-ethereum | 2 +- go.mod | 35 ++-- go.sum | 228 ++++------------------- precompiles/ArbOwner_test.go | 3 +- system_tests/common_test.go | 13 +- system_tests/das_test.go | 2 +- util/blobs/blobs.go | 4 +- util/headerreader/blob_client.go | 2 +- util/testhelpers/testhelpers.go | 2 +- 20 files changed, 120 insertions(+), 266 deletions(-) diff --git a/arbos/arbosState/initialize.go b/arbos/arbosState/initialize.go index 486c6ae33a..796f959c73 100644 --- a/arbos/arbosState/initialize.go +++ b/arbos/arbosState/initialize.go @@ -5,6 +5,7 @@ package arbosState import ( "errors" + "github.com/ethereum/go-ethereum/core/tracing" "math/big" "sort" @@ -143,7 +144,7 @@ func InitializeArbosInDatabase(db ethdb.Database, initData statetransfer.InitDat if err != nil { return common.Hash{}, err } - statedb.SetBalance(account.Addr, uint256.MustFromBig(account.EthBalance)) + statedb.SetBalance(account.Addr, uint256.MustFromBig(account.EthBalance), tracing.BalanceChangeUnspecified) statedb.SetNonce(account.Addr, account.Nonce) if account.ContractInfo != nil { statedb.SetCode(account.Addr, account.ContractInfo.Code) @@ -174,7 +175,7 @@ func initializeRetryables(statedb *state.StateDB, rs *retryables.RetryableState, return err } if r.Timeout <= currentTimestamp { - statedb.AddBalance(r.Beneficiary, uint256.MustFromBig(r.Callvalue)) + statedb.AddBalance(r.Beneficiary, uint256.MustFromBig(r.Callvalue), tracing.BalanceChangeUnspecified) continue } retryablesList = append(retryablesList, r) @@ -193,7 +194,7 @@ func initializeRetryables(statedb *state.StateDB, rs *retryables.RetryableState, addr := r.To to = &addr } - statedb.AddBalance(retryables.RetryableEscrowAddress(r.Id), uint256.MustFromBig(r.Callvalue)) + statedb.AddBalance(retryables.RetryableEscrowAddress(r.Id), uint256.MustFromBig(r.Callvalue), tracing.BalanceChangeUnspecified) _, err := rs.CreateRetryable(r.Id, r.Timeout, r.From, to, r.Callvalue, r.Beneficiary, r.Calldata) if err != nil { return err diff --git a/arbos/engine.go b/arbos/engine.go index 0014e8ab96..a4aa9c46a9 100644 --- a/arbos/engine.go +++ b/arbos/engine.go @@ -48,16 +48,15 @@ func (e Engine) Prepare(chain consensus.ChainHeaderReader, header *types.Header) return nil } -func (e Engine) Finalize(chain consensus.ChainHeaderReader, header *types.Header, state *state.StateDB, txs []*types.Transaction, uncles []*types.Header, withdrawals []*types.Withdrawal) { - FinalizeBlock(header, txs, state, chain.Config()) +func (e Engine) Finalize(chain consensus.ChainHeaderReader, header *types.Header, state *state.StateDB, body *types.Body) { + FinalizeBlock(header, body.Transactions, state, chain.Config()) } -func (e Engine) FinalizeAndAssemble(chain consensus.ChainHeaderReader, header *types.Header, state *state.StateDB, txs []*types.Transaction, - uncles []*types.Header, receipts []*types.Receipt, withdrawals []*types.Withdrawal) (*types.Block, error) { +func (e Engine) FinalizeAndAssemble(chain consensus.ChainHeaderReader, header *types.Header, state *state.StateDB, body *types.Body, receipts []*types.Receipt) (*types.Block, error) { - e.Finalize(chain, header, state, txs, uncles, withdrawals) + e.Finalize(chain, header, state, body) - block := types.NewBlock(header, txs, nil, receipts, trie.NewStackTrie(nil)) + block := types.NewBlock(header, body.Transactions, nil, receipts, trie.NewStackTrie(nil)) return block, nil } diff --git a/arbos/l1pricing_test.go b/arbos/l1pricing_test.go index 6e2b1b7eec..c8851bee8c 100644 --- a/arbos/l1pricing_test.go +++ b/arbos/l1pricing_test.go @@ -4,6 +4,7 @@ package arbos import ( + "github.com/ethereum/go-ethereum/core/tracing" "math/big" "testing" @@ -172,7 +173,7 @@ func _testL1PricingFundsDue(t *testing.T, testParams *l1PricingTest, expectedRes // create some fake collection balanceAdded := big.NewInt(int64(testParams.fundsCollectedPerSecond * 3)) unitsAdded := testParams.unitsPerSecond * 3 - evm.StateDB.AddBalance(l1pricing.L1PricerFundsPoolAddress, uint256.MustFromBig(balanceAdded)) + evm.StateDB.AddBalance(l1pricing.L1PricerFundsPoolAddress, uint256.MustFromBig(balanceAdded), tracing.BalanceChangeUnspecified) err = l1p.SetL1FeesAvailable(balanceAdded) Require(t, err) err = l1p.SetUnitsSinceUpdate(unitsAdded) diff --git a/arbos/programs/api.go b/arbos/programs/api.go index 65a58a47c2..d44d14098c 100644 --- a/arbos/programs/api.go +++ b/arbos/programs/api.go @@ -139,7 +139,7 @@ func newApiClosures( // Tracing: emit the call (value transfer is done later in evm.Call) if tracingInfo != nil { - tracingInfo.Tracer.CaptureState(0, opcode, startGas, baseCost+gas, scope, []byte{}, depth, nil) + tracingInfo.Tracer.OnOpcode(0, byte(opcode), startGas, baseCost+gas, scope, []byte{}, depth, nil) } // EVM rule: calls that pay get a stipend (opCall) @@ -204,7 +204,7 @@ func newApiClosures( // Tracing: emit the create if tracingInfo != nil { - tracingInfo.Tracer.CaptureState(0, opcode, startGas, baseCost+gas, scope, []byte{}, depth, nil) + tracingInfo.Tracer.OnOpcode(0, byte(opcode), startGas, baseCost+gas, scope, []byte{}, depth, nil) } var res []byte @@ -270,8 +270,8 @@ func newApiClosures( captureHostio := func(name string, args, outs []byte, startInk, endInk uint64) { tracingInfo.Tracer.CaptureStylusHostio(name, args, outs, startInk, endInk) if name == "evm_gas_left" || name == "evm_ink_left" { - tracingInfo.Tracer.CaptureState(0, vm.GAS, 0, 0, scope, []byte{}, depth, nil) - tracingInfo.Tracer.CaptureState(0, vm.POP, 0, 0, scope, []byte{}, depth, nil) + tracingInfo.Tracer.OnOpcode(0, byte(vm.GAS), 0, 0, scope, []byte{}, depth, nil) + tracingInfo.Tracer.OnOpcode(0, byte(vm.POP), 0, 0, scope, []byte{}, depth, nil) } } diff --git a/arbos/tx_processor.go b/arbos/tx_processor.go index b08c7c5d30..a7abbed6d5 100644 --- a/arbos/tx_processor.go +++ b/arbos/tx_processor.go @@ -153,13 +153,13 @@ func (p *TxProcessor) StartTxHook() (endTxNow bool, gasUsed uint64, err error, r } evm.IncrementDepth() // fake a call from := p.msg.From - tracer.CaptureStart(evm, from, *p.msg.To, false, p.msg.Data, p.msg.GasLimit, p.msg.Value) + tracer.OnEnter(evm.Depth(), byte(vm.CALL), from, *p.msg.To, p.msg.Data, p.msg.GasLimit, p.msg.Value) tracingInfo = util.NewTracingInfo(evm, from, *p.msg.To, util.TracingDuringEVM) p.state = arbosState.OpenSystemArbosStateOrPanic(evm.StateDB, tracingInfo, false) return func() { - tracer.CaptureEnd(nil, p.state.Burner.Burned(), nil) + tracer.OnExit(evm.Depth(), nil, p.state.Burner.Burned(), nil, false) evm.DecrementDepth() // fake the return to the first faked call tracingInfo = util.NewTracingInfo(evm, from, *p.msg.To, util.TracingAfterEVM) diff --git a/arbos/util/tracing.go b/arbos/util/tracing.go index f0f101bc20..e4cf2aa08c 100644 --- a/arbos/util/tracing.go +++ b/arbos/util/tracing.go @@ -5,6 +5,7 @@ package util import ( "fmt" + "github.com/ethereum/go-ethereum/core/tracing" "math/big" "github.com/ethereum/go-ethereum/common" @@ -21,7 +22,7 @@ const ( ) type TracingInfo struct { - Tracer vm.EVMLogger + Tracer *tracing.Hooks Scenario TracingScenario Contract *vm.Contract Depth int @@ -65,7 +66,7 @@ func (info *TracingInfo) RecordEmitLog(topics []common.Hash, data []byte) { Contract: info.Contract, } logType := fmt.Sprintf("LOG%d", len(topics)) - info.Tracer.CaptureState(0, vm.StringToOp(logType), 0, 0, scope, []byte{}, info.Depth, nil) + info.Tracer.OnOpcode(0, byte(vm.StringToOp(logType)), 0, 0, scope, []byte{}, info.Depth, nil) } func (info *TracingInfo) RecordStorageGet(key common.Hash) { @@ -76,7 +77,7 @@ func (info *TracingInfo) RecordStorageGet(key common.Hash) { Stack: TracingStackFromArgs(HashToUint256(key)), Contract: info.Contract, } - tracer.CaptureState(0, vm.SLOAD, 0, 0, scope, []byte{}, info.Depth, nil) + tracer.OnOpcode(0, byte(vm.SLOAD), 0, 0, scope, []byte{}, info.Depth, nil) } else { tracer.CaptureArbitrumStorageGet(key, info.Depth, info.Scenario == TracingBeforeEVM) } @@ -90,7 +91,7 @@ func (info *TracingInfo) RecordStorageSet(key, value common.Hash) { Stack: TracingStackFromArgs(HashToUint256(key), HashToUint256(value)), Contract: info.Contract, } - tracer.CaptureState(0, vm.SSTORE, 0, 0, scope, []byte{}, info.Depth, nil) + tracer.OnOpcode(0, byte(vm.SSTORE), 0, 0, scope, []byte{}, info.Depth, nil) } else { tracer.CaptureArbitrumStorageSet(key, value, info.Depth, info.Scenario == TracingBeforeEVM) } @@ -115,8 +116,8 @@ func (info *TracingInfo) MockCall(input []byte, gas uint64, from, to common.Addr ), Contract: contract, } - tracer.CaptureState(0, vm.CALL, 0, 0, scope, []byte{}, depth, nil) - tracer.CaptureEnter(vm.INVALID, from, to, input, 0, amount) + tracer.OnOpcode(0, byte(vm.CALL), 0, 0, scope, []byte{}, depth, nil) + tracer.OnEnter(depth, byte(vm.INVALID), from, to, input, 0, amount) retScope := &vm.ScopeContext{ Memory: vm.NewMemory(), @@ -126,8 +127,8 @@ func (info *TracingInfo) MockCall(input []byte, gas uint64, from, to common.Addr ), Contract: contract, } - tracer.CaptureState(0, vm.RETURN, 0, 0, retScope, []byte{}, depth+1, nil) - tracer.CaptureExit(nil, 0, nil) + tracer.OnOpcode(0, byte(vm.RETURN), 0, 0, retScope, []byte{}, depth+1, nil) + tracer.OnExit(depth+1, nil, 0, nil, false) popScope := &vm.ScopeContext{ Memory: vm.NewMemory(), @@ -136,7 +137,7 @@ func (info *TracingInfo) MockCall(input []byte, gas uint64, from, to common.Addr ), Contract: contract, } - tracer.CaptureState(0, vm.POP, 0, 0, popScope, []byte{}, depth, nil) + tracer.OnOpcode(0, byte(vm.POP), 0, 0, popScope, []byte{}, depth, nil) } func HashToUint256(hash common.Hash) uint256.Int { diff --git a/arbos/util/transfer.go b/arbos/util/transfer.go index e293ef13c3..1240928eb6 100644 --- a/arbos/util/transfer.go +++ b/arbos/util/transfer.go @@ -7,6 +7,7 @@ package util import ( "errors" "fmt" + "github.com/ethereum/go-ethereum/core/tracing" "math/big" "github.com/ethereum/go-ethereum/common" @@ -33,14 +34,14 @@ func TransferBalance( if arbmath.BigLessThan(balance.ToBig(), amount) { return fmt.Errorf("%w: addr %v have %v want %v", vm.ErrInsufficientBalance, *from, balance, amount) } - evm.StateDB.SubBalance(*from, uint256.MustFromBig(amount)) + evm.StateDB.SubBalance(*from, uint256.MustFromBig(amount), tracing.BalanceChangeTransfer) if evm.Context.ArbOSVersion >= 30 { // ensure the from account is "touched" for EIP-161 - evm.StateDB.AddBalance(*from, &uint256.Int{}) + evm.StateDB.AddBalance(*from, &uint256.Int{}, tracing.BalanceChangeTransfer) } } if to != nil { - evm.StateDB.AddBalance(*to, uint256.MustFromBig(amount)) + evm.StateDB.AddBalance(*to, uint256.MustFromBig(amount), tracing.BalanceChangeTransfer) } if tracer := evm.Config.Tracer; tracer != nil { if evm.Depth() != 0 && scenario != TracingDuringEVM { @@ -50,7 +51,7 @@ func TransferBalance( } if scenario != TracingDuringEVM { - tracer.CaptureArbitrumTransfer(evm, from, to, amount, scenario == TracingBeforeEVM, purpose) + tracer.CaptureArbitrumTransfer(from, to, amount, scenario == TracingBeforeEVM, purpose) return nil } diff --git a/blsSignatures/blsSignatures.go b/blsSignatures/blsSignatures.go index b597d6a07e..cfcbc34d80 100644 --- a/blsSignatures/blsSignatures.go +++ b/blsSignatures/blsSignatures.go @@ -7,10 +7,8 @@ import ( cryptorand "crypto/rand" "encoding/base64" "errors" - "math/big" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/crypto/bls12381" + bls12381 "github.com/kilic/bls12-381" ) type PublicKey struct { @@ -18,13 +16,13 @@ type PublicKey struct { validityProof *bls12381.PointG1 // if this is nil, key came from a trusted source } -type PrivateKey *big.Int +type PrivateKey *bls12381.Fr type Signature *bls12381.PointG1 func GeneratePrivKeyString() (string, error) { - g2 := bls12381.NewG2() - privKey, err := cryptorand.Int(cryptorand.Reader, g2.Q()) + fr := bls12381.NewFr() + privKey, err := fr.Rand(cryptorand.Reader) if err != nil { return "", err } @@ -35,8 +33,8 @@ func GeneratePrivKeyString() (string, error) { } func GenerateKeys() (PublicKey, PrivateKey, error) { - g2 := bls12381.NewG2() - privateKey, err := cryptorand.Int(cryptorand.Reader, g2.Q()) + fr := bls12381.NewFr() + privateKey, err := fr.Rand(cryptorand.Reader) if err != nil { return PublicKey{}, nil, err } @@ -120,7 +118,7 @@ func verifySignature2(sig Signature, message []byte, publicKey PublicKey, keyVal return false, err } - engine := bls12381.NewPairingEngine() + engine := bls12381.NewEngine() engine.Reset() engine.AddPair(pointOnCurve, publicKey.key) leftSide := engine.Result() @@ -156,7 +154,7 @@ func VerifyAggregatedSignatureDifferentMessages(sig Signature, messages [][]byte if len(messages) != len(pubKeys) { return false, errors.New("len(messages) does not match (len(pub keys) in verification") } - engine := bls12381.NewPairingEngine() + engine := bls12381.NewEngine() engine.Reset() for i, msg := range messages { pointOnCurve, err := hashToG1Curve(msg, false) @@ -242,11 +240,11 @@ func PublicKeyFromBytes(in []byte, trustedSource bool) (PublicKey, error) { } func PrivateKeyToBytes(priv PrivateKey) []byte { - return ((*big.Int)(priv)).Bytes() + return bls12381.NewFr().Set(priv).ToBytes() } func PrivateKeyFromBytes(in []byte) (PrivateKey, error) { - return new(big.Int).SetBytes(in), nil + return bls12381.NewFr().FromBytes(in), nil } func SignatureToBytes(sig Signature) []byte { diff --git a/cmd/genericconf/config.go b/cmd/genericconf/config.go index 06e1fcd12d..7c0c5034b3 100644 --- a/cmd/genericconf/config.go +++ b/cmd/genericconf/config.go @@ -6,12 +6,12 @@ package genericconf import ( "errors" "io" + "log/slog" "time" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" flag "github.com/spf13/pflag" - "golang.org/x/exp/slog" ) type ConfConfig struct { diff --git a/cmd/genericconf/loglevel.go b/cmd/genericconf/loglevel.go index f7ad05a2cc..79cba22439 100644 --- a/cmd/genericconf/loglevel.go +++ b/cmd/genericconf/loglevel.go @@ -5,11 +5,11 @@ package genericconf import ( "errors" + "log/slog" "strconv" "strings" "github.com/ethereum/go-ethereum/log" - "golang.org/x/exp/slog" ) func ToSlogLevel(str string) (slog.Level, error) { diff --git a/execution/nodeInterface/NodeInterface.go b/execution/nodeInterface/NodeInterface.go index 9179a52718..6fe1b2ef99 100644 --- a/execution/nodeInterface/NodeInterface.go +++ b/execution/nodeInterface/NodeInterface.go @@ -516,10 +516,10 @@ func (n NodeInterface) GasEstimateL1Component( args.Gas = (*hexutil.Uint64)(&randomGas) // We set the run mode to eth_call mode here because we want an exact estimate, not a padded estimate - msg, err := args.ToMessage(randomGas, n.header, evm.StateDB.(*state.StateDB), core.MessageEthcallMode) - if err != nil { + if err := args.CallDefaults(randomGas, evm.Context.BaseFee, evm.ChainConfig().ChainID); err != nil { return 0, nil, nil, err } + msg := args.ToMessage(evm.Context.BaseFee, randomGas, n.header, evm.StateDB.(*state.StateDB), core.MessageEthcallMode) pricing := c.State.L1PricingState() l1BaseFeeEstimate, err := pricing.PricePerUnit() @@ -572,10 +572,10 @@ func (n NodeInterface) GasEstimateComponents( // Setting the gas currently doesn't affect the PosterDataCost, // but we do it anyways for accuracy with potential future changes. args.Gas = &totalRaw - msg, err := args.ToMessage(gasCap, n.header, evm.StateDB.(*state.StateDB), core.MessageGasEstimationMode) - if err != nil { + if err := args.CallDefaults(gasCap, evm.Context.BaseFee, evm.ChainConfig().ChainID); err != nil { return 0, 0, nil, nil, err } + msg := args.ToMessage(evm.Context.BaseFee, gasCap, n.header, evm.StateDB.(*state.StateDB), core.MessageGasEstimationMode) brotliCompressionLevel, err := c.State.BrotliCompressionLevel() if err != nil { return 0, 0, nil, nil, fmt.Errorf("failed to get brotli compression level: %w", err) diff --git a/go-ethereum b/go-ethereum index 112bf69280..3ec24bb403 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 112bf69280b5a6954eb1ae80447a86cc71ab519a +Subproject commit 3ec24bb40398be604e7d122a83039bba1a2fc41b diff --git a/go.mod b/go.mod index d0c8d5e719..78acfc8162 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.10 github.com/aws/aws-sdk-go-v2/service/s3 v1.26.9 github.com/cavaliergopher/grab/v3 v3.0.1 - github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 + github.com/cockroachdb/pebble v1.1.0 github.com/codeclysm/extract/v3 v3.0.2 github.com/dgraph-io/badger/v4 v4.2.0 github.com/enescakir/emoji v1.0.0 @@ -32,6 +32,7 @@ require ( github.com/google/uuid v1.3.0 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/holiman/uint256 v1.2.4 + github.com/kilic/bls12-381 v0.1.0 github.com/knadh/koanf v1.4.0 github.com/mailru/easygo v0.0.0-20190618140210-3c14a0dc985f github.com/mitchellh/mapstructure v1.4.1 @@ -42,11 +43,11 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/wasmerio/wasmer-go v1.0.4 github.com/wealdtech/go-merkletree v1.0.0 - golang.org/x/crypto v0.21.0 + golang.org/x/crypto v0.22.0 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa - golang.org/x/sys v0.18.0 - golang.org/x/term v0.18.0 - golang.org/x/tools v0.16.0 + golang.org/x/sys v0.19.0 + golang.org/x/term v0.19.0 + golang.org/x/tools v0.20.0 gopkg.in/natefinch/lumberjack.v2 v2.0.0 ) @@ -76,15 +77,15 @@ require ( github.com/bits-and-blooms/bitset v1.10.0 // indirect github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/cockroachdb/errors v1.9.1 // indirect - github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect - github.com/cockroachdb/redact v1.1.3 // indirect + github.com/cockroachdb/errors v1.11.1 // indirect + github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect + github.com/cockroachdb/redact v1.1.5 // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect github.com/consensys/bavard v0.1.13 // indirect github.com/consensys/gnark-crypto v0.12.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 // indirect - github.com/crate-crypto/go-kzg-4844 v0.7.0 // indirect + github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/deckarep/golang-set/v2 v2.1.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect @@ -93,14 +94,14 @@ require ( github.com/dlclark/regexp2 v1.7.0 // indirect github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127 // indirect github.com/dustin/go-humanize v1.0.0 // indirect - github.com/ethereum/c-kzg-4844 v0.4.0 // indirect + github.com/ethereum/c-kzg-4844 v1.0.0 // indirect github.com/fjl/memsize v0.0.2 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/gammazero/deque v0.2.1 // indirect github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46 // indirect github.com/gdamore/encoding v1.0.0 // indirect - github.com/getsentry/sentry-go v0.12.0 // indirect + github.com/getsentry/sentry-go v0.18.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/gobwas/pool v0.2.1 // indirect @@ -109,7 +110,7 @@ require ( github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/glog v1.0.0 // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect github.com/google/flatbuffers v1.12.1 // indirect github.com/google/go-github/v62 v62.0.0 @@ -160,11 +161,11 @@ require ( github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/yuin/gopher-lua v1.1.1 // indirect go.opencensus.io v0.22.5 // indirect - golang.org/x/mod v0.14.0 // indirect - golang.org/x/net v0.21.0 // indirect - golang.org/x/sync v0.5.0 + golang.org/x/mod v0.17.0 // indirect + golang.org/x/net v0.24.0 // indirect + golang.org/x/sync v0.7.0 golang.org/x/text v0.14.0 // indirect - golang.org/x/time v0.3.0 // indirect - google.golang.org/protobuf v1.30.0 // indirect + golang.org/x/time v0.5.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) diff --git a/go.sum b/go.sum index ff4726b22f..44125e1bd1 100644 --- a/go.sum +++ b/go.sum @@ -31,26 +31,20 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= -github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1:1G1pk05UrOh0NlF1oeaaix1x8XzrfjIDK47TY0Zehcw= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= -github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -66,7 +60,6 @@ github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/arduino/go-paths-helper v1.2.0 h1:qDW93PR5IZUN/jzO4rCtexiwF8P4OIcOmcSgAYLZfY4= github.com/arduino/go-paths-helper v1.2.0/go.mod h1:HpxtKph+g238EJHq4geEPv9p+gl3v5YYu35Yb+w31Ck= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go-v2 v1.9.2/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= @@ -128,7 +121,6 @@ github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAm github.com/aws/smithy-go v1.11.2/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM= github.com/aws/smithy-go v1.15.0 h1:PS/durmlzvAFpQHDs4wi4sNNP9ExsqZh6IlfdHXgKK8= github.com/aws/smithy-go v1.15.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= -github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -157,37 +149,30 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= -github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8= -github.com/cockroachdb/errors v1.9.1/go.mod h1:2sxOtL2WIc096WSZqZ5h8fa17rdDq9HZOZLBCor4mBk= -github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f h1:6jduT9Hfc0njg5jJ1DdKCFPdMBrp/mdZfCpa5h+WM74= -github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= -github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 h1:aPEJyR4rPBvDmeyi+l/FS/VtA00IWvjeFvjen1m1l1A= -github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593/go.mod h1:6hk1eMY/u5t+Cf18q5lFMUA1Rc+Sm5I6Ra1QuPyxXCo= -github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= -github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/errors v1.11.1 h1:xSEW75zKaKCWzR3OfxXUxgrk/NtT4G1MiOv5lWZazG8= +github.com/cockroachdb/errors v1.11.1/go.mod h1:8MUxA3Gi6b25tYlFEBGLf+D8aISL+M4MIpiWMSNRfxw= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= +github.com/cockroachdb/pebble v1.1.0 h1:pcFh8CdCIt2kmEpK0OIatq67Ln9uGDYY3d5XnE0LJG4= +github.com/cockroachdb/pebble v1.1.0/go.mod h1:sEHm5NOXxyiAoKWhoFxT8xMgd/f3RA6qUqQ1BXKrh2E= +github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= +github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/codeclysm/extract/v3 v3.0.2 h1:sB4LcE3Php7LkhZwN0n2p8GCwZe92PEQutdbGURf5xc= github.com/codeclysm/extract/v3 v3.0.2/go.mod h1:NKsw+hqua9H+Rlwy/w/3Qgt9jDonYEgB6wJu+25eOKw= -github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M= github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 h1:d28BXYi+wUpz1KBmiF9bWrjEMacUEREV6MBi2ODnrfQ= github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs= -github.com/crate-crypto/go-kzg-4844 v0.7.0 h1:C0vgZRk4q4EZ/JgPfzuSoxdCq3C3mOZMBShovmncxvA= -github.com/crate-crypto/go-kzg-4844 v0.7.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= +github.com/crate-crypto/go-kzg-4844 v1.0.0 h1:TsSgHwrkTKecKJ4kadtHi4b3xHW5dCFUDFnUp1TsawI= +github.com/crate-crypto/go-kzg-4844 v1.0.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -198,7 +183,6 @@ github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5il github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= -github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs= github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= @@ -217,18 +201,14 @@ github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/enescakir/emoji v1.0.0 h1:W+HsNql8swfCQFtioDGDHCHri8nudlK1n5p2rHCJoog= github.com/enescakir/emoji v1.0.0/go.mod h1:Bt1EKuLnKDTYpLALApstIkAjdDrS/8IAgTkKp+WKFD0= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= -github.com/ethereum/c-kzg-4844 v0.4.0 h1:3MS1s4JtA868KpJxroZoepdV0ZKBp3u/O5HcZ7R3nlY= -github.com/ethereum/c-kzg-4844 v0.4.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= -github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= +github.com/ethereum/c-kzg-4844 v1.0.0 h1:0X1LBXxaEtYD9xsyj9B9ctQEZIpnvVDeoBx8aHEwTNA= +github.com/ethereum/c-kzg-4844 v1.0.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= @@ -241,7 +221,6 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0= github.com/gammazero/deque v0.2.1/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU= -github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46 h1:BAIP2GihuqhwdILrV+7GJel5lyPV3u1+PgzrWLc0TkE= @@ -250,13 +229,10 @@ github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdk github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg= github.com/gdamore/tcell/v2 v2.7.1 h1:TiCcmpWHiAU7F0rA2I3S2Y4mmLmO9KHxJ7E1QhYzQbc= github.com/gdamore/tcell/v2 v2.7.1/go.mod h1:dSXtXTSK0VsW1biw65DZLZ2NKr7j0qP/0J7ONmsraWg= -github.com/getsentry/sentry-go v0.12.0 h1:era7g0re5iY13bHSdN/xMkyV+5zZppjRVQhZrXCaEIk= -github.com/getsentry/sentry-go v0.12.0/go.mod h1:NSap0JBYWzHND8oMbyi0+XZhUalc1TBdRL1M71JZW2c= -github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= -github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= -github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= -github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= -github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0= +github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnRJRLTXZr51aKQ= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -269,7 +245,6 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= @@ -279,27 +254,19 @@ github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyL github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= -github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= -github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= -github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/gobwas/ws v1.2.1 h1:F2aeBZrm2NDsc7vbovKrWSogd4wvfAxg0FQ89/iqOTk= github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= github.com/gobwas/ws-examples v0.0.0-20190625122829-a9e8908d9484 h1:XC9N1eiAyO1zg62dpOU8bex8emB/zluUtKcbLNjJxGI= github.com/gobwas/ws-examples v0.0.0-20190625122829-a9e8908d9484/go.mod h1:5nDZF4afNA1S7ZKcBXCMvDo4nuCTp1931DND7/W4aXo= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= -github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= -github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= -github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -332,13 +299,12 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= @@ -360,7 +326,6 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github/v62 v62.0.0 h1:/6mGCaRywZz9MuHyw9gD1CwsbmBX8GWsbFkwMmHdhl4= github.com/google/go-github/v62 v62.0.0/go.mod h1:EMxeUqGJq2xRu9DYBMwel/mr7kZrzUOfQmmpYrZn2a4= -github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -380,13 +345,10 @@ github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b h1:RMpPgZTSApbPf7xaVe github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/graph-gophers/graphql-go v1.3.0 h1:Eb9x/q6MFpCLz7jBCiP/WTxjSDrYLR1QY41SORZyNJ0= @@ -409,7 +371,6 @@ github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjG github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= @@ -429,16 +390,8 @@ github.com/holiman/uint256 v1.2.4/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXei github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= -github.com/hydrogen18/memlistener v0.0.0-20200120041712-dcc25e7acd91/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= -github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= -github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= -github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk= -github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g= -github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= @@ -449,13 +402,11 @@ github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/juju/clock v0.0.0-20180524022203-d293bb356ca4/go.mod h1:nD0vlnrUjcjJhqN5WuCWZyzfd5AHZAC9/ajvbSx69xA= github.com/juju/errors v0.0.0-20150916125642-1b5e39b83d18/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5 h1:rhqTjzJlm7EbkELJDKMTU7udov+Se0xZkWmugr6zGok= @@ -470,19 +421,12 @@ github.com/juju/utils v0.0.0-20180808125547-9dfc6dbfb02b/go.mod h1:6/KLg8Wz/y2KV github.com/juju/version v0.0.0-20161031051906-1f41e27e54f2/go.mod h1:kE8gK5X0CImdr7qpSKl3xB2PmpySSmfj7zVbkZFs81U= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= -github.com/kataras/golog v0.0.10/go.mod h1:yJ8YKCmyL+nWjERB90Qwn+bdyBZsaQwU3bTVFgkFIp8= -github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYbq3UhfoFmE= -github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE= -github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro= -github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8= +github.com/kilic/bls12-381 v0.1.0 h1:encrdjqKMEvabVQ7qYOKu1OvhqpK4s47wDYtNiPtlp4= +github.com/kilic/bls12-381 v0.1.0/go.mod h1:vDTTHJONJ6G+P2R74EhnyotQDTliQDnFEwhdmfzw1ig= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/knadh/koanf v1.4.0 h1:/k0Bh49SqLyLNfte9r6cvuZWrApOQhglOmhIU3L/zDw= github.com/knadh/koanf v1.4.0/go.mod h1:1cfH5223ZeZUOs8FU2UdTmaNfHpqgtjV0+NHjRO43gs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -499,39 +443,25 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y= -github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easygo v0.0.0-20190618140210-3c14a0dc985f h1:4+gHs0jJFJ06bfN8PshnM6cHcxGjRUVRLo5jndDiKRQ= github.com/mailru/easygo v0.0.0-20190618140210-3c14a0dc985f/go.mod h1:tHCZHV8b2A90ObojrEAzY0Lb03gxUxjDHr5IJyAh4ew= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8= -github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= @@ -556,13 +486,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= -github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= -github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/npillmayer/nestext v0.1.3/go.mod h1:h2lrijH8jpicr25dFY+oAJLyzlya6jhnuG+zWp9L0Uk= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -572,7 +497,6 @@ github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQ github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= @@ -584,7 +508,6 @@ github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5h github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -636,34 +559,21 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= -github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= -github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobtDnDzA= github.com/status-im/keycard-go v0.2.0/go.mod h1:wlp8ZLbsmrF6g6WjugPAx+IzoLrkdf9+mHxBEeo3Hbg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -671,7 +581,6 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= @@ -686,18 +595,8 @@ github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+F github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= -github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= -github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= -github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= -github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= @@ -706,21 +605,12 @@ github.com/wasmerio/wasmer-go v1.0.4 h1:MnqHoOGfiQ8MMq2RF6wyCeebKOe84G88h5yv+vmx github.com/wasmerio/wasmer-go v1.0.4/go.mod h1:0gzVdSfg6pysA6QVp6iVRPTagC6Wq9pOE8J86WKb2Fk= github.com/wealdtech/go-merkletree v1.0.0 h1:DsF1xMzj5rK3pSQM6mPv8jlyJyHXhFxpnA2bwEjMMBY= github.com/wealdtech/go-merkletree v1.0.0/go.mod h1:cdil512d/8ZC7Kx3bfrDvGMQXB25NTKbsm0rFrmDax4= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= -github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= -github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= -github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= -github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= @@ -733,18 +623,14 @@ go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= golang.org/x/crypto v0.0.0-20180214000028-650f4a345ab4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -769,7 +655,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -778,21 +663,18 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180406214816-61147c48b25b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -801,7 +683,6 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -820,15 +701,13 @@ golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -849,18 +728,16 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -869,9 +746,7 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -881,7 +756,6 @@ golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -899,21 +773,15 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -926,21 +794,20 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= +golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= @@ -950,18 +817,14 @@ golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -998,11 +861,10 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM= -golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/tools v0.20.0 h1:hz/CVckiOxybQvFw6h7b/q80NTr9IUQb4s1IIzW7KNY= +golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1029,7 +891,6 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1060,8 +921,6 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1076,7 +935,6 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1089,8 +947,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1102,12 +960,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= -gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= -gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/mgo.v2 v2.0.0-20160818015218-f2b6f6c918c4 h1:hILp2hNrRnYjZpmIbx70psAHbBSEcQ1NIzDcUbJ1b6g= gopkg.in/mgo.v2 v2.0.0-20160818015218-f2b6f6c918c4/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= -gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU= -gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= @@ -1122,9 +976,7 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/precompiles/ArbOwner_test.go b/precompiles/ArbOwner_test.go index 1f8c7ae4cd..456c26fa3d 100644 --- a/precompiles/ArbOwner_test.go +++ b/precompiles/ArbOwner_test.go @@ -6,6 +6,7 @@ package precompiles import ( "bytes" "encoding/json" + "github.com/ethereum/go-ethereum/core/tracing" "math/big" "testing" @@ -114,7 +115,7 @@ func TestArbOwner(t *testing.T) { Fail(t, avail) } deposited := big.NewInt(1000000) - evm.StateDB.AddBalance(l1pricing.L1PricerFundsPoolAddress, uint256.MustFromBig(deposited)) + evm.StateDB.AddBalance(l1pricing.L1PricerFundsPoolAddress, uint256.MustFromBig(deposited), tracing.BalanceChangeUnspecified) avail, err = gasInfo.GetL1FeesAvailable(callCtx, evm) Require(t, err) if avail.Sign() != 0 { diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 16d6b2f131..120f45dd34 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -9,6 +9,7 @@ import ( "encoding/hex" "encoding/json" "io" + "log/slog" "math/big" "net" "net/http" @@ -76,7 +77,6 @@ import ( "github.com/offchainlabs/nitro/statetransfer" "github.com/offchainlabs/nitro/util/testhelpers" "github.com/offchainlabs/nitro/util/testhelpers/github" - "golang.org/x/exp/slog" ) type info = *BlockchainTestInfo @@ -665,6 +665,7 @@ func createTestL1BlockChainWithConfig(t *testing.T, l1info info, stackConfig *no l1Genesis.BaseFee = big.NewInt(50 * params.GWei) nodeConf.Genesis = l1Genesis nodeConf.Miner.Etherbase = l1info.GetAddress("Faucet") + nodeConf.Miner.PendingFeeRecipient = l1info.GetAddress("Faucet") nodeConf.SyncMode = downloader.FullSync l1backend, err := eth.New(stack, &nodeConf) @@ -675,26 +676,24 @@ func createTestL1BlockChainWithConfig(t *testing.T, l1info info, stackConfig *no catalyst.RegisterSimulatedBeaconAPIs(stack, simBeacon) stack.RegisterLifecycle(simBeacon) - tempKeyStore := keystore.NewPlaintextKeyStore(t.TempDir()) + tempKeyStore := keystore.NewKeyStore(t.TempDir(), keystore.LightScryptN, keystore.LightScryptP) faucetAccount, err := tempKeyStore.ImportECDSA(l1info.Accounts["Faucet"].PrivateKey, "passphrase") Require(t, err) Require(t, tempKeyStore.Unlock(faucetAccount, "passphrase")) l1backend.AccountManager().AddBackend(tempKeyStore) - l1backend.SetEtherbase(l1info.GetAddress("Faucet")) stack.RegisterLifecycle(&lifecycle{stop: func() error { - l1backend.StopMining() - return nil + return l1backend.Stop() }}) stack.RegisterAPIs([]rpc.API{{ Namespace: "eth", - Service: filters.NewFilterAPI(filters.NewFilterSystem(l1backend.APIBackend, filters.Config{}), false), + Service: filters.NewFilterAPI(filters.NewFilterSystem(l1backend.APIBackend, filters.Config{})), }}) stack.RegisterAPIs(tracers.APIs(l1backend.APIBackend)) Require(t, stack.Start()) - Require(t, l1backend.StartMining()) + Require(t, l1backend.Start()) rpcClient := stack.Attach() diff --git a/system_tests/das_test.go b/system_tests/das_test.go index 2332f4ee9e..12b81d9d12 100644 --- a/system_tests/das_test.go +++ b/system_tests/das_test.go @@ -7,6 +7,7 @@ import ( "context" "encoding/base64" "io" + "log/slog" "math/big" "net" "net/http" @@ -35,7 +36,6 @@ import ( "github.com/offchainlabs/nitro/util/signature" "github.com/offchainlabs/nitro/validator/server_common" "github.com/offchainlabs/nitro/validator/valnode" - "golang.org/x/exp/slog" ) func startLocalDASServer( diff --git a/util/blobs/blobs.go b/util/blobs/blobs.go index 405c776bad..5c06aa7c3d 100644 --- a/util/blobs/blobs.go +++ b/util/blobs/blobs.go @@ -116,7 +116,7 @@ func ComputeCommitmentsAndHashes(blobs []kzg4844.Blob) ([]kzg4844.Commitment, [] for i := range blobs { var err error - commitments[i], err = kzg4844.BlobToCommitment(blobs[i]) + commitments[i], err = kzg4844.BlobToCommitment(&blobs[i]) if err != nil { return nil, nil, err } @@ -133,7 +133,7 @@ func ComputeBlobProofs(blobs []kzg4844.Blob, commitments []kzg4844.Commitment) ( proofs := make([]kzg4844.Proof, len(blobs)) for i := range blobs { var err error - proofs[i], err = kzg4844.ComputeBlobProof(blobs[i], commitments[i]) + proofs[i], err = kzg4844.ComputeBlobProof(&blobs[i], commitments[i]) if err != nil { return nil, err } diff --git a/util/headerreader/blob_client.go b/util/headerreader/blob_client.go index 2b47a940c3..215a03d9d9 100644 --- a/util/headerreader/blob_client.go +++ b/util/headerreader/blob_client.go @@ -247,7 +247,7 @@ func (b *BlobClient) blobSidecars(ctx context.Context, slot uint64, versionedHas var proof kzg4844.Proof copy(proof[:], blobItem.KzgProof) - err = kzg4844.VerifyBlobProof(output[outputIdx], commitment, proof) + err = kzg4844.VerifyBlobProof(&output[outputIdx], commitment, proof) if err != nil { return nil, fmt.Errorf("failed to verify blob proof for blob at slot(%d) at index(%d), blob(%s)", slot, blobItem.Index, pretty.FirstFewChars(blobItem.Blob.String())) } diff --git a/util/testhelpers/testhelpers.go b/util/testhelpers/testhelpers.go index 071429879e..46a456387e 100644 --- a/util/testhelpers/testhelpers.go +++ b/util/testhelpers/testhelpers.go @@ -7,6 +7,7 @@ import ( "context" crypto "crypto/rand" "io" + "log/slog" "math/big" "math/rand" "os" @@ -17,7 +18,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" "github.com/offchainlabs/nitro/util/colors" - "golang.org/x/exp/slog" ) // Fail a test should an error occur From ac6e0e7846598dbcf1c3635fbe067c7877aa4b27 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Mon, 22 Jul 2024 16:25:31 +0530 Subject: [PATCH 011/489] update submodule --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 41caa90e92..f802673ea6 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 41caa90e923312bf7b39f8de4b8e9534724cd4cd +Subproject commit f802673ea6499c14f567b28836bfeaeae31faf8d From 89c65f8607f2ae4800921e9e49676d8cc7f390fe Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Wed, 24 Jul 2024 17:20:24 +0530 Subject: [PATCH 012/489] update submodule --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 84fe1c5ee2..725aecd9bb 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 84fe1c5ee2b681fa6cfc81e60d57720d1717758e +Subproject commit 725aecd9bb4de8eda8a9ab1305c72d3567d2b1fc From 1ca29cf433c6e1e415765072e8212af301854701 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Thu, 25 Jul 2024 17:18:38 +0530 Subject: [PATCH 013/489] fix build --- arbos/arbosState/initialization_test.go | 5 +++-- arbos/arbosState/initialize.go | 6 +++++- arbos/programs/api.go | 2 +- system_tests/state_fuzz_test.go | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/arbos/arbosState/initialization_test.go b/arbos/arbosState/initialization_test.go index 34802392fe..e3de93f4ce 100644 --- a/arbos/arbosState/initialization_test.go +++ b/arbos/arbosState/initialization_test.go @@ -66,8 +66,9 @@ func tryMarshalUnmarshal(input *statetransfer.ArbosInitializationInfo, t *testin cacheConfig := core.DefaultCacheConfigWithScheme(env.GetTestStateScheme()) stateroot, err := InitializeArbosInDatabase(raw, cacheConfig, initReader, chainConfig, arbostypes.TestInitMessage, 0, 0) Require(t, err) - - triedbConfig := cacheConfig.TriedbConfig() + number, err := initReader.GetNextBlockNumber() + Require(t, err) + triedbConfig := cacheConfig.TriedbConfig(chainConfig.IsVerkle(new(big.Int).SetUint64(number), 0)) stateDb, err := state.New(stateroot, state.NewDatabaseWithConfig(raw, triedbConfig), nil) Require(t, err) diff --git a/arbos/arbosState/initialize.go b/arbos/arbosState/initialize.go index 3ccc4d3d96..f9f31ed5bd 100644 --- a/arbos/arbosState/initialize.go +++ b/arbos/arbosState/initialize.go @@ -54,7 +54,11 @@ func MakeGenesisBlock(parentHash common.Hash, blockNumber uint64, timestamp uint } func InitializeArbosInDatabase(db ethdb.Database, cacheConfig *core.CacheConfig, initData statetransfer.InitDataReader, chainConfig *params.ChainConfig, initMessage *arbostypes.ParsedInitMessage, timestamp uint64, accountsPerSync uint) (root common.Hash, err error) { - triedbConfig := cacheConfig.TriedbConfig() + number, err := initData.GetNextBlockNumber() + if err != nil { + return common.Hash{}, err + } + triedbConfig := cacheConfig.TriedbConfig(chainConfig.IsVerkle(new(big.Int).SetUint64(number), timestamp)) triedbConfig.Preimages = false stateDatabase := state.NewDatabaseWithConfig(db, triedbConfig) defer func() { diff --git a/arbos/programs/api.go b/arbos/programs/api.go index 231ec2c816..6e136affe9 100644 --- a/arbos/programs/api.go +++ b/arbos/programs/api.go @@ -164,7 +164,7 @@ func newApiClosures( Stack: util.TracingStackFromArgs(args...), Contract: scope.Contract, } - tracingInfo.Tracer.CaptureState(0, opcode, startGas, baseCost+gas, s, []byte{}, depth, nil) + tracingInfo.Tracer.OnOpcode(0, byte(opcode), startGas, baseCost+gas, s, []byte{}, depth, nil) } var ret []byte diff --git a/system_tests/state_fuzz_test.go b/system_tests/state_fuzz_test.go index 713aefcfba..1714478046 100644 --- a/system_tests/state_fuzz_test.go +++ b/system_tests/state_fuzz_test.go @@ -151,7 +151,7 @@ func FuzzStateTransition(f *testing.F) { if err != nil { panic(err) } - trieDBConfig := cacheConfig.TriedbConfig() + trieDBConfig := cacheConfig.TriedbConfig(chainConfig.IsVerkle(new(big.Int), 0)) statedb, err := state.New(stateRoot, state.NewDatabaseWithConfig(chainDb, trieDBConfig), nil) if err != nil { panic(err) From 7fe3d9fed5dea2c4e736732dee716a7afcf37cf6 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Fri, 26 Jul 2024 18:18:53 +0530 Subject: [PATCH 014/489] Support seamless switchover redis for sequencer coordinator --- arbnode/node.go | 2 +- arbnode/seq_coordinator.go | 66 +++++++++++++++++++++++-- system_tests/seq_coordinator_test.go | 74 ++++++++++++++++++++++++++++ util/redisutil/redis_coordinator.go | 1 + 4 files changed, 138 insertions(+), 5 deletions(-) diff --git a/arbnode/node.go b/arbnode/node.go index ac18a6c7d4..78b308387e 100644 --- a/arbnode/node.go +++ b/arbnode/node.go @@ -446,7 +446,7 @@ func createNodeImpl( } if config.SeqCoordinator.Enable { - coordinator, err = NewSeqCoordinator(dataSigner, bpVerifier, txStreamer, exec, syncMonitor, config.SeqCoordinator) + coordinator, err = NewSeqCoordinator(dataSigner, bpVerifier, txStreamer, exec, config.SeqCoordinator) if err != nil { return nil, err } diff --git a/arbnode/seq_coordinator.go b/arbnode/seq_coordinator.go index 98c19ce361..bee1a93704 100644 --- a/arbnode/seq_coordinator.go +++ b/arbnode/seq_coordinator.go @@ -38,6 +38,8 @@ type SeqCoordinator struct { stopwaiter.StopWaiter redisutil.RedisCoordinator + prevRedisCoordinator *redisutil.RedisCoordinator + prevRedisMessageCount arbutil.MessageIndex streamer *TransactionStreamer sequencer execution.ExecutionSequencer @@ -60,6 +62,7 @@ type SeqCoordinatorConfig struct { Enable bool `koanf:"enable"` ChosenHealthcheckAddr string `koanf:"chosen-healthcheck-addr"` RedisUrl string `koanf:"redis-url"` + NewRedisUrl string `koanf:"new-redis-url"` LockoutDuration time.Duration `koanf:"lockout-duration"` LockoutSpare time.Duration `koanf:"lockout-spare"` SeqNumDuration time.Duration `koanf:"seq-num-duration"` @@ -84,6 +87,7 @@ func (c *SeqCoordinatorConfig) Url() string { func SeqCoordinatorConfigAddOptions(prefix string, f *flag.FlagSet) { f.Bool(prefix+".enable", DefaultSeqCoordinatorConfig.Enable, "enable sequence coordinator") f.String(prefix+".redis-url", DefaultSeqCoordinatorConfig.RedisUrl, "the Redis URL to coordinate via") + f.String(prefix+".new-redis-url", DefaultSeqCoordinatorConfig.NewRedisUrl, "switch to the new Redis URL to coordinate via") f.String(prefix+".chosen-healthcheck-addr", DefaultSeqCoordinatorConfig.ChosenHealthcheckAddr, "if non-empty, launch an HTTP service binding to this address that returns status code 200 when chosen and 503 otherwise") f.Duration(prefix+".lockout-duration", DefaultSeqCoordinatorConfig.LockoutDuration, "") f.Duration(prefix+".lockout-spare", DefaultSeqCoordinatorConfig.LockoutSpare, "") @@ -102,6 +106,7 @@ var DefaultSeqCoordinatorConfig = SeqCoordinatorConfig{ Enable: false, ChosenHealthcheckAddr: "", RedisUrl: "", + NewRedisUrl: "", LockoutDuration: time.Minute, LockoutSpare: 30 * time.Second, SeqNumDuration: 24 * time.Hour, @@ -118,6 +123,7 @@ var DefaultSeqCoordinatorConfig = SeqCoordinatorConfig{ var TestSeqCoordinatorConfig = SeqCoordinatorConfig{ Enable: false, RedisUrl: "", + NewRedisUrl: "", LockoutDuration: time.Second * 2, LockoutSpare: time.Millisecond * 10, SeqNumDuration: time.Minute * 10, @@ -136,7 +142,6 @@ func NewSeqCoordinator( bpvalidator *contracts.AddressVerifier, streamer *TransactionStreamer, sequencer execution.ExecutionSequencer, - sync *SyncMonitor, config SeqCoordinatorConfig, ) (*SeqCoordinator, error) { redisCoordinator, err := redisutil.NewRedisCoordinator(config.RedisUrl) @@ -522,12 +527,28 @@ func (c *SeqCoordinator) update(ctx context.Context) time.Duration { log.Error("cannot read message count", "err", err) return c.config.UpdateInterval } + // Cache the previous redis coordinator's message count + if c.prevRedisCoordinator != nil && c.prevRedisMessageCount == 0 { + prevRemoteMsgCount, err := c.getRemoteMsgCountImpl(ctx, c.prevRedisCoordinator.Client) + if err != nil { + log.Warn("cannot get remote message count", "err", err) + return c.retryAfterRedisError() + } + c.prevRedisMessageCount = prevRemoteMsgCount + } remoteMsgCount, err := c.GetRemoteMsgCount() if err != nil { log.Warn("cannot get remote message count", "err", err) return c.retryAfterRedisError() } readUntil := remoteMsgCount + client := c.Client + // If we have a previous redis coordinator, + // we can read from it until the local message count catches up to the prev coordinator's message count + if c.prevRedisMessageCount > localMsgCount { + readUntil = c.prevRedisMessageCount + client = c.prevRedisCoordinator.Client + } if readUntil > localMsgCount+c.config.MsgPerPoll { readUntil = localMsgCount + c.config.MsgPerPoll } @@ -536,7 +557,7 @@ func (c *SeqCoordinator) update(ctx context.Context) time.Duration { var msgReadErr error for msgToRead < readUntil { var resString string - resString, msgReadErr = c.Client.Get(ctx, redisutil.MessageKeyFor(msgToRead)).Result() + resString, msgReadErr = client.Get(ctx, redisutil.MessageKeyFor(msgToRead)).Result() if msgReadErr != nil { log.Warn("coordinator failed reading message", "pos", msgToRead, "err", msgReadErr) break @@ -545,7 +566,7 @@ func (c *SeqCoordinator) update(ctx context.Context) time.Duration { var sigString string var sigBytes []byte sigSeparateKey := true - sigString, msgReadErr = c.Client.Get(ctx, redisutil.MessageSigKeyFor(msgToRead)).Result() + sigString, msgReadErr = client.Get(ctx, redisutil.MessageSigKeyFor(msgToRead)).Result() if errors.Is(msgReadErr, redis.Nil) { // no separate signature. Try reading old-style sig if len(rsBytes) < 32 { @@ -731,12 +752,49 @@ func (c *SeqCoordinator) launchHealthcheckServer(ctx context.Context) { func (c *SeqCoordinator) Start(ctxIn context.Context) { c.StopWaiter.Start(ctxIn, c) - c.CallIteratively(c.update) + c.CallIteratively(c.chooseRedisAndUpdate) if c.config.ChosenHealthcheckAddr != "" { c.StopWaiter.LaunchThread(c.launchHealthcheckServer) } } +func (c *SeqCoordinator) chooseRedisAndUpdate(ctx context.Context) time.Duration { + // If we have a new redis coordinator, and we haven't switched to it yet, try to switch. + if c.config.NewRedisUrl != "" && c.prevRedisCoordinator == nil { + // If we fail to try to switch, we'll retry soon. + if err := c.trySwitchingRedis(ctx); err != nil { + log.Warn("error while trying to switch redis coordinator", "err", err) + return c.retryAfterRedisError() + } + } + return c.update(ctx) +} + +func (c *SeqCoordinator) trySwitchingRedis(ctx context.Context) error { + current, err := c.Client.Get(ctx, redisutil.CHOSENSEQ_KEY).Result() + var wasEmpty bool + if errors.Is(err, redis.Nil) { + wasEmpty = true + err = nil + } + if err != nil { + log.Warn("failed to get current chosen sequencer", "err", err) + return err + } + // If the chosen key is set to switch, we need to switch to the new redis coordinator. + if !wasEmpty && (current == redisutil.SWITCHED_REDIS) { + newRedisCoordinator, err := redisutil.NewRedisCoordinator(c.config.NewRedisUrl) + if err != nil { + log.Warn("failed to create new redis coordinator", "err", + err, "newRedisUrl", c.config.NewRedisUrl) + return err + } + c.prevRedisCoordinator = &c.RedisCoordinator + c.RedisCoordinator = *newRedisCoordinator + } + return nil +} + // Calls check() every c.config.RetryInterval until it returns true, or the context times out. func (c *SeqCoordinator) waitFor(ctx context.Context, check func() bool) bool { for { diff --git a/system_tests/seq_coordinator_test.go b/system_tests/seq_coordinator_test.go index 1b8926a1b9..a93f287546 100644 --- a/system_tests/seq_coordinator_test.go +++ b/system_tests/seq_coordinator_test.go @@ -373,3 +373,77 @@ func TestRedisSeqCoordinatorMessageSync(t *testing.T) { func TestRedisSeqCoordinatorWrongKeyMessageSync(t *testing.T) { testCoordinatorMessageSync(t, false) } + +func TestRedisSwitchover(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, true) + builder.nodeConfig.SeqCoordinator.Enable = true + builder.nodeConfig.SeqCoordinator.RedisUrl = redisutil.CreateTestRedis(ctx, t) + builder.nodeConfig.SeqCoordinator.NewRedisUrl = redisutil.CreateTestRedis(ctx, t) + builder.nodeConfig.BatchPoster.Enable = false + + nodeNames := []string{"stdio://A", "stdio://B"} + initRedisForTest(t, ctx, builder.nodeConfig.SeqCoordinator.RedisUrl, nodeNames) + initRedisForTest(t, ctx, builder.nodeConfig.SeqCoordinator.NewRedisUrl, nodeNames) + builder.nodeConfig.SeqCoordinator.MyUrl = nodeNames[0] + + cleanup := builder.Build(t) + defer cleanup() + + redisClient, err := redisutil.RedisClientFromURL(builder.nodeConfig.SeqCoordinator.RedisUrl) + Require(t, err) + + // wait for sequencerA to become master + for { + err := redisClient.Get(ctx, redisutil.CHOSENSEQ_KEY).Err() + if errors.Is(err, redis.Nil) { + time.Sleep(builder.nodeConfig.SeqCoordinator.UpdateInterval) + continue + } + Require(t, err) + break + } + + builder.L2Info.GenerateAccount("User2") + + nodeConfigDup := *builder.nodeConfig + builder.nodeConfig = &nodeConfigDup + builder.nodeConfig.Feed.Output = *newBroadcasterConfigTest() + builder.nodeConfig.SeqCoordinator.MyUrl = nodeNames[1] + testClientB, cleanupB := builder.Build2ndNode(t, &SecondNodeParams{nodeConfig: builder.nodeConfig}) + defer cleanupB() + + verifyTxIsProcessed(t, ctx, builder, testClientB, 1e12) + + redisClient.Set(ctx, redisutil.CHOSENSEQ_KEY, redisutil.SWITCHED_REDIS, time.Duration(-1)) + + verifyTxIsProcessed(t, ctx, builder, testClientB, 1e12*2) + + // Wait for all messages to be processed, before closing old redisClient + time.Sleep(1 * time.Second) + err = redisClient.Close() + Require(t, err) + + verifyTxIsProcessed(t, ctx, builder, testClientB, 1e12*3) + +} + +func verifyTxIsProcessed(t *testing.T, ctx context.Context, builder *NodeBuilder, testClientB *TestClient, balance int64) { + tx := builder.L2Info.PrepareTx("Owner", "User2", builder.L2Info.TransferGas, big.NewInt(1e12), nil) + + err := builder.L2.Client.SendTransaction(ctx, tx) + Require(t, err) + + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + + _, err = WaitForTx(ctx, testClientB.Client, tx.Hash(), time.Second*5) + Require(t, err) + l2balance, err := testClientB.Client.BalanceAt(ctx, builder.L2Info.GetAddress("User2"), nil) + Require(t, err) + if l2balance.Cmp(big.NewInt(balance)) != 0 { + t.Fatal("Unexpected balance:", l2balance) + } +} diff --git a/util/redisutil/redis_coordinator.go b/util/redisutil/redis_coordinator.go index 59e3b0e0f9..cbb562d576 100644 --- a/util/redisutil/redis_coordinator.go +++ b/util/redisutil/redis_coordinator.go @@ -20,6 +20,7 @@ const WANTS_LOCKOUT_KEY_PREFIX string = "coordinator.liveliness." // Per server. const MESSAGE_KEY_PREFIX string = "coordinator.msg." // Per Message. Only written by sequencer holding CHOSEN const SIGNATURE_KEY_PREFIX string = "coordinator.msg.sig." // Per Message. Only written by sequencer holding CHOSEN const WANTS_LOCKOUT_VAL string = "OK" +const SWITCHED_REDIS string = "SWITCHED_REDIS" const INVALID_VAL string = "INVALID" const INVALID_URL string = "" From e5be704010a6afa5654cb5d9f734c06eafdb7c90 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Mon, 29 Jul 2024 16:28:51 +0530 Subject: [PATCH 015/489] update geth --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 725aecd9bb..660c03975d 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 725aecd9bb4de8eda8a9ab1305c72d3567d2b1fc +Subproject commit 660c03975dedc4ec3e36c3f5835db00165b6fc59 From 203e767c3454c601e447bc82b57a5dfb1ffc0ac5 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Mon, 29 Jul 2024 16:36:43 +0530 Subject: [PATCH 016/489] fix lint --- system_tests/das_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/system_tests/das_test.go b/system_tests/das_test.go index 41a0ce62b1..7670803b3e 100644 --- a/system_tests/das_test.go +++ b/system_tests/das_test.go @@ -31,7 +31,6 @@ import ( "github.com/offchainlabs/nitro/solgen/go/precompilesgen" "github.com/offchainlabs/nitro/util/headerreader" "github.com/offchainlabs/nitro/util/testhelpers" - "golang.org/x/exp/slog" ) func startLocalDASServer( From be0a4bb1aba764c195d65fc3d2d1bbb6a1f5c98d Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Mon, 29 Jul 2024 17:22:36 +0530 Subject: [PATCH 017/489] update geth --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 660c03975d..dd1701edf4 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 660c03975dedc4ec3e36c3f5835db00165b6fc59 +Subproject commit dd1701edf4f000e534eed76965f01f4413b3edc5 From ab6b652ab642313867a163ae98c6354c661b5c19 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Wed, 31 Jul 2024 18:18:12 +0530 Subject: [PATCH 018/489] update geth --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index dd1701edf4..42a71d88fb 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit dd1701edf4f000e534eed76965f01f4413b3edc5 +Subproject commit 42a71d88fb341c9614fd8348b05a56bf5297ccec From d9739f2f63d741439ceb815fcffb8e0e305c2cb3 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Wed, 31 Jul 2024 19:53:21 +0530 Subject: [PATCH 019/489] fix batch poster --- arbnode/batch_poster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arbnode/batch_poster.go b/arbnode/batch_poster.go index 965f6c2695..1c26e13be4 100644 --- a/arbnode/batch_poster.go +++ b/arbnode/batch_poster.go @@ -1077,7 +1077,7 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) } var use4844 bool config := b.config() - if config.Post4844Blobs && b.dapWriter == nil && latestHeader.ExcessBlobGas != nil && latestHeader.BlobGasUsed != nil { + if config.Post4844Blobs && b.dapWriter == nil && latestHeader.ExcessBlobGas != nil && *latestHeader.ExcessBlobGas != 0 && latestHeader.BlobGasUsed != nil && *latestHeader.BlobGasUsed != 0 { arbOSVersion, err := b.arbOSVersionGetter.ArbOSVersionForMessageNumber(arbutil.MessageIndex(arbmath.SaturatingUSub(uint64(batchPosition.MessageCount), 1))) if err != nil { return false, err From bc4b8761fca016a7d042c39daa680cc464e0a944 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Wed, 31 Jul 2024 20:22:44 +0530 Subject: [PATCH 020/489] update geth --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 42a71d88fb..4d6cd19f7f 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 42a71d88fb341c9614fd8348b05a56bf5297ccec +Subproject commit 4d6cd19f7f74f18686826bb6c5fbd0d029742f32 From 16a11eaa805470d66745718fc8e2146587065a6f Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Wed, 31 Jul 2024 20:36:04 +0530 Subject: [PATCH 021/489] update geth --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 4d6cd19f7f..43fe8c893f 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 4d6cd19f7f74f18686826bb6c5fbd0d029742f32 +Subproject commit 43fe8c893f649e161d668d78ec93eb56c3f14a7c From 24560120d2bb78c1451ed00917a9c1581c14b163 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Wed, 31 Jul 2024 21:50:14 +0530 Subject: [PATCH 022/489] fix null reference --- arbos/programs/api.go | 10 ++++++---- arbos/tx_processor.go | 8 ++++++-- arbos/util/tracing.go | 32 ++++++++++++++++++++++++-------- go-ethereum | 2 +- 4 files changed, 37 insertions(+), 15 deletions(-) diff --git a/arbos/programs/api.go b/arbos/programs/api.go index 6e136affe9..161789ece6 100644 --- a/arbos/programs/api.go +++ b/arbos/programs/api.go @@ -138,7 +138,7 @@ func newApiClosures( gas := am.MinInt(startGas, gasReq) // Tracing: emit the call (value transfer is done later in evm.Call) - if tracingInfo != nil { + if tracingInfo != nil && tracingInfo.Tracer.OnOpcode != nil { tracingInfo.Tracer.OnOpcode(0, byte(opcode), startGas, baseCost+gas, scope, []byte{}, depth, nil) } @@ -164,7 +164,9 @@ func newApiClosures( Stack: util.TracingStackFromArgs(args...), Contract: scope.Contract, } - tracingInfo.Tracer.OnOpcode(0, byte(opcode), startGas, baseCost+gas, s, []byte{}, depth, nil) + if tracingInfo.Tracer.OnOpcode != nil { + tracingInfo.Tracer.OnOpcode(0, byte(opcode), startGas, baseCost+gas, s, []byte{}, depth, nil) + } } var ret []byte @@ -223,7 +225,7 @@ func newApiClosures( gas -= one64th // Tracing: emit the create - if tracingInfo != nil { + if tracingInfo != nil && tracingInfo.Tracer.OnOpcode != nil { tracingInfo.Tracer.OnOpcode(0, byte(opcode), startGas, baseCost+gas, scope, []byte{}, depth, nil) } @@ -289,7 +291,7 @@ func newApiClosures( } captureHostio := func(name string, args, outs []byte, startInk, endInk uint64) { tracingInfo.Tracer.CaptureStylusHostio(name, args, outs, startInk, endInk) - if name == "evm_gas_left" || name == "evm_ink_left" { + if (name == "evm_gas_left" || name == "evm_ink_left") && tracingInfo.Tracer.OnOpcode != nil { tracingInfo.Tracer.OnOpcode(0, byte(vm.GAS), 0, 0, scope, []byte{}, depth, nil) tracingInfo.Tracer.OnOpcode(0, byte(vm.POP), 0, 0, scope, []byte{}, depth, nil) } diff --git a/arbos/tx_processor.go b/arbos/tx_processor.go index a7abbed6d5..1708291db8 100644 --- a/arbos/tx_processor.go +++ b/arbos/tx_processor.go @@ -153,13 +153,17 @@ func (p *TxProcessor) StartTxHook() (endTxNow bool, gasUsed uint64, err error, r } evm.IncrementDepth() // fake a call from := p.msg.From - tracer.OnEnter(evm.Depth(), byte(vm.CALL), from, *p.msg.To, p.msg.Data, p.msg.GasLimit, p.msg.Value) + if tracer.OnEnter != nil { + tracer.OnEnter(evm.Depth(), byte(vm.CALL), from, *p.msg.To, p.msg.Data, p.msg.GasLimit, p.msg.Value) + } tracingInfo = util.NewTracingInfo(evm, from, *p.msg.To, util.TracingDuringEVM) p.state = arbosState.OpenSystemArbosStateOrPanic(evm.StateDB, tracingInfo, false) return func() { - tracer.OnExit(evm.Depth(), nil, p.state.Burner.Burned(), nil, false) + if tracer.OnExit != nil { + tracer.OnExit(evm.Depth(), nil, p.state.Burner.Burned(), nil, false) + } evm.DecrementDepth() // fake the return to the first faked call tracingInfo = util.NewTracingInfo(evm, from, *p.msg.To, util.TracingAfterEVM) diff --git a/arbos/util/tracing.go b/arbos/util/tracing.go index 4801cc57d2..b62a55b8bc 100644 --- a/arbos/util/tracing.go +++ b/arbos/util/tracing.go @@ -63,7 +63,9 @@ func (info *TracingInfo) RecordEmitLog(topics []common.Hash, data []byte) { Contract: info.Contract, } logType := fmt.Sprintf("LOG%d", len(topics)) - info.Tracer.OnOpcode(0, byte(vm.StringToOp(logType)), 0, 0, scope, []byte{}, info.Depth, nil) + if info.Tracer.OnOpcode != nil { + info.Tracer.OnOpcode(0, byte(vm.StringToOp(logType)), 0, 0, scope, []byte{}, info.Depth, nil) + } } func (info *TracingInfo) RecordStorageGet(key common.Hash) { @@ -74,7 +76,9 @@ func (info *TracingInfo) RecordStorageGet(key common.Hash) { Stack: TracingStackFromArgs(HashToUint256(key)), Contract: info.Contract, } - tracer.OnOpcode(0, byte(vm.SLOAD), 0, 0, scope, []byte{}, info.Depth, nil) + if tracer.OnOpcode != nil { + tracer.OnOpcode(0, byte(vm.SLOAD), 0, 0, scope, []byte{}, info.Depth, nil) + } } else { tracer.CaptureArbitrumStorageGet(key, info.Depth, info.Scenario == TracingBeforeEVM) } @@ -88,7 +92,9 @@ func (info *TracingInfo) RecordStorageSet(key, value common.Hash) { Stack: TracingStackFromArgs(HashToUint256(key), HashToUint256(value)), Contract: info.Contract, } - tracer.OnOpcode(0, byte(vm.SSTORE), 0, 0, scope, []byte{}, info.Depth, nil) + if tracer.OnOpcode != nil { + tracer.OnOpcode(0, byte(vm.SSTORE), 0, 0, scope, []byte{}, info.Depth, nil) + } } else { tracer.CaptureArbitrumStorageSet(key, value, info.Depth, info.Scenario == TracingBeforeEVM) } @@ -113,8 +119,12 @@ func (info *TracingInfo) MockCall(input []byte, gas uint64, from, to common.Addr ), Contract: contract, } - tracer.OnOpcode(0, byte(vm.CALL), 0, 0, scope, []byte{}, depth, nil) - tracer.OnEnter(depth, byte(vm.INVALID), from, to, input, 0, amount) + if tracer.OnOpcode != nil { + tracer.OnOpcode(0, byte(vm.CALL), 0, 0, scope, []byte{}, depth, nil) + } + if tracer.OnEnter != nil { + tracer.OnEnter(depth, byte(vm.CALL), from, to, input, gas, amount) + } retScope := &vm.ScopeContext{ Memory: vm.NewMemory(), @@ -124,8 +134,12 @@ func (info *TracingInfo) MockCall(input []byte, gas uint64, from, to common.Addr ), Contract: contract, } - tracer.OnOpcode(0, byte(vm.RETURN), 0, 0, retScope, []byte{}, depth+1, nil) - tracer.OnExit(depth+1, nil, 0, nil, false) + if tracer.OnOpcode != nil { + tracer.OnOpcode(0, byte(vm.RETURN), 0, 0, retScope, []byte{}, depth+1, nil) + } + if tracer.OnExit != nil { + tracer.OnExit(depth+1, nil, 0, nil, false) + } popScope := &vm.ScopeContext{ Memory: vm.NewMemory(), @@ -134,7 +148,9 @@ func (info *TracingInfo) MockCall(input []byte, gas uint64, from, to common.Addr ), Contract: contract, } - tracer.OnOpcode(0, byte(vm.POP), 0, 0, popScope, []byte{}, depth, nil) + if tracer.OnOpcode != nil { + tracer.OnOpcode(0, byte(vm.POP), 0, 0, popScope, []byte{}, depth, nil) + } } func HashToUint256(hash common.Hash) uint256.Int { diff --git a/go-ethereum b/go-ethereum index 43fe8c893f..dd1701edf4 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 43fe8c893f649e161d668d78ec93eb56c3f14a7c +Subproject commit dd1701edf4f000e534eed76965f01f4413b3edc5 From 15834ae164b5d999a9daf8f6a326796aafc0dcfc Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Thu, 1 Aug 2024 15:47:38 +0530 Subject: [PATCH 023/489] fix test --- execution/nodeInterface/NodeInterface.go | 10 ++++++---- go-ethereum | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/execution/nodeInterface/NodeInterface.go b/execution/nodeInterface/NodeInterface.go index 6fe1b2ef99..33a2abe670 100644 --- a/execution/nodeInterface/NodeInterface.go +++ b/execution/nodeInterface/NodeInterface.go @@ -516,10 +516,11 @@ func (n NodeInterface) GasEstimateL1Component( args.Gas = (*hexutil.Uint64)(&randomGas) // We set the run mode to eth_call mode here because we want an exact estimate, not a padded estimate - if err := args.CallDefaults(randomGas, evm.Context.BaseFee, evm.ChainConfig().ChainID); err != nil { + gasLimitNotSetByUser := args.Gas == nil + if err := args.CallDefaults(randomGas, evm.Context.BaseFee, evm.ChainConfig().ChainID, gasLimitNotSetByUser); err != nil { return 0, nil, nil, err } - msg := args.ToMessage(evm.Context.BaseFee, randomGas, n.header, evm.StateDB.(*state.StateDB), core.MessageEthcallMode) + msg := args.ToMessage(evm.Context.BaseFee, randomGas, n.header, evm.StateDB.(*state.StateDB), core.MessageEthcallMode, evm.ChainConfig().ChainID, gasLimitNotSetByUser) pricing := c.State.L1PricingState() l1BaseFeeEstimate, err := pricing.PricePerUnit() @@ -572,10 +573,11 @@ func (n NodeInterface) GasEstimateComponents( // Setting the gas currently doesn't affect the PosterDataCost, // but we do it anyways for accuracy with potential future changes. args.Gas = &totalRaw - if err := args.CallDefaults(gasCap, evm.Context.BaseFee, evm.ChainConfig().ChainID); err != nil { + gasLimitNotSetByUser := args.Gas == nil + if err := args.CallDefaults(gasCap, evm.Context.BaseFee, evm.ChainConfig().ChainID, gasLimitNotSetByUser); err != nil { return 0, 0, nil, nil, err } - msg := args.ToMessage(evm.Context.BaseFee, gasCap, n.header, evm.StateDB.(*state.StateDB), core.MessageGasEstimationMode) + msg := args.ToMessage(evm.Context.BaseFee, gasCap, n.header, evm.StateDB.(*state.StateDB), core.MessageGasEstimationMode, evm.ChainConfig().ChainID, gasLimitNotSetByUser) brotliCompressionLevel, err := c.State.BrotliCompressionLevel() if err != nil { return 0, 0, nil, nil, fmt.Errorf("failed to get brotli compression level: %w", err) diff --git a/go-ethereum b/go-ethereum index dd1701edf4..1793a3be33 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit dd1701edf4f000e534eed76965f01f4413b3edc5 +Subproject commit 1793a3be33cacd8610055a982d82bd062737a64b From c6ad8fabd220fc175ded93870ef39ed8014cbc49 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Thu, 1 Aug 2024 16:17:34 +0530 Subject: [PATCH 024/489] fix test --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 1793a3be33..cbecc2f9ee 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 1793a3be33cacd8610055a982d82bd062737a64b +Subproject commit cbecc2f9eeeade5d31bd6846ab24ae7f8b21d0f7 From ed5f8cdef2721b57e76d7d2c939a113bc1ae8d90 Mon Sep 17 00:00:00 2001 From: Tristan Wilson Date: Fri, 2 Aug 2024 12:20:36 +0200 Subject: [PATCH 025/489] DAS RPC Client Metrics This PR adds metrics to the client side of DAS batch posting (eg on the Nitro batch poster). The metrics are designed to mirror the metrics on the daserver side. There are combined metrics for both legacy and chunked Store methods: arb_das_rpcclient_store_requests: Count of initiated batch stores arb_das_rpcclient_store_success: Count of successful batch stores arb_das_rpcclient_store_failure: Count of failed batch stores arb_das_rpcclient_store_bytes: Bytes stored arb_das_rpcclient_store_duration: Histogram of total duration of batch stores (ns) There are metrics specific for chunked Store requests: arb_das_rpcclient_sendchunk_success: Count of chunks sent successfully arb_das_rpcclieny_sendchunk_failure: Count of chunks that failed --- das/dasRpcClient.go | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/das/dasRpcClient.go b/das/dasRpcClient.go index ca2ee8e7d4..ace975079c 100644 --- a/das/dasRpcClient.go +++ b/das/dasRpcClient.go @@ -12,6 +12,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "golang.org/x/sync/errgroup" "github.com/ethereum/go-ethereum/rpc" @@ -21,6 +22,17 @@ import ( "github.com/offchainlabs/nitro/util/signature" ) +var ( + rpcClientStoreRequestGauge = metrics.NewRegisteredGauge("arb/das/rpcclient/store/requests", nil) + rpcClientStoreSuccessGauge = metrics.NewRegisteredGauge("arb/das/rpcclient/store/success", nil) + rpcClientStoreFailureGauge = metrics.NewRegisteredGauge("arb/das/rpcclient/store/failure", nil) + rpcClientStoreStoredBytesGauge = metrics.NewRegisteredGauge("arb/das/rpcclient/store/bytes", nil) + rpcClientStoreDurationHistogram = metrics.NewRegisteredHistogram("arb/das/rpcclient/store/duration", nil, metrics.NewBoundedHistogramSample()) + + rpcClientSendChunkSuccessGauge = metrics.NewRegisteredGauge("arb/das/rpcclient/sendchunk/success", nil) + rpcClientSendChunkFailureGauge = metrics.NewRegisteredGauge("arb/das/rpcclient/sendchunk/failure", nil) +) + type DASRPCClient struct { // implements DataAvailabilityService clnt *rpc.Client url string @@ -58,7 +70,19 @@ func NewDASRPCClient(target string, signer signature.DataSignerFunc, maxStoreChu } func (c *DASRPCClient) Store(ctx context.Context, message []byte, timeout uint64) (*daprovider.DataAvailabilityCertificate, error) { - timestamp := uint64(time.Now().Unix()) + rpcClientStoreRequestGauge.Inc(1) + start := time.Now() + success := false + defer func() { + if success { + rpcClientStoreSuccessGauge.Inc(1) + } else { + rpcClientStoreFailureGauge.Inc(1) + } + rpcClientStoreDurationHistogram.Update(time.Since(start).Nanoseconds()) + }() + + timestamp := uint64(start.Unix()) nChunks := uint64(len(message)) / c.chunkSize lastChunkSize := uint64(len(message)) % c.chunkSize if lastChunkSize > 0 { @@ -115,6 +139,9 @@ func (c *DASRPCClient) Store(ctx context.Context, message []byte, timeout uint64 return nil, err } + rpcClientStoreStoredBytesGauge.Inc(int64(len(message))) + success = true + return &daprovider.DataAvailabilityCertificate{ DataHash: common.BytesToHash(storeResult.DataHash), Timeout: uint64(storeResult.Timeout), @@ -132,8 +159,10 @@ func (c *DASRPCClient) sendChunk(ctx context.Context, batchId, i uint64, chunk [ } if err := c.clnt.CallContext(ctx, nil, "das_sendChunk", hexutil.Uint64(batchId), hexutil.Uint64(i), hexutil.Bytes(chunk), hexutil.Bytes(chunkReqSig)); err != nil { + rpcClientSendChunkFailureGauge.Inc(1) return err } + rpcClientSendChunkSuccessGauge.Inc(1) return nil } From 49d28d81fb2465ae3ac2cb676413deb5731ea8fc Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Mon, 5 Aug 2024 07:38:11 +0530 Subject: [PATCH 026/489] update geth --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index cbecc2f9ee..4b4742a134 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit cbecc2f9eeeade5d31bd6846ab24ae7f8b21d0f7 +Subproject commit 4b4742a13481b7e22c86b1258115d565e5d237ca From 6ec1de8f104525abe5c50b1feab4dbd7f85b57f4 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Tue, 6 Aug 2024 21:27:10 +0530 Subject: [PATCH 027/489] Changes based on PR comments --- arbnode/seq_coordinator.go | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/arbnode/seq_coordinator.go b/arbnode/seq_coordinator.go index bee1a93704..431ea43512 100644 --- a/arbnode/seq_coordinator.go +++ b/arbnode/seq_coordinator.go @@ -358,20 +358,20 @@ func (c *SeqCoordinator) GetRemoteMsgCount() (arbutil.MessageIndex, error) { return c.getRemoteMsgCountImpl(c.GetContext(), c.Client) } -func (c *SeqCoordinator) wantsLockoutUpdate(ctx context.Context) error { +func (c *SeqCoordinator) wantsLockoutUpdate(ctx context.Context, client redis.UniversalClient) error { c.wantsLockoutMutex.Lock() defer c.wantsLockoutMutex.Unlock() - return c.wantsLockoutUpdateWithMutex(ctx) + return c.wantsLockoutUpdateWithMutex(ctx, client) } // Requires the caller hold the wantsLockoutMutex -func (c *SeqCoordinator) wantsLockoutUpdateWithMutex(ctx context.Context) error { +func (c *SeqCoordinator) wantsLockoutUpdateWithMutex(ctx context.Context, client redis.UniversalClient) error { if c.avoidLockout > 0 { return nil } myWantsLockoutKey := redisutil.WantsLockoutKeyFor(c.config.Url()) wantsLockoutUntil := time.Now().Add(c.config.LockoutDuration) - pipe := c.Client.TxPipeline() + pipe := client.TxPipeline() initialDuration := c.config.LockoutDuration if initialDuration < 2*time.Second { initialDuration = 2 * time.Second @@ -657,7 +657,7 @@ func (c *SeqCoordinator) update(ctx context.Context) time.Duration { // this could be just new messages we didn't get yet - even then, we should retry soon log.Info("sequencer failed to become chosen", "err", err, "msgcount", localMsgCount) // make sure we're marked as wanting the lockout - if err := c.wantsLockoutUpdate(ctx); err != nil { + if err := c.wantsLockoutUpdate(ctx, c.Client); err != nil { log.Warn("failed to update wants lockout key", "err", err) } c.prevChosenSequencer = "" @@ -685,7 +685,7 @@ func (c *SeqCoordinator) update(ctx context.Context) time.Duration { // update wanting the lockout var wantsLockoutErr error if synced && !c.AvoidingLockout() { - wantsLockoutErr = c.wantsLockoutUpdate(ctx) + wantsLockoutErr = c.wantsLockoutUpdate(ctx, c.Client) } else { wantsLockoutErr = c.wantsLockoutRelease(ctx) } @@ -789,6 +789,14 @@ func (c *SeqCoordinator) trySwitchingRedis(ctx context.Context) error { err, "newRedisUrl", c.config.NewRedisUrl) return err } + err = c.wantsLockoutUpdate(ctx, c.Client) + if err != nil { + return err + } + err = c.wantsLockoutUpdate(ctx, newRedisCoordinator.Client) + if err != nil { + return err + } c.prevRedisCoordinator = &c.RedisCoordinator c.RedisCoordinator = *newRedisCoordinator } @@ -908,7 +916,7 @@ func (c *SeqCoordinator) SeekLockout(ctx context.Context) { log.Info("seeking lockout", "myUrl", c.config.Url()) if c.sequencer.Synced() { // Even if this errors we still internally marked ourselves as wanting the lockout - err := c.wantsLockoutUpdateWithMutex(ctx) + err := c.wantsLockoutUpdateWithMutex(ctx, c.Client) if err != nil { log.Warn("failed to set wants lockout key in redis after seeking lockout again", "err", err) } From 2a403985792aba2721e0649671e21b0c2672aa19 Mon Sep 17 00:00:00 2001 From: Pepper Lebeck-Jobe Date: Wed, 7 Aug 2024 17:50:06 +0200 Subject: [PATCH 028/489] Support simpler capture and replay of ValidatorInputs There are several changes in this commit. 1. Move the `parse_inputs.rs` and `prepare.rs` files from the benchbin package to the prover. This enables the prover to ingest most of the information it needs from the `block_inputs_.json` file. 2. Add a `--json-inputs` flag to the `prover` bin which obviates several competing flags preferring to read the data from the json. 3. Write the json file to disk in the working directory before the arbitrator spawner writes the `run_prover.sh` file which is attempting to make a similar validaiton easy to reproduce. 4. Introduce a new `recordBlock` method to the tests suite which can be called once a block has been recorded to L1. 5. Wire the call to `recordBlock` into the `TestProgramStorage` to be sure to capture a block that actually uses a Stylus contract so that the prover can be shown to work with that block as well. 6. Enhance the `parse_inputs.rs` to know how to handle the `UserWasms` deserialization. To reproduce the failure state at this commit: 1. make clean && make all 2. go test -timeout 10m -tags challengetest -run ^TestProgramStorage$ github.com/offchainlabs/nitro/system_tests 3. Observe, there are now two new files in `ls system_tests/block_inputs_{2,5}.json` 4. target/bin/prover target/machines/latest/machine.wavm.br -b --json-inputs system_tests/block_inputs_5.json 5. The error is: ``` thread 'main' panicked at prover/src/machine.rs:690:58: unknown dictionary: TryFromPrimitiveError { number: 10 } ``` If you make changes to the rust code, you can quickly rebuild and install with: ``` cargo build --manifest-path arbitrator/Cargo.toml --release --bin prover && install arbitrator/target/release/prover target/bin/prover ``` --- arbitrator/Cargo.lock | 78 +---- arbitrator/arbutil/src/types.rs | 16 ++ arbitrator/bench/Cargo.toml | 1 - arbitrator/bench/src/bin.rs | 6 +- arbitrator/bench/src/lib.rs | 2 - arbitrator/prover/Cargo.toml | 2 +- arbitrator/prover/src/lib.rs | 2 + arbitrator/prover/src/main.rs | 167 ++++++----- .../{bench => prover}/src/parse_input.rs | 29 +- arbitrator/{bench => prover}/src/prepare.rs | 18 +- arbitrator/wasm-libraries/Cargo.lock | 266 +++++++++++++++--- staker/block_validator.go | 6 + staker/stateless_block_validator.go | 25 ++ system_tests/program_norace_test.go | 13 + system_tests/program_test.go | 8 +- validator/client/validation_client.go | 5 - validator/server_arb/validator_spawner.go | 6 + 17 files changed, 440 insertions(+), 210 deletions(-) delete mode 100644 arbitrator/bench/src/lib.rs rename arbitrator/{bench => prover}/src/parse_input.rs (74%) rename arbitrator/{bench => prover}/src/prepare.rs (73%) diff --git a/arbitrator/Cargo.lock b/arbitrator/Cargo.lock index 79a9117a31..2aa9864b08 100644 --- a/arbitrator/Cargo.lock +++ b/arbitrator/Cargo.lock @@ -215,7 +215,6 @@ dependencies = [ "prover", "serde", "serde_json", - "serde_with 3.9.0", ] [[package]] @@ -705,38 +704,14 @@ dependencies = [ "typenum", ] -[[package]] -name = "darling" -version = "0.13.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" -dependencies = [ - "darling_core 0.13.4", - "darling_macro 0.13.4", -] - [[package]] name = "darling" version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" dependencies = [ - "darling_core 0.20.10", - "darling_macro 0.20.10", -] - -[[package]] -name = "darling_core" -version = "0.13.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim 0.10.0", - "syn 1.0.109", + "darling_core", + "darling_macro", ] [[package]] @@ -753,24 +728,13 @@ dependencies = [ "syn 2.0.72", ] -[[package]] -name = "darling_macro" -version = "0.13.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" -dependencies = [ - "darling_core 0.13.4", - "quote", - "syn 1.0.109", -] - [[package]] name = "darling_macro" version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ - "darling_core 0.20.10", + "darling_core", "quote", "syn 2.0.72", ] @@ -928,7 +892,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59c3b24c345d8c314966bdc1832f6c2635bfcce8e7cf363bd115987bba2ee242" dependencies = [ - "darling 0.20.10", + "darling", "proc-macro2", "quote", "syn 2.0.72", @@ -1750,7 +1714,7 @@ dependencies = [ "rustc-demangle", "serde", "serde_json", - "serde_with 1.14.0", + "serde_with", "sha2 0.9.9", "sha3 0.9.1", "smallvec", @@ -2073,16 +2037,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_with" -version = "1.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678b5a069e50bf00ecd22d0cd8ddf7c236f68581b03db652061ed5eb13a312ff" -dependencies = [ - "serde", - "serde_with_macros 1.5.2", -] - [[package]] name = "serde_with" version = "3.9.0" @@ -2097,29 +2051,17 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "serde_with_macros 3.9.0", + "serde_with_macros", "time", ] -[[package]] -name = "serde_with_macros" -version = "1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" -dependencies = [ - "darling 0.13.4", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "serde_with_macros" version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8fee4991ef4f274617a51ad4af30519438dacb2f56ac773b08a1922ff743350" dependencies = [ - "darling 0.20.10", + "darling", "proc-macro2", "quote", "syn 2.0.72", @@ -2226,12 +2168,6 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" -[[package]] -name = "strsim" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" - [[package]] name = "strsim" version = "0.11.1" diff --git a/arbitrator/arbutil/src/types.rs b/arbitrator/arbutil/src/types.rs index 6cf1d6cdf7..207b8ab949 100644 --- a/arbitrator/arbutil/src/types.rs +++ b/arbitrator/arbutil/src/types.rs @@ -8,6 +8,7 @@ use std::{ borrow::Borrow, fmt, ops::{Deref, DerefMut}, + str::FromStr, }; // These values must be kept in sync with `arbutil/preimage_type.go`, @@ -83,6 +84,21 @@ impl From for Bytes32 { } } +impl FromStr for Bytes32 { + type Err = hex::FromHexError; + + fn from_str(s: &str) -> Result { + let trimmed = match s.strip_prefix("0x") { + Some(t) => t, + None => s, + }; + let bytes = hex::decode(trimmed)?; + let mut b = [0u8; 32]; + b.copy_from_slice(&bytes); + Ok(Self(b)) + } +} + impl TryFrom<&[u8]> for Bytes32 { type Error = std::array::TryFromSliceError; diff --git a/arbitrator/bench/Cargo.toml b/arbitrator/bench/Cargo.toml index 3ab5b99b08..284180dc46 100644 --- a/arbitrator/bench/Cargo.toml +++ b/arbitrator/bench/Cargo.toml @@ -20,7 +20,6 @@ clap = { version = "4.4.8", features = ["derive"] } gperftools = { version = "0.2.0", optional = true } serde = { version = "1.0.130", features = ["derive", "rc"] } serde_json = "1.0.67" -serde_with = { version = "3.8.1", features = ["base64"] } [features] counters = [] diff --git a/arbitrator/bench/src/bin.rs b/arbitrator/bench/src/bin.rs index f7e69f5373..60a7036e2b 100644 --- a/arbitrator/bench/src/bin.rs +++ b/arbitrator/bench/src/bin.rs @@ -1,6 +1,5 @@ use std::{path::PathBuf, time::Duration}; -use bench::prepare::*; use clap::Parser; use eyre::bail; @@ -10,11 +9,12 @@ use gperftools::profiler::PROFILER; #[cfg(feature = "heapprof")] use gperftools::heap_profiler::HEAP_PROFILER; -use prover::machine::MachineStatus; - #[cfg(feature = "counters")] use prover::{machine, memory, merkle}; +use prover::machine::MachineStatus; +use prover::prepare::prepare_machine; + #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] struct Args { diff --git a/arbitrator/bench/src/lib.rs b/arbitrator/bench/src/lib.rs deleted file mode 100644 index 5f7c024094..0000000000 --- a/arbitrator/bench/src/lib.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod parse_input; -pub mod prepare; diff --git a/arbitrator/prover/Cargo.toml b/arbitrator/prover/Cargo.toml index 5475647765..da329b1cb5 100644 --- a/arbitrator/prover/Cargo.toml +++ b/arbitrator/prover/Cargo.toml @@ -19,10 +19,10 @@ num = "0.4" rustc-demangle = "0.1.21" serde = { version = "1.0.130", features = ["derive", "rc"] } serde_json = "1.0.67" +serde_with = { version = "3.8.1", features = ["base64"] } sha3 = "0.9.1" static_assertions = "1.1.0" structopt = "0.3.23" -serde_with = "1.12.1" parking_lot = "0.12.1" lazy_static.workspace = true itertools = "0.10.5" diff --git a/arbitrator/prover/src/lib.rs b/arbitrator/prover/src/lib.rs index 0f537478eb..08473c2598 100644 --- a/arbitrator/prover/src/lib.rs +++ b/arbitrator/prover/src/lib.rs @@ -11,6 +11,8 @@ pub mod machine; /// cbindgen:ignore pub mod memory; pub mod merkle; +pub mod parse_input; +pub mod prepare; mod print; pub mod programs; mod reinterpret; diff --git a/arbitrator/prover/src/main.rs b/arbitrator/prover/src/main.rs index dba32e0e72..002b693bd9 100644 --- a/arbitrator/prover/src/main.rs +++ b/arbitrator/prover/src/main.rs @@ -8,6 +8,7 @@ use eyre::{eyre, Context, Result}; use fnv::{FnvHashMap as HashMap, FnvHashSet as HashSet}; use prover::{ machine::{GlobalState, InboxIdentifier, Machine, MachineStatus, PreimageResolver, ProofInfo}, + prepare::prepare_machine, utils::{file_bytes, hash_preimage, CBytes}, wavm::Opcode, }; @@ -86,6 +87,10 @@ struct Opts { skip_until_host_io: bool, #[structopt(long)] max_steps: Option, + // JSON inputs are supercede any of the command-line inputs which could + // be specified in the JSON file. + #[structopt(long)] + json_inputs: Option, } fn file_with_stub_header(path: &Path, headerlength: usize) -> Result> { @@ -135,83 +140,8 @@ fn main() -> Result<()> { } } } - let mut inbox_contents = HashMap::default(); - let mut inbox_position = opts.inbox_position; - let mut delayed_position = opts.delayed_inbox_position; - let inbox_header_len; - let delayed_header_len; - if opts.inbox_add_stub_headers { - inbox_header_len = INBOX_HEADER_LEN; - delayed_header_len = DELAYED_HEADER_LEN + 1; - } else { - inbox_header_len = 0; - delayed_header_len = 0; - } - - for path in opts.inbox { - inbox_contents.insert( - (InboxIdentifier::Sequencer, inbox_position), - file_with_stub_header(&path, inbox_header_len)?, - ); - println!("read file {:?} to seq. inbox {}", &path, inbox_position); - inbox_position += 1; - } - for path in opts.delayed_inbox { - inbox_contents.insert( - (InboxIdentifier::Delayed, delayed_position), - file_with_stub_header(&path, delayed_header_len)?, - ); - delayed_position += 1; - } - let mut preimages: HashMap> = HashMap::default(); - if let Some(path) = opts.preimages { - let mut file = BufReader::new(File::open(path)?); - loop { - let mut ty_buf = [0u8; 1]; - match file.read_exact(&mut ty_buf) { - Ok(()) => {} - Err(e) if e.kind() == ErrorKind::UnexpectedEof => break, - Err(e) => return Err(e.into()), - } - let preimage_ty: PreimageType = ty_buf[0].try_into()?; - - let mut size_buf = [0u8; 8]; - file.read_exact(&mut size_buf)?; - let size = u64::from_le_bytes(size_buf) as usize; - let mut buf = vec![0u8; size]; - file.read_exact(&mut buf)?; - - let hash = hash_preimage(&buf, preimage_ty)?; - preimages - .entry(preimage_ty) - .or_default() - .insert(hash.into(), buf.as_slice().into()); - } - } - let preimage_resolver = - Arc::new(move |_, ty, hash| preimages.get(&ty).and_then(|m| m.get(&hash)).cloned()) - as PreimageResolver; - - let last_block_hash = decode_hex_arg(&opts.last_block_hash, "--last-block-hash")?; - let last_send_root = decode_hex_arg(&opts.last_send_root, "--last-send-root")?; - - let global_state = GlobalState { - u64_vals: [opts.inbox_position, opts.position_within_message], - bytes32_vals: [last_block_hash, last_send_root], - }; - - let mut mach = Machine::from_paths( - &opts.libraries, - &opts.binary, - true, - opts.allow_hostapi, - opts.debug_funcs, - true, - global_state, - inbox_contents, - preimage_resolver, - )?; + let mut mach = initialize_machine(&opts)?; for path in &opts.stylus_modules { let err = || eyre!("failed to read module at {}", path.to_string_lossy().red()); @@ -462,7 +392,6 @@ fn main() -> Result<()> { } } } - let opts_binary = opts.binary; let opts_libraries = opts.libraries; let format_pc = |module_num: usize, func_num: usize| -> (String, String) { @@ -543,3 +472,87 @@ fn main() -> Result<()> { } Ok(()) } + +fn initialize_machine(opts: &Opts) -> eyre::Result { + if let Some(json_inputs) = opts.json_inputs.clone() { + prepare_machine(json_inputs, opts.binary.clone()) + } else { + let mut inbox_contents = HashMap::default(); + let mut inbox_position = opts.inbox_position; + let mut delayed_position = opts.delayed_inbox_position; + let inbox_header_len; + let delayed_header_len; + if opts.inbox_add_stub_headers { + inbox_header_len = INBOX_HEADER_LEN; + delayed_header_len = DELAYED_HEADER_LEN + 1; + } else { + inbox_header_len = 0; + delayed_header_len = 0; + } + + for path in opts.inbox.clone() { + inbox_contents.insert( + (InboxIdentifier::Sequencer, inbox_position), + file_with_stub_header(&path, inbox_header_len)?, + ); + println!("read file {:?} to seq. inbox {}", &path, inbox_position); + inbox_position += 1; + } + for path in opts.delayed_inbox.clone() { + inbox_contents.insert( + (InboxIdentifier::Delayed, delayed_position), + file_with_stub_header(&path, delayed_header_len)?, + ); + delayed_position += 1; + } + + let mut preimages: HashMap> = HashMap::default(); + if let Some(path) = opts.preimages.clone() { + let mut file = BufReader::new(File::open(path)?); + loop { + let mut ty_buf = [0u8; 1]; + match file.read_exact(&mut ty_buf) { + Ok(()) => {} + Err(e) if e.kind() == ErrorKind::UnexpectedEof => break, + Err(e) => return Err(e.into()), + } + let preimage_ty: PreimageType = ty_buf[0].try_into()?; + + let mut size_buf = [0u8; 8]; + file.read_exact(&mut size_buf)?; + let size = u64::from_le_bytes(size_buf) as usize; + let mut buf = vec![0u8; size]; + file.read_exact(&mut buf)?; + + let hash = hash_preimage(&buf, preimage_ty)?; + preimages + .entry(preimage_ty) + .or_default() + .insert(hash.into(), buf.as_slice().into()); + } + } + let preimage_resolver = + Arc::new(move |_, ty, hash| preimages.get(&ty).and_then(|m| m.get(&hash)).cloned()) + as PreimageResolver; + + let last_block_hash = decode_hex_arg(&opts.last_block_hash, "--last-block-hash")?; + let last_send_root = decode_hex_arg(&opts.last_send_root, "--last-send-root")?; + + let global_state = GlobalState { + u64_vals: [opts.inbox_position, opts.position_within_message], + bytes32_vals: [last_block_hash, last_send_root], + }; + + Machine::from_paths( + &opts.libraries, + &opts.binary, + true, + opts.allow_hostapi, + opts.debug_funcs, + true, + global_state, + inbox_contents, + preimage_resolver, + ) + } +} diff --git a/arbitrator/bench/src/parse_input.rs b/arbitrator/prover/src/parse_input.rs similarity index 74% rename from arbitrator/bench/src/parse_input.rs rename to arbitrator/prover/src/parse_input.rs index decc67372a..9af8652c96 100644 --- a/arbitrator/bench/src/parse_input.rs +++ b/arbitrator/prover/src/parse_input.rs @@ -35,6 +35,31 @@ mod prefixed_hex { #[derive(Debug, Clone, Deserialize, Serialize)] pub struct PreimageMap(HashMap>); +#[derive(Debug)] +pub struct UserWasm { + data: Vec, +} + +impl UserWasm { + pub fn as_vec(&self) -> Vec { + self.data.clone() + } +} + +impl AsRef<[u8]> for UserWasm { + fn as_ref(&self) -> &[u8] { + &self.data + } +} + +/// The Vec is compressed using brotli, and must be decompressed before use. +impl From> for UserWasm { + fn from(data: Vec) -> Self { + let decompressed = brotli::decompress(&data, brotli::Dictionary::Empty).unwrap(); + Self { data: decompressed } + } +} + #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(rename_all = "PascalCase")] pub struct BatchInfo { @@ -54,7 +79,7 @@ pub struct StartState { pub pos_in_batch: u64, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, Deserialize)] #[serde(rename_all = "PascalCase")] pub struct FileData { pub id: u64, @@ -66,6 +91,8 @@ pub struct FileData { #[serde(with = "As::")] pub delayed_msg_b64: Vec, pub start_state: StartState, + #[serde(with = "As::>>")] + pub user_wasms: HashMap>>, } impl FileData { diff --git a/arbitrator/bench/src/prepare.rs b/arbitrator/prover/src/prepare.rs similarity index 73% rename from arbitrator/bench/src/prepare.rs rename to arbitrator/prover/src/prepare.rs index 741a7350ac..08bbd1476c 100644 --- a/arbitrator/bench/src/prepare.rs +++ b/arbitrator/prover/src/prepare.rs @@ -1,13 +1,13 @@ use arbutil::{Bytes32, PreimageType}; -use prover::machine::{argument_data_to_inbox, GlobalState, Machine}; -use prover::utils::CBytes; use std::collections::HashMap; use std::fs::File; use std::io::BufReader; use std::path::{Path, PathBuf}; use std::sync::Arc; +use crate::machine::{argument_data_to_inbox, GlobalState, Machine, Module}; use crate::parse_input::*; +use crate::utils::CBytes; pub fn prepare_machine(preimages: PathBuf, machines: PathBuf) -> eyre::Result { let file = File::open(preimages)?; @@ -40,6 +40,20 @@ pub fn prepare_machine(preimages: PathBuf, machines: PathBuf) -> eyre::Result.json files when WriteToFile() is called. + BlockInputsFilePath string `koanf:"block-inputs-file-path"` memoryFreeLimit int } @@ -163,6 +166,7 @@ func BlockValidatorConfigAddOptions(prefix string, f *pflag.FlagSet) { f.Bool(prefix+".failure-is-fatal", DefaultBlockValidatorConfig.FailureIsFatal, "failing a validation is treated as a fatal error") BlockValidatorDangerousConfigAddOptions(prefix+".dangerous", f) f.String(prefix+".memory-free-limit", DefaultBlockValidatorConfig.MemoryFreeLimit, "minimum free-memory limit after reaching which the blockvalidator pauses validation. Enabled by default as 1GB, to disable provide empty string") + f.String(prefix+".block-inputs-file-path", DefaultBlockValidatorConfig.BlockInputsFilePath, "directory to write block validation inputs files") } func BlockValidatorDangerousConfigAddOptions(prefix string, f *pflag.FlagSet) { @@ -181,6 +185,7 @@ var DefaultBlockValidatorConfig = BlockValidatorConfig{ PendingUpgradeModuleRoot: "latest", FailureIsFatal: true, Dangerous: DefaultBlockValidatorDangerousConfig, + BlockInputsFilePath: "./target/validation_inputs", MemoryFreeLimit: "default", } @@ -196,6 +201,7 @@ var TestBlockValidatorConfig = BlockValidatorConfig{ PendingUpgradeModuleRoot: "latest", FailureIsFatal: true, Dangerous: DefaultBlockValidatorDangerousConfig, + BlockInputsFilePath: "./target/validation_inputs", MemoryFreeLimit: "default", } diff --git a/staker/stateless_block_validator.go b/staker/stateless_block_validator.go index c8842aedc4..c2704b89ab 100644 --- a/staker/stateless_block_validator.go +++ b/staker/stateless_block_validator.go @@ -463,6 +463,31 @@ func (v *StatelessBlockValidator) ValidateResult( return true, &entry.End, nil } +func (v *StatelessBlockValidator) RecordValidationInput(ctx context.Context, pos arbutil.MessageIndex, moduleRoot common.Hash) error { + entry, err := v.CreateReadyValidationEntry(ctx, pos) + if err != nil { + return err + } + found := false + for _, spawner := range v.execSpawners { + if validator.SpawnerSupportsModule(spawner, moduleRoot) { + found = true + input, err := entry.ToInput(spawner.StylusArchs()) + if err != nil { + return err + } + _, err = spawner.WriteToFile(input, entry.End, moduleRoot).Await(ctx) + if err != nil { + return err + } + } + } + if !found { + return fmt.Errorf("validation with WasmModuleRoot %v not supported by node", moduleRoot) + } + return nil +} + func (v *StatelessBlockValidator) OverrideRecorder(t *testing.T, recorder execution.ExecutionRecorder) { v.recorder = recorder } diff --git a/system_tests/program_norace_test.go b/system_tests/program_norace_test.go index 56b2046716..2ccd77c5b5 100644 --- a/system_tests/program_norace_test.go +++ b/system_tests/program_norace_test.go @@ -104,6 +104,19 @@ func validateBlockRange( } } +// recordBlock writes a json file with all of the data needed to validate a block. +// +// This can be used as an input to the arbitrator prover to validate a block. +func recordBlock(t *testing.T, block uint64, builder *NodeBuilder) { + t.Helper() + ctx := builder.ctx + wasmModuleRoot := currentRootModule(t) + inboxPos := arbutil.MessageIndex(block) + if err := builder.L2.ConsensusNode.StatelessBlockValidator.RecordValidationInput(ctx, inboxPos, wasmModuleRoot); err != nil { + Fatal(t, "failed to record block", block, err) + } +} + func TestProgramEvmData(t *testing.T) { t.Parallel() testEvmData(t, true) diff --git a/system_tests/program_test.go b/system_tests/program_test.go index 5fa5db95c2..1327c67ab7 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -368,7 +368,7 @@ func errorTest(t *testing.T, jit bool) { func TestProgramStorage(t *testing.T) { t.Parallel() - storageTest(t, true) + storageTest(t, false) } func storageTest(t *testing.T, jit bool) { @@ -390,10 +390,13 @@ func storageTest(t *testing.T, jit bool) { key := testhelpers.RandomHash() value := testhelpers.RandomHash() tx := l2info.PrepareTxTo("Owner", &programAddress, l2info.TransferGas, nil, argsForStorageWrite(key, value)) - ensure(tx, l2client.SendTransaction(ctx, tx)) + receipt := ensure(tx, l2client.SendTransaction(ctx, tx)) + assertStorageAt(t, ctx, l2client, programAddress, key, value) validateBlocks(t, 2, jit, builder) + recordBlock(t, 2, builder) + recordBlock(t, receipt.BlockNumber.Uint64(), builder) } func TestProgramTransientStorage(t *testing.T) { @@ -1143,6 +1146,7 @@ func testActivateFails(t *testing.T, jit bool) { }) validateBlockRange(t, []uint64{blockToValidate}, jit, builder) + recordBlock(t, blockToValidate, builder) } func TestProgramSdkStorage(t *testing.T) { diff --git a/validator/client/validation_client.go b/validator/client/validation_client.go index d6743b109e..91b0c71a79 100644 --- a/validator/client/validation_client.go +++ b/validator/client/validation_client.go @@ -181,11 +181,6 @@ func (c *ExecutionClient) LatestWasmModuleRoot() containers.PromiseInterface[com func (c *ExecutionClient) WriteToFile(input *validator.ValidationInput, expOut validator.GoGlobalState, moduleRoot common.Hash) containers.PromiseInterface[struct{}] { jsonInput := server_api.ValidationInputToJson(input) - if err := jsonInput.WriteToFile(); err != nil { - return stopwaiter.LaunchPromiseThread[struct{}](c, func(ctx context.Context) (struct{}, error) { - return struct{}{}, err - }) - } return stopwaiter.LaunchPromiseThread[struct{}](c, func(ctx context.Context) (struct{}, error) { err := c.client.CallContext(ctx, nil, server_api.Namespace+"_writeToFile", jsonInput, expOut, moduleRoot) return struct{}{}, err diff --git a/validator/server_arb/validator_spawner.go b/validator/server_arb/validator_spawner.go index 1d4126dc7c..43ad7930b6 100644 --- a/validator/server_arb/validator_spawner.go +++ b/validator/server_arb/validator_spawner.go @@ -17,6 +17,7 @@ import ( "github.com/offchainlabs/nitro/util/containers" "github.com/offchainlabs/nitro/util/stopwaiter" "github.com/offchainlabs/nitro/validator" + "github.com/offchainlabs/nitro/validator/server_api" "github.com/offchainlabs/nitro/validator/server_common" "github.com/offchainlabs/nitro/validator/valnode/redis" @@ -187,6 +188,7 @@ func (v *ArbitratorSpawner) execute( } func (v *ArbitratorSpawner) Launch(entry *validator.ValidationInput, moduleRoot common.Hash) validator.ValidationRun { + println("LAUCHING ARBITRATOR VALIDATION") v.count.Add(1) promise := stopwaiter.LaunchPromiseThread[validator.GoGlobalState](v, func(ctx context.Context) (validator.GoGlobalState, error) { defer v.count.Add(-1) @@ -207,6 +209,10 @@ var launchTime = time.Now().Format("2006_01_02__15_04") //nolint:gosec func (v *ArbitratorSpawner) writeToFile(ctx context.Context, input *validator.ValidationInput, expOut validator.GoGlobalState, moduleRoot common.Hash) error { + jsonInput := server_api.ValidationInputToJson(input) + if err := jsonInput.WriteToFile(); err != nil { + return err + } outDirPath := filepath.Join(v.locator.RootPath(), v.config().OutputPath, launchTime, fmt.Sprintf("block_%d", input.Id)) err := os.MkdirAll(outDirPath, 0755) if err != nil { From 7113c52fe3fc3d4b9843466d0cc7a33df906e99f Mon Sep 17 00:00:00 2001 From: Pepper Lebeck-Jobe Date: Wed, 7 Aug 2024 19:10:52 +0200 Subject: [PATCH 029/489] Fix the deserialization of the UserWasm data This change also: 1. Documents the `parse_inputs.rs` file in an attempt to give context to why the deserialization is special. 2. Hardcodes the architecture to `wasm` when recording ValidationInputs for use with the prover. Previously, the `jit` validator would use the go system architecture, and the `arbitrator` validator would use `wavm`. 3. Switches the `TestStorageProgram` test back to the `jit` validator. Note: As of this commit, there is still a small problem that when running the validator using the json inputs, it does not come up with the same end has as `entry.End.Hash`. --- arbitrator/prover/src/parse_input.rs | 51 ++++++++++++++++------------ arbitrator/prover/src/prepare.rs | 18 ++++------ staker/stateless_block_validator.go | 3 +- system_tests/program_test.go | 2 +- 4 files changed, 40 insertions(+), 34 deletions(-) diff --git a/arbitrator/prover/src/parse_input.rs b/arbitrator/prover/src/parse_input.rs index 9af8652c96..fa7adb4c41 100644 --- a/arbitrator/prover/src/parse_input.rs +++ b/arbitrator/prover/src/parse_input.rs @@ -1,5 +1,5 @@ use arbutil::Bytes32; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; use serde_json; use serde_with::base64::Base64; use serde_with::As; @@ -9,15 +9,14 @@ use std::{ io::{self, BufRead}, }; +/// prefixed_hex deserializes hex strings which are prefixed with `0x` +/// +/// The default hex deserializer does not support prefixed hex strings. +/// +/// It is an error to use this deserializer on a string that does not +/// begin with `0x`. mod prefixed_hex { - use serde::{self, Deserialize, Deserializer, Serializer}; - - pub fn serialize(bytes: &Vec, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_str(&format!("0x{}", hex::encode(bytes))) - } + use serde::{self, Deserialize, Deserializer}; pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> where @@ -32,23 +31,25 @@ mod prefixed_hex { } } -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct PreimageMap(HashMap>); - #[derive(Debug)] -pub struct UserWasm { - data: Vec, -} +pub struct UserWasm(Vec); +/// UserWasm is a wrapper around Vec +/// +/// It is useful for decompressing a brotli-compressed wasm module. +/// +/// Note: The wrapped Vec is already Base64 decoded before +/// from(Vec) is called by serde. impl UserWasm { + /// as_vec returns the decompressed wasm module as a Vec pub fn as_vec(&self) -> Vec { - self.data.clone() + self.0.clone() } } impl AsRef<[u8]> for UserWasm { fn as_ref(&self) -> &[u8] { - &self.data + &self.0 } } @@ -56,11 +57,11 @@ impl AsRef<[u8]> for UserWasm { impl From> for UserWasm { fn from(data: Vec) -> Self { let decompressed = brotli::decompress(&data, brotli::Dictionary::Empty).unwrap(); - Self { data: decompressed } + Self(decompressed) } } -#[derive(Debug, Clone, Deserialize, Serialize)] +#[derive(Debug, Clone, Deserialize)] #[serde(rename_all = "PascalCase")] pub struct BatchInfo { pub number: u64, @@ -68,7 +69,7 @@ pub struct BatchInfo { pub data_b64: Vec, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, Deserialize)] #[serde(rename_all = "PascalCase")] pub struct StartState { #[serde(with = "prefixed_hex")] @@ -79,6 +80,14 @@ pub struct StartState { pub pos_in_batch: u64, } +/// FileData is the deserialized form of the input JSON file. +/// +/// The go JSON library in json.go uses some custom serialization and +/// compression logic that needs to be reversed when deserializing the +/// JSON in rust. +/// +/// Note: It is important to change this file whenever the go JSON +/// serialization changes. #[derive(Debug, Deserialize)] #[serde(rename_all = "PascalCase")] pub struct FileData { @@ -92,7 +101,7 @@ pub struct FileData { pub delayed_msg_b64: Vec, pub start_state: StartState, #[serde(with = "As::>>")] - pub user_wasms: HashMap>>, + pub user_wasms: HashMap>, } impl FileData { diff --git a/arbitrator/prover/src/prepare.rs b/arbitrator/prover/src/prepare.rs index 08bbd1476c..ecfb517735 100644 --- a/arbitrator/prover/src/prepare.rs +++ b/arbitrator/prover/src/prepare.rs @@ -5,7 +5,7 @@ use std::io::BufReader; use std::path::{Path, PathBuf}; use std::sync::Arc; -use crate::machine::{argument_data_to_inbox, GlobalState, Machine, Module}; +use crate::machine::{argument_data_to_inbox, GlobalState, Machine}; use crate::parse_input::*; use crate::utils::CBytes; @@ -40,17 +40,13 @@ pub fn prepare_machine(preimages: PathBuf, machines: PathBuf) -> eyre::Result Date: Wed, 7 Aug 2024 21:11:24 +0200 Subject: [PATCH 030/489] Print the End global state This makes it easier to visually confirm the expected outcome if a human is looking at the output of the prover and knows what the expected end global state should be. --- arbitrator/prover/src/main.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/arbitrator/prover/src/main.rs b/arbitrator/prover/src/main.rs index 002b693bd9..8c90ee44a4 100644 --- a/arbitrator/prover/src/main.rs +++ b/arbitrator/prover/src/main.rs @@ -344,6 +344,13 @@ fn main() -> Result<()> { }); } + println!( + "End GlobalState:\n BlockHash: {:?}\n SendRoot: {:?}\n Batch: {}\n PosInBatch: {}", + mach.get_global_state().bytes32_vals[0], + mach.get_global_state().bytes32_vals[1], + mach.get_global_state().u64_vals[0], + mach.get_global_state().u64_vals[1] + ); println!("End machine status: {:?}", mach.get_status()); println!("End machine hash: {}", mach.hash()); println!("End machine stack: {:?}", mach.get_data_stack()); From 3eed9bf6ec46cd81f6011778cef2ff03ba2040de Mon Sep 17 00:00:00 2001 From: Pepper Lebeck-Jobe Date: Wed, 7 Aug 2024 21:15:47 +0200 Subject: [PATCH 031/489] Remove two calls to recordBlock The thrid one is left around as an example of how to use the new method. --- system_tests/program_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/system_tests/program_test.go b/system_tests/program_test.go index a7eb94733d..29c5099912 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -395,7 +395,8 @@ func storageTest(t *testing.T, jit bool) { assertStorageAt(t, ctx, l2client, programAddress, key, value) validateBlocks(t, 2, jit, builder) - recordBlock(t, 2, builder) + // Captures a block_input_.json file for the block that included the + // storage write transaction. recordBlock(t, receipt.BlockNumber.Uint64(), builder) } @@ -1146,7 +1147,6 @@ func testActivateFails(t *testing.T, jit bool) { }) validateBlockRange(t, []uint64{blockToValidate}, jit, builder) - recordBlock(t, blockToValidate, builder) } func TestProgramSdkStorage(t *testing.T) { From ec7a386cd6364c4dcced822e452e74de3b49c7c5 Mon Sep 17 00:00:00 2001 From: Pepper Lebeck-Jobe Date: Thu, 8 Aug 2024 11:49:14 +0200 Subject: [PATCH 032/489] A few small changes bundled together in here. 1. Remove the part of the writeToFile method on ArbitratorSpawner which was writing the bash scripts. (TODO: Write the block_inputs_.json files somewhere better.) 2. Drop the expOut from the WriteToFile signature since none of the implementations need it any more. 3. Rename some unused function arguments to "_". --- arbitrator/bench/Cargo.toml | 4 - staker/block_validator.go | 2 +- staker/stateless_block_validator.go | 2 +- system_tests/validation_mock_test.go | 2 +- validator/client/validation_client.go | 4 +- validator/interface.go | 2 +- validator/server_arb/validator_spawner.go | 133 +--------------------- validator/server_jit/jit_machine.go | 2 +- validator/valnode/validation_api.go | 4 +- 9 files changed, 13 insertions(+), 142 deletions(-) diff --git a/arbitrator/bench/Cargo.toml b/arbitrator/bench/Cargo.toml index 284180dc46..74b948aca8 100644 --- a/arbitrator/bench/Cargo.toml +++ b/arbitrator/bench/Cargo.toml @@ -3,10 +3,6 @@ name = "bench" version = "0.1.0" edition = "2021" -[lib] -name = "bench" -path = "src/lib.rs" - [[bin]] name = "benchbin" path = "src/bin.rs" diff --git a/staker/block_validator.go b/staker/block_validator.go index 8d25ddf3c9..5d37cb17bd 100644 --- a/staker/block_validator.go +++ b/staker/block_validator.go @@ -496,7 +496,7 @@ func (v *BlockValidator) writeToFile(validationEntry *validationEntry, moduleRoo } for _, spawner := range v.execSpawners { if validator.SpawnerSupportsModule(spawner, moduleRoot) { - _, err = spawner.WriteToFile(input, validationEntry.End, moduleRoot).Await(v.GetContext()) + _, err = spawner.WriteToFile(input, moduleRoot).Await(v.GetContext()) return err } } diff --git a/staker/stateless_block_validator.go b/staker/stateless_block_validator.go index d19b1e2d1a..d97b02d340 100644 --- a/staker/stateless_block_validator.go +++ b/staker/stateless_block_validator.go @@ -477,7 +477,7 @@ func (v *StatelessBlockValidator) RecordValidationInput(ctx context.Context, pos if err != nil { return err } - _, err = spawner.WriteToFile(input, entry.End, moduleRoot).Await(ctx) + _, err = spawner.WriteToFile(input, moduleRoot).Await(ctx) if err != nil { return err } diff --git a/system_tests/validation_mock_test.go b/system_tests/validation_mock_test.go index 2c6321d009..48328d778e 100644 --- a/system_tests/validation_mock_test.go +++ b/system_tests/validation_mock_test.go @@ -95,7 +95,7 @@ func (s *mockSpawner) LatestWasmModuleRoot() containers.PromiseInterface[common. return containers.NewReadyPromise[common.Hash](mockWasmModuleRoots[0], nil) } -func (s *mockSpawner) WriteToFile(input *validator.ValidationInput, expOut validator.GoGlobalState, moduleRoot common.Hash) containers.PromiseInterface[struct{}] { +func (s *mockSpawner) WriteToFile(input *validator.ValidationInput, moduleRoot common.Hash) containers.PromiseInterface[struct{}] { return containers.NewReadyPromise[struct{}](struct{}{}, nil) } diff --git a/validator/client/validation_client.go b/validator/client/validation_client.go index 91b0c71a79..5a50a9525c 100644 --- a/validator/client/validation_client.go +++ b/validator/client/validation_client.go @@ -179,10 +179,10 @@ func (c *ExecutionClient) LatestWasmModuleRoot() containers.PromiseInterface[com }) } -func (c *ExecutionClient) WriteToFile(input *validator.ValidationInput, expOut validator.GoGlobalState, moduleRoot common.Hash) containers.PromiseInterface[struct{}] { +func (c *ExecutionClient) WriteToFile(input *validator.ValidationInput, moduleRoot common.Hash) containers.PromiseInterface[struct{}] { jsonInput := server_api.ValidationInputToJson(input) return stopwaiter.LaunchPromiseThread[struct{}](c, func(ctx context.Context) (struct{}, error) { - err := c.client.CallContext(ctx, nil, server_api.Namespace+"_writeToFile", jsonInput, expOut, moduleRoot) + err := c.client.CallContext(ctx, nil, server_api.Namespace+"_writeToFile", jsonInput, moduleRoot) return struct{}{}, err }) } diff --git a/validator/interface.go b/validator/interface.go index 80aa2c1fcc..96d3d2da18 100644 --- a/validator/interface.go +++ b/validator/interface.go @@ -26,7 +26,7 @@ type ExecutionSpawner interface { ValidationSpawner CreateExecutionRun(wasmModuleRoot common.Hash, input *ValidationInput) containers.PromiseInterface[ExecutionRun] LatestWasmModuleRoot() containers.PromiseInterface[common.Hash] - WriteToFile(input *ValidationInput, expOut GoGlobalState, moduleRoot common.Hash) containers.PromiseInterface[struct{}] + WriteToFile(input *ValidationInput, moduleRoot common.Hash) containers.PromiseInterface[struct{}] } type ExecutionRun interface { diff --git a/validator/server_arb/validator_spawner.go b/validator/server_arb/validator_spawner.go index 43ad7930b6..8df390514b 100644 --- a/validator/server_arb/validator_spawner.go +++ b/validator/server_arb/validator_spawner.go @@ -2,11 +2,8 @@ package server_arb import ( "context" - "encoding/binary" "errors" "fmt" - "os" - "path/filepath" "runtime" "sync/atomic" "time" @@ -97,7 +94,7 @@ func (s *ArbitratorSpawner) Name() string { return "arbitrator" } -func (v *ArbitratorSpawner) loadEntryToMachine(ctx context.Context, entry *validator.ValidationInput, mach *ArbitratorMachine) error { +func (v *ArbitratorSpawner) loadEntryToMachine(_ context.Context, entry *validator.ValidationInput, mach *ArbitratorMachine) error { resolver := func(ty arbutil.PreimageType, hash common.Hash) ([]byte, error) { // Check if it's a known preimage if preimage, ok := entry.Preimages[ty][hash]; ok { @@ -205,139 +202,17 @@ func (v *ArbitratorSpawner) Room() int { return avail } -var launchTime = time.Now().Format("2006_01_02__15_04") - -//nolint:gosec -func (v *ArbitratorSpawner) writeToFile(ctx context.Context, input *validator.ValidationInput, expOut validator.GoGlobalState, moduleRoot common.Hash) error { +func (v *ArbitratorSpawner) writeToFile(_ context.Context, input *validator.ValidationInput, _ common.Hash) error { jsonInput := server_api.ValidationInputToJson(input) if err := jsonInput.WriteToFile(); err != nil { return err } - outDirPath := filepath.Join(v.locator.RootPath(), v.config().OutputPath, launchTime, fmt.Sprintf("block_%d", input.Id)) - err := os.MkdirAll(outDirPath, 0755) - if err != nil { - return err - } - if ctx.Err() != nil { - return ctx.Err() - } - - rootPathAssign := "" - if executable, err := os.Executable(); err == nil { - rootPathAssign = "ROOTPATH=\"" + filepath.Dir(executable) + "\"\n" - } - cmdFile, err := os.OpenFile(filepath.Join(outDirPath, "run-prover.sh"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755) - if err != nil { - return err - } - defer cmdFile.Close() - _, err = cmdFile.WriteString("#!/bin/bash\n" + - fmt.Sprintf("# expected output: batch %d, postion %d, hash %s\n", expOut.Batch, expOut.PosInBatch, expOut.BlockHash) + - "MACHPATH=\"" + v.locator.GetMachinePath(moduleRoot) + "\"\n" + - rootPathAssign + - "if (( $# > 1 )); then\n" + - " if [[ $1 == \"-m\" ]]; then\n" + - " MACHPATH=$2\n" + - " shift\n" + - " shift\n" + - " fi\n" + - "fi\n" + - "${ROOTPATH}/bin/prover ${MACHPATH}/replay.wasm") - if err != nil { - return err - } - if ctx.Err() != nil { - return ctx.Err() - } - - libraries := []string{"soft-float.wasm", "wasi_stub.wasm", "go_stub.wasm", "host_io.wasm", "brotli.wasm"} - for _, module := range libraries { - _, err = cmdFile.WriteString(" -l " + "${MACHPATH}/" + module) - if err != nil { - return err - } - } - _, err = cmdFile.WriteString(fmt.Sprintf(" --inbox-position %d --position-within-message %d --last-block-hash %s", input.StartState.Batch, input.StartState.PosInBatch, input.StartState.BlockHash)) - if err != nil { - return err - } - - for _, msg := range input.BatchInfo { - if ctx.Err() != nil { - return ctx.Err() - } - sequencerFileName := fmt.Sprintf("sequencer_%d.bin", msg.Number) - err = os.WriteFile(filepath.Join(outDirPath, sequencerFileName), msg.Data, 0644) - if err != nil { - return err - } - _, err = cmdFile.WriteString(" --inbox " + sequencerFileName) - if err != nil { - return err - } - } - - preimageFile, err := os.Create(filepath.Join(outDirPath, "preimages.bin")) - if err != nil { - return err - } - defer preimageFile.Close() - for ty, preimages := range input.Preimages { - _, err = preimageFile.Write([]byte{byte(ty)}) - if err != nil { - return err - } - for _, data := range preimages { - if ctx.Err() != nil { - return ctx.Err() - } - lenbytes := make([]byte, 8) - binary.LittleEndian.PutUint64(lenbytes, uint64(len(data))) - _, err := preimageFile.Write(lenbytes) - if err != nil { - return err - } - _, err = preimageFile.Write(data) - if err != nil { - return err - } - } - } - - _, err = cmdFile.WriteString(" --preimages preimages.bin") - if err != nil { - return err - } - - if input.HasDelayedMsg { - if ctx.Err() != nil { - return ctx.Err() - } - _, err = cmdFile.WriteString(fmt.Sprintf(" --delayed-inbox-position %d", input.DelayedMsgNr)) - if err != nil { - return err - } - filename := fmt.Sprintf("delayed_%d.bin", input.DelayedMsgNr) - err = os.WriteFile(filepath.Join(outDirPath, filename), input.DelayedMsg, 0644) - if err != nil { - return err - } - _, err = cmdFile.WriteString(fmt.Sprintf(" --delayed-inbox %s", filename)) - if err != nil { - return err - } - } - - _, err = cmdFile.WriteString(" \"$@\"\n") - if err != nil { - return err - } return nil } -func (v *ArbitratorSpawner) WriteToFile(input *validator.ValidationInput, expOut validator.GoGlobalState, moduleRoot common.Hash) containers.PromiseInterface[struct{}] { +func (v *ArbitratorSpawner) WriteToFile(input *validator.ValidationInput, moduleRoot common.Hash) containers.PromiseInterface[struct{}] { return stopwaiter.LaunchPromiseThread[struct{}](v, func(ctx context.Context) (struct{}, error) { - err := v.writeToFile(ctx, input, expOut, moduleRoot) + err := v.writeToFile(ctx, input, moduleRoot) return struct{}{}, err }) } diff --git a/validator/server_jit/jit_machine.go b/validator/server_jit/jit_machine.go index e4fb840cbb..5ba011727f 100644 --- a/validator/server_jit/jit_machine.go +++ b/validator/server_jit/jit_machine.go @@ -31,7 +31,7 @@ type JitMachine struct { wasmMemoryUsageLimit int } -func createJitMachine(jitBinary string, binaryPath string, cranelift bool, wasmMemoryUsageLimit int, moduleRoot common.Hash, fatalErrChan chan error) (*JitMachine, error) { +func createJitMachine(jitBinary string, binaryPath string, cranelift bool, wasmMemoryUsageLimit int, _ common.Hash, fatalErrChan chan error) (*JitMachine, error) { invocation := []string{"--binary", binaryPath, "--forks"} if cranelift { invocation = append(invocation, "--cranelift") diff --git a/validator/valnode/validation_api.go b/validator/valnode/validation_api.go index 6245ffc5e3..6561e7f0fd 100644 --- a/validator/valnode/validation_api.go +++ b/validator/valnode/validation_api.go @@ -117,12 +117,12 @@ func (a *ExecServerAPI) Start(ctx_in context.Context) { a.CallIteratively(a.removeOldRuns) } -func (a *ExecServerAPI) WriteToFile(ctx context.Context, jsonInput *server_api.InputJSON, expOut validator.GoGlobalState, moduleRoot common.Hash) error { +func (a *ExecServerAPI) WriteToFile(ctx context.Context, jsonInput *server_api.InputJSON, moduleRoot common.Hash) error { input, err := server_api.ValidationInputFromJson(jsonInput) if err != nil { return err } - _, err = a.execSpawner.WriteToFile(input, expOut, moduleRoot).Await(ctx) + _, err = a.execSpawner.WriteToFile(input, moduleRoot).Await(ctx) return err } From 7e4963592f83f2cb3b79c5c3e72766f8aab0953f Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Thu, 15 Aug 2024 17:25:43 +0530 Subject: [PATCH 033/489] Support multiple producers in redis streams --- arbnode/dataposter/data_poster.go | 2 +- arbnode/dataposter/redis/redisstorage.go | 4 +- arbnode/redislock/redis.go | 2 +- arbnode/seq_coordinator.go | 2 +- .../rediscoordinator/redis_coordinator.go | 2 +- das/redis_storage_service.go | 2 +- go.mod | 8 +- go.sum | 17 +- pubsub/common.go | 7 +- pubsub/consumer.go | 155 ++++----- pubsub/producer.go | 307 ++++++------------ pubsub/pubsub_test.go | 210 ++++++++---- system_tests/common_test.go | 2 +- system_tests/seq_coordinator_test.go | 2 +- util/redisutil/redis_coordinator.go | 2 +- util/redisutil/redisutil.go | 2 +- validator/client/redis/producer.go | 5 +- validator/validation_entry.go | 16 + validator/valnode/redis/consumer.go | 2 +- 19 files changed, 388 insertions(+), 361 deletions(-) diff --git a/arbnode/dataposter/data_poster.go b/arbnode/dataposter/data_poster.go index 15446fe855..f90dc9bf3f 100644 --- a/arbnode/dataposter/data_poster.go +++ b/arbnode/dataposter/data_poster.go @@ -33,7 +33,6 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/signer/core/apitypes" - "github.com/go-redis/redis/v8" "github.com/holiman/uint256" "github.com/offchainlabs/nitro/arbnode/dataposter/dbstorage" "github.com/offchainlabs/nitro/arbnode/dataposter/noop" @@ -46,6 +45,7 @@ import ( "github.com/offchainlabs/nitro/util/rpcclient" "github.com/offchainlabs/nitro/util/signature" "github.com/offchainlabs/nitro/util/stopwaiter" + "github.com/redis/go-redis/v9" "github.com/spf13/pflag" redisstorage "github.com/offchainlabs/nitro/arbnode/dataposter/redis" diff --git a/arbnode/dataposter/redis/redisstorage.go b/arbnode/dataposter/redis/redisstorage.go index 8b6dcf65ac..b54abf618b 100644 --- a/arbnode/dataposter/redis/redisstorage.go +++ b/arbnode/dataposter/redis/redisstorage.go @@ -9,9 +9,9 @@ import ( "errors" "fmt" - "github.com/go-redis/redis/v8" "github.com/offchainlabs/nitro/arbnode/dataposter/storage" "github.com/offchainlabs/nitro/util/signature" + "github.com/redis/go-redis/v9" ) // Storage implements redis sorted set backed storage. It does not support @@ -196,7 +196,7 @@ func (s *Storage) Put(ctx context.Context, index uint64, prev, new *storage.Queu if err != nil { return err } - if err := pipe.ZAdd(ctx, s.key, &redis.Z{ + if err := pipe.ZAdd(ctx, s.key, redis.Z{ Score: float64(index), Member: string(signedItem), }).Err(); err != nil { diff --git a/arbnode/redislock/redis.go b/arbnode/redislock/redis.go index 7e26010cae..de9508323a 100644 --- a/arbnode/redislock/redis.go +++ b/arbnode/redislock/redis.go @@ -12,8 +12,8 @@ import ( "time" "github.com/ethereum/go-ethereum/log" - "github.com/go-redis/redis/v8" "github.com/offchainlabs/nitro/util/stopwaiter" + "github.com/redis/go-redis/v9" flag "github.com/spf13/pflag" ) diff --git a/arbnode/seq_coordinator.go b/arbnode/seq_coordinator.go index 98c19ce361..80c22ab510 100644 --- a/arbnode/seq_coordinator.go +++ b/arbnode/seq_coordinator.go @@ -14,7 +14,7 @@ import ( "sync/atomic" "time" - "github.com/go-redis/redis/v8" + "github.com/redis/go-redis/v9" flag "github.com/spf13/pflag" "github.com/ethereum/go-ethereum/log" diff --git a/cmd/seq-coordinator-manager/rediscoordinator/redis_coordinator.go b/cmd/seq-coordinator-manager/rediscoordinator/redis_coordinator.go index e963c0e96c..b897b23252 100644 --- a/cmd/seq-coordinator-manager/rediscoordinator/redis_coordinator.go +++ b/cmd/seq-coordinator-manager/rediscoordinator/redis_coordinator.go @@ -5,8 +5,8 @@ import ( "errors" "strings" - "github.com/go-redis/redis/v8" "github.com/offchainlabs/nitro/util/redisutil" + "github.com/redis/go-redis/v9" ) // RedisCoordinator builds upon RedisCoordinator of redisutil with additional functionality diff --git a/das/redis_storage_service.go b/das/redis_storage_service.go index 210d5cb2d4..e57240992c 100644 --- a/das/redis_storage_service.go +++ b/das/redis_storage_service.go @@ -12,11 +12,11 @@ import ( "golang.org/x/crypto/sha3" - "github.com/go-redis/redis/v8" "github.com/offchainlabs/nitro/arbstate/daprovider" "github.com/offchainlabs/nitro/das/dastree" "github.com/offchainlabs/nitro/util/pretty" "github.com/offchainlabs/nitro/util/redisutil" + "github.com/redis/go-redis/v9" flag "github.com/spf13/pflag" "github.com/ethereum/go-ethereum/common" diff --git a/go.mod b/go.mod index 6649973725..5453205c5d 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,6 @@ require ( github.com/ethereum/go-ethereum v1.10.26 github.com/fatih/structtag v1.2.0 github.com/gdamore/tcell/v2 v2.7.1 - github.com/go-redis/redis/v8 v8.11.5 github.com/gobwas/httphead v0.1.0 github.com/gobwas/ws v1.2.1 github.com/gobwas/ws-examples v0.0.0-20190625122829-a9e8908d9484 @@ -38,6 +37,7 @@ require ( github.com/mitchellh/mapstructure v1.4.1 github.com/pkg/errors v0.9.1 github.com/r3labs/diff/v3 v3.0.1 + github.com/redis/go-redis/v9 v9.6.1 github.com/rivo/tview v0.0.0-20240307173318-e804876934a1 github.com/spf13/pflag v1.0.5 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 @@ -51,7 +51,11 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.0.0 ) -require github.com/google/go-querystring v1.1.0 // indirect +require ( + github.com/google/go-querystring v1.1.0 // indirect + github.com/onsi/ginkgo v1.16.5 // indirect + github.com/onsi/gomega v1.18.1 // indirect +) require ( github.com/DataDog/zstd v1.4.5 // indirect diff --git a/go.sum b/go.sum index 8529b2497d..bf0b385631 100644 --- a/go.sum +++ b/go.sum @@ -136,6 +136,10 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bits-and-blooms/bitset v1.10.0 h1:ePXTeiPEazB5+opbv5fr8umg2R/1NlzgDsyepwsSr88= github.com/bits-and-blooms/bitset v1.10.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k= github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= @@ -273,11 +277,10 @@ github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AE github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= -github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= -github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= @@ -377,6 +380,7 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b h1:RMpPgZTSApbPf7xaVel+QkoGPRLFLrwFO89uDUHEGf0= github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= @@ -433,6 +437,7 @@ github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/hydrogen18/memlistener v0.0.0-20200120041712-dcc25e7acd91/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= @@ -577,10 +582,13 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= @@ -628,6 +636,8 @@ github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5 github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/r3labs/diff/v3 v3.0.1 h1:CBKqf3XmNRHXKmdU7mZP1w7TV0pDyVCis1AUHtA4Xtg= github.com/r3labs/diff/v3 v3.0.1/go.mod h1:f1S9bourRbiM66NskseyUdo0fTmEE0qKrikYJX63dgo= +github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4= +github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA= github.com/rhnvrm/simples3 v0.6.1 h1:H0DJwybR6ryQE+Odi9eqkHuzjYAeJgtGcGtuBwOhsH8= github.com/rhnvrm/simples3 v0.6.1/go.mod h1:Y+3vYm2V7Y4VijFoJHHTrja6OgPrJ2cBti8dPGkC3sA= github.com/rivo/tview v0.0.0-20240307173318-e804876934a1 h1:bWLHTRekAy497pE7+nXSuzXwwFHI0XauRzz6roUvY+s= @@ -823,6 +833,7 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -904,6 +915,7 @@ golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1001,6 +1013,7 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= diff --git a/pubsub/common.go b/pubsub/common.go index d7f041af15..4b5778b9ba 100644 --- a/pubsub/common.go +++ b/pubsub/common.go @@ -2,12 +2,17 @@ package pubsub import ( "context" + "fmt" "strings" "github.com/ethereum/go-ethereum/log" - "github.com/go-redis/redis/v8" + "github.com/redis/go-redis/v9" ) +const UNIQUEID_MSGID_MAP_KEY string = ".msgId" // Is used to map unique identifier to msgId of the message consisting request in the stream + +func MessageKeyFor(streamName, id string) string { return fmt.Sprintf("%s.%s", streamName, id) } + // CreateStream tries to create stream with given name, if it already exists // does not return an error. func CreateStream(ctx context.Context, streamName string, client redis.UniversalClient) error { diff --git a/pubsub/consumer.go b/pubsub/consumer.go index df3695606d..3adb571343 100644 --- a/pubsub/consumer.go +++ b/pubsub/consumer.go @@ -8,44 +8,44 @@ import ( "time" "github.com/ethereum/go-ethereum/log" - "github.com/go-redis/redis/v8" "github.com/google/uuid" "github.com/offchainlabs/nitro/util/stopwaiter" + "github.com/redis/go-redis/v9" "github.com/spf13/pflag" ) type ConsumerConfig struct { // Timeout of result entry in Redis. ResponseEntryTimeout time.Duration `koanf:"response-entry-timeout"` - // Duration after which consumer is considered to be dead if heartbeat - // is not updated. - KeepAliveTimeout time.Duration `koanf:"keepalive-timeout"` + // Minimum idle time after which messages will be autoclaimed + IdletimeToAutoclaim time.Duration `koanf:"Idletime-to-autoclaim"` } var DefaultConsumerConfig = ConsumerConfig{ ResponseEntryTimeout: time.Hour, - KeepAliveTimeout: 5 * time.Minute, + IdletimeToAutoclaim: 30 * time.Minute, } var TestConsumerConfig = ConsumerConfig{ ResponseEntryTimeout: time.Minute, - KeepAliveTimeout: 30 * time.Millisecond, + IdletimeToAutoclaim: time.Second, } func ConsumerConfigAddOptions(prefix string, f *pflag.FlagSet) { f.Duration(prefix+".response-entry-timeout", DefaultConsumerConfig.ResponseEntryTimeout, "timeout for response entry") - f.Duration(prefix+".keepalive-timeout", DefaultConsumerConfig.KeepAliveTimeout, "timeout after which consumer is considered inactive if heartbeat wasn't performed") + f.Duration(prefix+".Idletime-to-autoclaim", DefaultConsumerConfig.IdletimeToAutoclaim, "After a message spends this amount of time in PEL (Pending Entries List i.e claimed by another consumer but not Acknowledged) it will be allowed to be autoclaimed by other consumers") } // Consumer implements a consumer for redis stream provides heartbeat to // indicate it is alive. type Consumer[Request any, Response any] struct { stopwaiter.StopWaiter - id string - client redis.UniversalClient - redisStream string - redisGroup string - cfg *ConsumerConfig + id string + client redis.UniversalClient + redisStream string + redisGroup string + cfg *ConsumerConfig + ackNotifiers map[string]chan struct{} } type Message[Request any] struct { @@ -58,32 +58,22 @@ func NewConsumer[Request any, Response any](client redis.UniversalClient, stream return nil, fmt.Errorf("redis stream name cannot be empty") } return &Consumer[Request, Response]{ - id: uuid.NewString(), - client: client, - redisStream: streamName, - redisGroup: streamName, // There is 1-1 mapping of redis stream and consumer group. - cfg: cfg, + id: uuid.NewString(), + client: client, + redisStream: streamName, + redisGroup: streamName, // There is 1-1 mapping of redis stream and consumer group. + cfg: cfg, + ackNotifiers: make(map[string]chan struct{}), }, nil } // Start starts the consumer to iteratively perform heartbeat in configured intervals. func (c *Consumer[Request, Response]) Start(ctx context.Context) { c.StopWaiter.Start(ctx, c) - c.StopWaiter.CallIteratively( - func(ctx context.Context) time.Duration { - c.heartBeat(ctx) - return c.cfg.KeepAliveTimeout / 10 - }, - ) } func (c *Consumer[Request, Response]) StopAndWait() { c.StopWaiter.StopAndWait() - c.deleteHeartBeat(c.GetParentContext()) -} - -func heartBeatKey(id string) string { - return fmt.Sprintf("consumer:%s:heartbeat", id) } func (c *Consumer[Request, Response]) RedisClient() redis.UniversalClient { @@ -94,55 +84,44 @@ func (c *Consumer[Request, Response]) StreamName() string { return c.redisStream } -func (c *Consumer[Request, Response]) heartBeatKey() string { - return heartBeatKey(c.id) -} - -// deleteHeartBeat deletes the heartbeat to indicate it is being shut down. -func (c *Consumer[Request, Response]) deleteHeartBeat(ctx context.Context) { - if err := c.client.Del(ctx, c.heartBeatKey()).Err(); err != nil { - l := log.Info - if ctx.Err() != nil { - l = log.Error - } - l("Deleting heardbeat", "consumer", c.id, "error", err) - } -} - -// heartBeat updates the heartBeat key indicating aliveness. -func (c *Consumer[Request, Response]) heartBeat(ctx context.Context) { - if err := c.client.Set(ctx, c.heartBeatKey(), time.Now().UnixMilli(), 2*c.cfg.KeepAliveTimeout).Err(); err != nil { - l := log.Info - if ctx.Err() != nil { - l = log.Error - } - l("Updating heardbeat", "consumer", c.id, "error", err) - } -} - // Consumer first checks it there exists pending message that is claimed by // unresponsive consumer, if not then reads from the stream. func (c *Consumer[Request, Response]) Consume(ctx context.Context) (*Message[Request], error) { - res, err := c.client.XReadGroup(ctx, &redis.XReadGroupArgs{ + // First try to XAUTOCLAIM, this prioritizes processing PEL messages + // that have been waiting for more than IdletimeToAutoclaim duration + messages, _, err := c.client.XAutoClaim(ctx, &redis.XAutoClaimArgs{ Group: c.redisGroup, Consumer: c.id, - // Receive only messages that were never delivered to any other consumer, - // that is, only new messages. - Streams: []string{c.redisStream, ">"}, - Count: 1, - Block: time.Millisecond, // 0 seems to block the read instead of immediately returning + MinIdle: c.cfg.IdletimeToAutoclaim, // Minimum idle time for messages to claim (in milliseconds) + Stream: c.redisStream, + Start: "0", + Count: 1, // Limit the number of messages to claim }).Result() - if errors.Is(err, redis.Nil) { - return nil, nil - } - if err != nil { - return nil, fmt.Errorf("reading message for consumer: %q: %w", c.id, err) - } - if len(res) != 1 || len(res[0].Messages) != 1 { - return nil, fmt.Errorf("redis returned entries: %+v, for querying single message", res) + if len(messages) != 1 || err != nil { + // Fallback to reading new messages + res, err := c.client.XReadGroup(ctx, &redis.XReadGroupArgs{ + Group: c.redisGroup, + Consumer: c.id, + // Receive only messages that were never delivered to any other consumer, + // that is, only new messages. + Streams: []string{c.redisStream, ">"}, + Count: 1, + Block: time.Millisecond, // 0 seems to block the read instead of immediately returning + }).Result() + if errors.Is(err, redis.Nil) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("reading message for consumer: %q: %w", c.id, err) + } + if len(res) != 1 || len(res[0].Messages) != 1 { + return nil, fmt.Errorf("redis returned entries: %+v, for querying single message", res) + } + messages = res[0].Messages } + var ( - value = res[0].Messages[0].Values[messageKey] + value = messages[0].Values[messageKey] data, ok = (value).(string) ) if !ok { @@ -152,24 +131,52 @@ func (c *Consumer[Request, Response]) Consume(ctx context.Context) (*Message[Req if err := json.Unmarshal([]byte(data), &req); err != nil { return nil, fmt.Errorf("unmarshaling value: %v, error: %w", value, err) } - log.Debug("Redis stream consuming", "consumer_id", c.id, "message_id", res[0].Messages[0].ID) + ackNotifier := make(chan struct{}) + c.StopWaiter.LaunchThread(func(ctx context.Context) { + for { + if err := c.client.XClaim(ctx, &redis.XClaimArgs{ + Stream: c.redisStream, + Group: c.redisGroup, + Consumer: c.id, + MinIdle: 0, + Messages: []string{messages[0].ID}, + }).Err(); err != nil { + log.Error("error claiming message, it might be possible that other consumers might pick this request", "msgID", messages[0].ID) + } + select { + case <-ackNotifier: + return + case <-ctx.Done(): + log.Info("Context done while claiming message to indicate hearbeat", "error", ctx.Err().Error()) + return + case <-time.After(c.cfg.IdletimeToAutoclaim / 3): + } + } + }) + c.ackNotifiers[messages[0].ID] = ackNotifier + log.Debug("Redis stream consuming", "consumer_id", c.id, "message_id", messages[0].ID) return &Message[Request]{ - ID: res[0].Messages[0].ID, + ID: messages[0].ID, Value: req, }, nil } -func (c *Consumer[Request, Response]) SetResult(ctx context.Context, messageID string, result Response) error { +func (c *Consumer[Request, Response]) SetResult(ctx context.Context, id string, messageID string, result Response) error { + if id == "" { + log.Info("Request doesn't have a unique identifier (SelfHash field is not set), defaulting to using redis stream messageId", "msgId", messageID) + id = messageID + } resp, err := json.Marshal(result) if err != nil { return fmt.Errorf("marshaling result: %w", err) } - acquired, err := c.client.SetNX(ctx, messageID, resp, c.cfg.ResponseEntryTimeout).Result() + acquired, err := c.client.SetNX(ctx, MessageKeyFor(c.StreamName(), id), resp, c.cfg.ResponseEntryTimeout).Result() if err != nil || !acquired { - return fmt.Errorf("setting result for message: %v, error: %w", messageID, err) + return fmt.Errorf("setting result for message with message-id in stream: %v, unique request identifier: %v, error: %w", messageID, id, err) } if _, err := c.client.XAck(ctx, c.redisStream, c.redisGroup, messageID).Result(); err != nil { return fmt.Errorf("acking message: %v, error: %w", messageID, err) } + close(c.ackNotifiers[messageID]) return nil } diff --git a/pubsub/producer.go b/pubsub/producer.go index 2b1cdb5e3f..df6e7d5a28 100644 --- a/pubsub/producer.go +++ b/pubsub/producer.go @@ -13,17 +13,16 @@ import ( "encoding/json" "errors" "fmt" - "math" "strconv" "strings" "sync" "time" "github.com/ethereum/go-ethereum/log" - "github.com/go-redis/redis/v8" "github.com/google/uuid" "github.com/offchainlabs/nitro/util/containers" "github.com/offchainlabs/nitro/util/stopwaiter" + "github.com/redis/go-redis/v9" "github.com/spf13/pflag" ) @@ -32,6 +31,11 @@ const ( defaultGroup = "default_consumer_group" ) +type MsgIdAndPromise[Response any] struct { + msgID string + promise *containers.Promise[Response] +} + type Producer[Request any, Response any] struct { stopwaiter.StopWaiter id string @@ -41,52 +45,33 @@ type Producer[Request any, Response any] struct { cfg *ProducerConfig promisesLock sync.RWMutex - promises map[string]*containers.Promise[Response] + promises map[string]*MsgIdAndPromise[Response] - // Used for running checks for pending messages with inactive consumers - // and checking responses from consumers iteratively for the first time when - // Produce is called. + // Used for checking responses from consumers iteratively + // For the first time when Produce is called. once sync.Once } type ProducerConfig struct { - // When enabled, messages that are sent to consumers that later die before - // processing them, will be re-inserted into the stream to be proceesed by - // another consumer - EnableReproduce bool `koanf:"enable-reproduce"` - // Interval duration in which producer checks for pending messages delivered - // to the consumers that are currently inactive. - CheckPendingInterval time.Duration `koanf:"check-pending-interval"` - // Duration after which consumer is considered to be dead if heartbeat - // is not updated. - KeepAliveTimeout time.Duration `koanf:"keepalive-timeout"` // Interval duration for checking the result set by consumers. CheckResultInterval time.Duration `koanf:"check-result-interval"` - CheckPendingItems int64 `koanf:"check-pending-items"` + // Timeout of entry's written to redis by producer + ResponseEntryTimeout time.Duration `koanf:"response-entry-timeout"` } var DefaultProducerConfig = ProducerConfig{ - EnableReproduce: true, - CheckPendingInterval: time.Second, - KeepAliveTimeout: 5 * time.Minute, CheckResultInterval: 5 * time.Second, - CheckPendingItems: 256, + ResponseEntryTimeout: time.Hour, } var TestProducerConfig = ProducerConfig{ - EnableReproduce: false, - CheckPendingInterval: 10 * time.Millisecond, - KeepAliveTimeout: 100 * time.Millisecond, CheckResultInterval: 5 * time.Millisecond, - CheckPendingItems: 256, + ResponseEntryTimeout: time.Minute, } func ProducerAddConfigAddOptions(prefix string, f *pflag.FlagSet) { - f.Bool(prefix+".enable-reproduce", DefaultProducerConfig.EnableReproduce, "when enabled, messages with dead consumer will be re-inserted into the stream") - f.Duration(prefix+".check-pending-interval", DefaultProducerConfig.CheckPendingInterval, "interval in which producer checks pending messages whether consumer processing them is inactive") f.Duration(prefix+".check-result-interval", DefaultProducerConfig.CheckResultInterval, "interval in which producer checks pending messages whether consumer processing them is inactive") - f.Duration(prefix+".keepalive-timeout", DefaultProducerConfig.KeepAliveTimeout, "timeout after which consumer is considered inactive if heartbeat wasn't performed") - f.Int64(prefix+".check-pending-items", DefaultProducerConfig.CheckPendingItems, "items to screen during check-pending") + f.Duration(prefix+".response-entry-timeout", DefaultProducerConfig.ResponseEntryTimeout, "timeout after which responses written from producer to the redis are cleared. Currently used for the key mapping unique request id to redis stream message id") } func NewProducer[Request any, Response any](client redis.UniversalClient, streamName string, cfg *ProducerConfig) (*Producer[Request, Response], error) { @@ -102,150 +87,88 @@ func NewProducer[Request any, Response any](client redis.UniversalClient, stream redisStream: streamName, redisGroup: streamName, // There is 1-1 mapping of redis stream and consumer group. cfg: cfg, - promises: make(map[string]*containers.Promise[Response]), + promises: make(map[string]*MsgIdAndPromise[Response]), }, nil } -func (p *Producer[Request, Response]) errorPromisesFor(msgIds []string) { - p.promisesLock.Lock() - defer p.promisesLock.Unlock() - for _, msg := range msgIds { - if promise, found := p.promises[msg]; found { - promise.ProduceError(fmt.Errorf("internal error, consumer died while serving the request")) - delete(p.promises, msg) - } - } -} - -// checkAndReproduce reproduce pending messages that were sent to consumers -// that are currently inactive. -func (p *Producer[Request, Response]) checkAndReproduce(ctx context.Context) time.Duration { - staleIds, err := p.checkPending(ctx) - if err != nil { - log.Error("Checking pending messages", "error", err) - return p.cfg.CheckPendingInterval - } - if len(staleIds) == 0 { - return p.cfg.CheckPendingInterval - } - if p.cfg.EnableReproduce { - err = p.reproduceIds(ctx, staleIds) - if err != nil { - log.Warn("filed reproducing messages", "err", err) - } - } else { - p.errorPromisesFor(staleIds) - } - return p.cfg.CheckPendingInterval -} - -func (p *Producer[Request, Response]) reproduceIds(ctx context.Context, staleIds []string) error { - log.Info("Attempting to claim", "messages", staleIds) - claimedMsgs, err := p.client.XClaim(ctx, &redis.XClaimArgs{ - Stream: p.redisStream, - Group: p.redisGroup, - Consumer: p.id, - MinIdle: p.cfg.KeepAliveTimeout, - Messages: staleIds, - }).Result() - if err != nil { - return fmt.Errorf("claiming ownership on messages: %v, error: %w", staleIds, err) - } - for _, msg := range claimedMsgs { - data, ok := (msg.Values[messageKey]).(string) - if !ok { - log.Error("redis producer reproduce: message not string", "id", msg.ID, "value", msg.Values[messageKey]) - continue - } - var req Request - if err := json.Unmarshal([]byte(data), &req); err != nil { - log.Error("redis producer reproduce: message not a request", "id", msg.ID, "err", err, "value", msg.Values[messageKey]) - continue - } - if _, err := p.client.XAck(ctx, p.redisStream, p.redisGroup, msg.ID).Result(); err != nil { - log.Error("redis producer reproduce: could not ACK", "id", msg.ID, "err", err) - continue - } - // Only re-insert messages that were removed the the pending list first. - if _, err := p.reproduce(ctx, req, msg.ID); err != nil { - log.Error("redis producer reproduce: error", "err", err) - } - } - return nil -} - -func setMinIdInt(min *[2]uint64, id string) error { - idParts := strings.Split(id, "-") +func setMaxMsgIdInt(maxMsgIdInt *[2]uint64, msgId string) error { + idParts := strings.Split(msgId, "-") if len(idParts) != 2 { - return fmt.Errorf("invalid i.d: %v", id) + return fmt.Errorf("invalid i.d: %v", msgId) } idTimeStamp, err := strconv.ParseUint(idParts[0], 10, 64) if err != nil { - return fmt.Errorf("invalid i.d: %v err: %w", id, err) + return fmt.Errorf("invalid i.d: %v err: %w", msgId, err) } - if idTimeStamp > min[0] { + if idTimeStamp < maxMsgIdInt[0] { return nil } idSerial, err := strconv.ParseUint(idParts[1], 10, 64) if err != nil { - return fmt.Errorf("invalid i.d serial: %v err: %w", id, err) + return fmt.Errorf("invalid i.d serial: %v err: %w", msgId, err) } - if idTimeStamp < min[0] { - min[0] = idTimeStamp - min[1] = idSerial + if idTimeStamp > maxMsgIdInt[0] { + maxMsgIdInt[0] = idTimeStamp + maxMsgIdInt[1] = idSerial return nil } - // idTimeStamp == min[0] - if idSerial < min[1] { - min[1] = idSerial + // idTimeStamp == maxMsgIdInt[0] + if idSerial > maxMsgIdInt[1] { + maxMsgIdInt[1] = idSerial } return nil } // checkResponses checks iteratively whether response for the promise is ready. func (p *Producer[Request, Response]) checkResponses(ctx context.Context) time.Duration { - minIdInt := [2]uint64{math.MaxUint64, math.MaxUint64} + maxMsgIdInt := [2]uint64{0, 0} p.promisesLock.Lock() defer p.promisesLock.Unlock() responded := 0 errored := 0 - for id, promise := range p.promises { + for id, msgIDAndPromise := range p.promises { if ctx.Err() != nil { return 0 } - res, err := p.client.Get(ctx, id).Result() + msgKey := MessageKeyFor(p.redisStream, id) + res, err := p.client.Get(ctx, msgKey).Result() if err != nil { - errSetId := setMinIdInt(&minIdInt, id) - if errSetId != nil { - log.Error("error setting minId", "err", err) - return p.cfg.CheckResultInterval - } if !errors.Is(err, redis.Nil) { log.Error("Error reading value in redis", "key", id, "error", err) } continue } + // We keep track of a maxMsgId of a successfully solved request, because messages + // with id lower than this are either ack-ed or in PEL, so its safe to call XTRIMMINID on maxMsgId + errSetId := setMaxMsgIdInt(&maxMsgIdInt, msgIDAndPromise.msgID) + if errSetId != nil { + log.Error("error setting maxMsgId", "err", err) + return p.cfg.CheckResultInterval + } var resp Response if err := json.Unmarshal([]byte(res), &resp); err != nil { - promise.ProduceError(fmt.Errorf("error unmarshalling: %w", err)) + msgIDAndPromise.promise.ProduceError(fmt.Errorf("error unmarshalling: %w", err)) log.Error("Error unmarshaling", "value", res, "error", err) errored++ } else { - promise.Produce(resp) + msgIDAndPromise.promise.Produce(resp) responded++ } + // Try deleting UNIQUEID_MSGID_MAP_KEY corresponding to this id from redis + if err := p.client.Del(ctx, msgKey+UNIQUEID_MSGID_MAP_KEY).Err(); err != nil { + log.Error("Error deleting key from redis that flags that a request is being processed", "err", err) + } delete(p.promises, id) } var trimmed int64 var trimErr error - minId := "+" - if minIdInt[0] < math.MaxUint64 { - minId = fmt.Sprintf("%d-%d", minIdInt[0], minIdInt[1]) - trimmed, trimErr = p.client.XTrimMinID(ctx, p.redisStream, minId).Result() - } else { - trimmed, trimErr = p.client.XTrimMaxLen(ctx, p.redisStream, 0).Result() + maxMsgId := "+" + // If at least response for one promise was found, find the maximum of the found ones and XTRIMMINID from that msg id + 1 + if maxMsgIdInt[0] > 0 { + maxMsgId = fmt.Sprintf("%d-%d", maxMsgIdInt[0], maxMsgIdInt[1]+1) + trimmed, trimErr = p.client.XTrimMinID(ctx, p.redisStream, maxMsgId).Result() } - log.Trace("trimming", "id", minId, "trimmed", trimmed, "responded", responded, "errored", errored, "trim-err", trimErr) + log.Trace("trimming", "xTrimMinID", maxMsgId, "trimmed", trimmed, "responded", responded, "errored", errored, "trim-err", trimErr) return p.cfg.CheckResultInterval } @@ -259,101 +182,77 @@ func (p *Producer[Request, Response]) promisesLen() int { return len(p.promises) } -// reproduce is used when Producer claims ownership on the pending -// message that was sent to inactive consumer and reinserts it into the stream, -// so that seamlessly return the answer in the same promise. -func (p *Producer[Request, Response]) reproduce(ctx context.Context, value Request, oldKey string) (*containers.Promise[Response], error) { +func (p *Producer[Request, Response]) produce(ctx context.Context, id string, value Request) (*containers.Promise[Response], error) { + if id != "" { + msgKey := MessageKeyFor(p.redisStream, id) + + // If the request has already been solved by a consumer + if res, err := p.client.Get(ctx, msgKey).Result(); err == nil { + var resp Response + if err := json.Unmarshal([]byte(res), &resp); err != nil { + log.Error("Error unmarshaling", "value", res, "error", err) + return nil, fmt.Errorf("error unmarshalling: %w", err) + } else { + pr := containers.NewPromise[Response](nil) + pr.Produce(resp) + return &pr, nil + } + } else if !errors.Is(err, redis.Nil) { + log.Error("error while checking for response to a request in redis", "err", err) + } + + // Check for duplicate unsolved request messages in stream + if res, err := p.client.Get(ctx, msgKey+UNIQUEID_MSGID_MAP_KEY).Result(); err == nil { + log.Info("Request already submitted by another producer", "msgId", res, "requestUniqueId", id) + p.promisesLock.Lock() + defer p.promisesLock.Unlock() + pr := containers.NewPromise[Response](nil) + p.promises[id] = &MsgIdAndPromise[Response]{ + msgID: res, + promise: &pr, + } + return &pr, nil + } + } + val, err := json.Marshal(value) if err != nil { return nil, fmt.Errorf("marshaling value: %w", err) } - // catching the promiseLock before we sendXadd makes sure promise ids will - // be always ascending + // catching the promiseLock before we sendXadd makes sure promise ids will be always ascending p.promisesLock.Lock() defer p.promisesLock.Unlock() - id, err := p.client.XAdd(ctx, &redis.XAddArgs{ + msgId, err := p.client.XAdd(ctx, &redis.XAddArgs{ Stream: p.redisStream, Values: map[string]any{messageKey: val}, }).Result() if err != nil { return nil, fmt.Errorf("adding values to redis: %w", err) } - promise := p.promises[oldKey] - if oldKey != "" && promise == nil { - // This will happen if the old consumer became inactive but then ack_d - // the message afterwards. - // don't error - log.Warn("tried reproducing a message but it wasn't found - probably got response", "oldKey", oldKey) + + if id == "" { + // If unique id doesn't exist, use the newly created msgId as unique id and follow the same steps as before + log.Info("Request doesn't have a unique identifier (SelfHash field set), defaulting to using redis stream messageId", "msgId", msgId) + id = msgId } - if oldKey == "" || promise == nil { - pr := containers.NewPromise[Response](nil) - promise = &pr + + // Try adding key that flags that request is being processed + if err := p.client.Set(ctx, MessageKeyFor(p.redisStream, id)+UNIQUEID_MSGID_MAP_KEY, msgId, p.cfg.ResponseEntryTimeout).Err(); err != nil { + log.Error("Error adding key to redis that flags that a request is being processed, stream may encounter duplicate requests", "err", err) + } + + pr := containers.NewPromise[Response](nil) + p.promises[id] = &MsgIdAndPromise[Response]{ + msgID: msgId, + promise: &pr, } - delete(p.promises, oldKey) - p.promises[id] = promise - return promise, nil + return &pr, nil } -func (p *Producer[Request, Response]) Produce(ctx context.Context, value Request) (*containers.Promise[Response], error) { +func (p *Producer[Request, Response]) Produce(ctx context.Context, id string, value Request) (*containers.Promise[Response], error) { log.Debug("Redis stream producing", "value", value) p.once.Do(func() { - p.StopWaiter.CallIteratively(p.checkAndReproduce) p.StopWaiter.CallIteratively(p.checkResponses) }) - return p.reproduce(ctx, value, "") -} - -// Check if a consumer is with specified ID is alive. -func (p *Producer[Request, Response]) isConsumerAlive(ctx context.Context, consumerID string) bool { - if _, err := p.client.Get(ctx, heartBeatKey(consumerID)).Int64(); err != nil { - return false - } - return true -} - -func (p *Producer[Request, Response]) havePromiseFor(messageID string) bool { - p.promisesLock.Lock() - defer p.promisesLock.Unlock() - _, found := p.promises[messageID] - return found -} - -// returns ids of pending messages that's worker doesn't appear alive -func (p *Producer[Request, Response]) checkPending(ctx context.Context) ([]string, error) { - pendingMessages, err := p.client.XPendingExt(ctx, &redis.XPendingExtArgs{ - Stream: p.redisStream, - Group: p.redisGroup, - Start: "-", - End: "+", - Count: p.cfg.CheckPendingItems, - }).Result() - - if err != nil && !errors.Is(err, redis.Nil) { - return nil, fmt.Errorf("querying pending messages: %w", err) - } - if len(pendingMessages) == 0 { - return nil, nil - } - if len(pendingMessages) >= int(p.cfg.CheckPendingItems) { - log.Warn("redis producer: many pending items found", "stream", p.redisStream, "check-pending-items", p.cfg.CheckPendingItems) - } - // IDs of the pending messages with inactive consumers. - var ids []string - active := make(map[string]bool) - for _, msg := range pendingMessages { - // Ignore messages not produced by this producer. - if !p.havePromiseFor(msg.ID) { - continue - } - alive, found := active[msg.Consumer] - if !found { - alive = p.isConsumerAlive(ctx, msg.Consumer) - active[msg.Consumer] = alive - } - if alive { - continue - } - ids = append(ids, msg.ID) - } - return ids, nil + return p.produce(ctx, id, value) } diff --git a/pubsub/pubsub_test.go b/pubsub/pubsub_test.go index 9f774b6372..69839737e3 100644 --- a/pubsub/pubsub_test.go +++ b/pubsub/pubsub_test.go @@ -2,6 +2,9 @@ package pubsub import ( "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" "errors" "fmt" "os" @@ -10,11 +13,11 @@ import ( "time" "github.com/ethereum/go-ethereum/log" - "github.com/go-redis/redis/v8" "github.com/google/go-cmp/cmp" "github.com/google/uuid" "github.com/offchainlabs/nitro/util/containers" "github.com/offchainlabs/nitro/util/redisutil" + "github.com/redis/go-redis/v9" ) var ( @@ -23,7 +26,18 @@ var ( ) type testRequest struct { - Request string + Request string + SelfHash string // Is a unique identifier which can be used to compare any two validationInputs +} + +// SetSelfHash should be only called once. In the context of redis streams- by the producer +func (t *testRequest) SetSelfHash() { + jsonData, err := json.Marshal(t) + if err != nil { + return + } + hash := sha256.Sum256(jsonData) + t.SelfHash = hex.EncodeToString(hash[:]) } type testResponse struct { @@ -45,36 +59,21 @@ func destroyRedisGroup(ctx context.Context, t *testing.T, streamName string, cli } } -type configOpt interface { - apply(consCfg *ConsumerConfig, prodCfg *ProducerConfig) -} - -type withReproduce struct { - reproduce bool -} - -func (e *withReproduce) apply(_ *ConsumerConfig, prodCfg *ProducerConfig) { - prodCfg.EnableReproduce = e.reproduce -} - func producerCfg() *ProducerConfig { return &ProducerConfig{ - EnableReproduce: TestProducerConfig.EnableReproduce, - CheckPendingInterval: TestProducerConfig.CheckPendingInterval, - KeepAliveTimeout: TestProducerConfig.KeepAliveTimeout, CheckResultInterval: TestProducerConfig.CheckResultInterval, - CheckPendingItems: TestProducerConfig.CheckPendingItems, + ResponseEntryTimeout: TestProducerConfig.ResponseEntryTimeout, } } func consumerCfg() *ConsumerConfig { return &ConsumerConfig{ ResponseEntryTimeout: TestConsumerConfig.ResponseEntryTimeout, - KeepAliveTimeout: TestConsumerConfig.KeepAliveTimeout, + IdletimeToAutoclaim: TestConsumerConfig.IdletimeToAutoclaim, } } -func newProducerConsumers(ctx context.Context, t *testing.T, opts ...configOpt) (redis.UniversalClient, string, *Producer[testRequest, testResponse], []*Consumer[testRequest, testResponse]) { +func newProducerConsumers(ctx context.Context, t *testing.T) (redis.UniversalClient, string, *Producer[testRequest, testResponse], []*Consumer[testRequest, testResponse]) { t.Helper() redisClient, err := redisutil.RedisClientFromURL(redisutil.CreateTestRedis(ctx, t)) if err != nil { @@ -82,9 +81,7 @@ func newProducerConsumers(ctx context.Context, t *testing.T, opts ...configOpt) } prodCfg, consCfg := producerCfg(), consumerCfg() streamName := fmt.Sprintf("stream:%s", uuid.NewString()) - for _, o := range opts { - o.apply(consCfg, prodCfg) - } + producer, err := NewProducer[testRequest, testResponse](redisClient, streamName, prodCfg) if err != nil { t.Fatalf("Error creating new producer: %v", err) @@ -102,13 +99,6 @@ func newProducerConsumers(ctx context.Context, t *testing.T, opts ...configOpt) t.Cleanup(func() { ctx := context.Background() destroyRedisGroup(ctx, t, streamName, producer.client) - var keys []string - for _, c := range consumers { - keys = append(keys, c.heartBeatKey()) - } - if _, err := producer.client.Del(ctx, keys...).Result(); err != nil { - log.Debug("Error deleting heartbeat keys", "error", err) - } }) return redisClient, streamName, producer, consumers } @@ -125,10 +115,13 @@ func msgForIndex(idx int) string { return fmt.Sprintf("msg: %d", idx) } -func wantMessages(n int) []string { +func wantMessages(n int, group string, withDuplicates bool) []string { var ret []string for i := 0; i < n; i++ { - ret = append(ret, msgForIndex(i)) + ret = append(ret, group+msgForIndex(i)) + if withDuplicates && i%3 == 0 { + ret = append(ret, msgForIndex(i)) + } } sort.Strings(ret) return ret @@ -143,10 +136,14 @@ func flatten(responses [][]string) []string { return ret } -func produceMessages(ctx context.Context, msgs []string, producer *Producer[testRequest, testResponse]) ([]*containers.Promise[testResponse], error) { +func produceMessages(ctx context.Context, msgs []string, producer *Producer[testRequest, testResponse], useUniqueIdentifier bool) ([]*containers.Promise[testResponse], error) { var promises []*containers.Promise[testResponse] - for i := 0; i < messagesCount; i++ { - promise, err := producer.Produce(ctx, testRequest{Request: msgs[i]}) + for i := 0; i < len(msgs); i++ { + req := testRequest{Request: msgs[i]} + if useUniqueIdentifier { + req.SetSelfHash() + } + promise, err := producer.Produce(ctx, req.SelfHash, req) if err != nil { return nil, err } @@ -198,7 +195,7 @@ func consume(ctx context.Context, t *testing.T, consumers []*Consumer[testReques } gotMessages[idx][res.ID] = res.Value.Request resp := fmt.Sprintf("result for: %v", res.ID) - if err := c.SetResult(ctx, res.ID, testResponse{Response: resp}); err != nil { + if err := c.SetResult(ctx, res.Value.SelfHash, res.ID, testResponse{Response: resp}); err != nil { t.Errorf("Error setting a result: %v", err) } wantResponses[idx] = append(wantResponses[idx], resp) @@ -208,40 +205,86 @@ func consume(ctx context.Context, t *testing.T, consumers []*Consumer[testReques return wantResponses } -func TestRedisProduce(t *testing.T) { +func TestRedisProduceComplex(t *testing.T) { log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelTrace, true))) t.Parallel() for _, tc := range []struct { - name string - killConsumers bool - autoRecover bool + name string + entries1Count int + entries2Count int + numProducers int + withDuplicates bool // If this is set, then every fourth entry (while generation) of each entries list is equal + killConsumers bool }{ { - name: "all consumers are active", - killConsumers: false, - autoRecover: false, + name: "one producer, all consumers are active", + entries1Count: messagesCount, + numProducers: 1, }, { - name: "some consumers killed, others should take over their work", + name: "one producer, some consumers killed, others should take over their work", + entries1Count: messagesCount, + numProducers: 1, killConsumers: true, - autoRecover: true, }, { - name: "some consumers killed, should return failure", - killConsumers: true, - autoRecover: false, + name: "two producers, all consumers are active, all unique entries", + entries1Count: 20, + entries2Count: 20, + numProducers: 2, + }, + { + name: "two producers, all consumers are active, some duplicate entries", + entries1Count: 20, + entries2Count: 20, + numProducers: 2, + withDuplicates: true, + }, + { + name: "two producers, some consumers killed, others should take over their work, some duplicate entries, unequal number of requests from producers", + entries1Count: messagesCount, + entries2Count: 2 * messagesCount, + numProducers: 2, + withDuplicates: true, + killConsumers: true, }, } { t.Run(tc.name, func(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - redisClient, streamName, producer, consumers := newProducerConsumers(ctx, t, &withReproduce{tc.autoRecover}) - producer.Start(ctx) - wantMsgs := wantMessages(messagesCount) - promises, err := produceMessages(ctx, wantMsgs, producer) - if err != nil { - t.Fatalf("Error producing messages: %v", err) + + var producers []*Producer[testRequest, testResponse] + redisClient, streamName, producer, consumers := newProducerConsumers(ctx, t) + producers = append(producers, producer) + if tc.numProducers == 2 { + producer, err := NewProducer[testRequest, testResponse](redisClient, streamName, producerCfg()) + if err != nil { + t.Fatalf("Error creating second producer: %v", err) + } + producers = append(producers, producer) } + + for _, producer := range producers { + producer.Start(ctx) + } + + var entries [][]string + if tc.numProducers == 2 { + entries = append(entries, wantMessages(tc.entries1Count, "1.", tc.withDuplicates)) + entries = append(entries, wantMessages(tc.entries2Count, "2.", tc.withDuplicates)) + } else { + entries = append(entries, wantMessages(tc.entries1Count, "", tc.withDuplicates)) + } + + var promises [][]*containers.Promise[testResponse] + for i := 0; i < tc.numProducers; i++ { + prs, err := produceMessages(ctx, entries[i], producers[i], tc.numProducers == 2) + if err != nil { + t.Fatalf("Error producing messages from producer%d: %v", i, err) + } + promises = append(promises, prs) + } + gotMessages := messagesMaps(len(consumers)) if tc.killConsumers { // Consumer messages in every third consumer but don't ack them to check @@ -252,40 +295,66 @@ func TestRedisProduce(t *testing.T) { if err != nil { t.Errorf("Error consuming message: %v", err) } - if !tc.autoRecover { - gotMessages[i][req.ID] = req.Value.Request + if req == nil { + t.Error("Didn't consume any message") } consumers[i].StopAndWait() } } + time.Sleep(time.Second) wantResponses := consume(ctx, t, consumers, gotMessages) - gotResponses, errIndexes := awaitResponses(ctx, promises) - if len(errIndexes) != 0 && tc.autoRecover { - t.Fatalf("Error awaiting responses: %v", errIndexes) + + var gotResponses []string + for i := 0; i < tc.numProducers; i++ { + grs, errIndexes := awaitResponses(ctx, promises[i]) + if len(errIndexes) != 0 { + t.Fatalf("Error awaiting responses from promises%d: %v", i, errIndexes) + } + gotResponses = append(gotResponses, grs...) } - producer.StopAndWait() + for _, c := range consumers { c.StopAndWait() } + got, err := mergeValues(gotMessages) if err != nil { t.Fatalf("mergeMaps() unexpected error: %v", err) } + + var combinedEntries []string + for i := 0; i < tc.numProducers; i++ { + combinedEntries = append(combinedEntries, entries[i]...) + } + wantMsgs := removeDuplicates(combinedEntries) if diff := cmp.Diff(wantMsgs, got); diff != "" { t.Errorf("Unexpected diff (-want +got):\n%s\n", diff) } + + // Consumers are not supposed to get duplicate requests + gotResponses = removeDuplicates(gotResponses) wantResp := flatten(wantResponses) - sort.Strings(gotResponses) if diff := cmp.Diff(wantResp, gotResponses); diff != "" { t.Errorf("Unexpected diff in responses:\n%s\n", diff) } - if cnt := producer.promisesLen(); cnt != 0 { - t.Errorf("Producer still has %d unfullfilled promises", cnt) + + // Check each producers all promises were responded to + for i := 0; i < tc.numProducers; i++ { + if cnt := producers[i].promisesLen(); cnt != 0 { + t.Errorf("Producer%d still has %d unfullfilled promises", i, cnt) + } } + // Trigger a trim - producer.checkResponses(ctx) + time.Sleep(time.Second) + for i := 0; i < tc.numProducers; i++ { + producers[i].checkResponses(ctx) + producers[i].StopAndWait() + } + + // Check that no messages remain in the stream msgs, err := redisClient.XRange(ctx, streamName, "-", "+").Result() if err != nil { t.Errorf("XRange failed: %v", err) @@ -297,6 +366,19 @@ func TestRedisProduce(t *testing.T) { } } +func removeDuplicates(list []string) []string { + capture := map[string]bool{} + var ret []string + for _, elem := range list { + if _, found := capture[elem]; !found { + ret = append(ret, elem) + capture[elem] = true + } + } + sort.Strings(ret) + return ret +} + // mergeValues merges maps from the slice and returns their values. // Returns and error if there exists duplicate key. func mergeValues(messages []map[string]string) ([]string, error) { diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 62053c17f1..4d864b6a77 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -19,7 +19,6 @@ import ( "testing" "time" - "github.com/go-redis/redis/v8" "github.com/offchainlabs/nitro/arbos" "github.com/offchainlabs/nitro/arbos/arbostypes" "github.com/offchainlabs/nitro/arbos/util" @@ -39,6 +38,7 @@ import ( "github.com/offchainlabs/nitro/validator/server_common" "github.com/offchainlabs/nitro/validator/valnode" rediscons "github.com/offchainlabs/nitro/validator/valnode/redis" + "github.com/redis/go-redis/v9" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi" diff --git a/system_tests/seq_coordinator_test.go b/system_tests/seq_coordinator_test.go index 1b8926a1b9..e9b2adabe8 100644 --- a/system_tests/seq_coordinator_test.go +++ b/system_tests/seq_coordinator_test.go @@ -12,7 +12,7 @@ import ( "testing" "time" - "github.com/go-redis/redis/v8" + "github.com/redis/go-redis/v9" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" diff --git a/util/redisutil/redis_coordinator.go b/util/redisutil/redis_coordinator.go index 59e3b0e0f9..c30a59cb8c 100644 --- a/util/redisutil/redis_coordinator.go +++ b/util/redisutil/redis_coordinator.go @@ -6,7 +6,7 @@ import ( "fmt" "strings" - "github.com/go-redis/redis/v8" + "github.com/redis/go-redis/v9" "github.com/ethereum/go-ethereum/log" diff --git a/util/redisutil/redisutil.go b/util/redisutil/redisutil.go index f89c250e9a..01ba836d5b 100644 --- a/util/redisutil/redisutil.go +++ b/util/redisutil/redisutil.go @@ -1,6 +1,6 @@ package redisutil -import "github.com/go-redis/redis/v8" +import "github.com/redis/go-redis/v9" func RedisClientFromURL(url string) (redis.UniversalClient, error) { if url == "" { diff --git a/validator/client/redis/producer.go b/validator/client/redis/producer.go index b3ad0f8839..7e2578d100 100644 --- a/validator/client/redis/producer.go +++ b/validator/client/redis/producer.go @@ -7,7 +7,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" - "github.com/go-redis/redis/v8" "github.com/offchainlabs/nitro/pubsub" "github.com/offchainlabs/nitro/util/containers" "github.com/offchainlabs/nitro/util/redisutil" @@ -15,6 +14,7 @@ import ( "github.com/offchainlabs/nitro/validator" "github.com/offchainlabs/nitro/validator/server_api" "github.com/offchainlabs/nitro/validator/server_common" + "github.com/redis/go-redis/v9" "github.com/spf13/pflag" ) @@ -125,7 +125,8 @@ func (c *ValidationClient) Launch(entry *validator.ValidationInput, moduleRoot c errPromise := containers.NewReadyPromise(validator.GoGlobalState{}, fmt.Errorf("no validation is configured for wasm root %v", moduleRoot)) return server_common.NewValRun(errPromise, moduleRoot) } - promise, err := producer.Produce(c.GetContext(), entry) + entry.SetSelfHash() + promise, err := producer.Produce(c.GetContext(), entry.SelfHash, entry) if err != nil { errPromise := containers.NewReadyPromise(validator.GoGlobalState{}, fmt.Errorf("error producing input: %w", err)) return server_common.NewValRun(errPromise, moduleRoot) diff --git a/validator/validation_entry.go b/validator/validation_entry.go index 133a67a8a8..326bbc355b 100644 --- a/validator/validation_entry.go +++ b/validator/validation_entry.go @@ -1,6 +1,10 @@ package validator import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "github.com/ethereum/go-ethereum/common" "github.com/offchainlabs/nitro/arbutil" ) @@ -21,4 +25,16 @@ type ValidationInput struct { DelayedMsg []byte StartState GoGlobalState DebugChain bool + + SelfHash string // Is a unique identifier which can be used to compare any two instances of validationInput +} + +// SetSelfHash should be only called once. In the context of redis streams- by the producer, before submitting a request +func (v *ValidationInput) SetSelfHash() { + jsonData, err := json.Marshal(v) + if err != nil { + return + } + hash := sha256.Sum256(jsonData) + v.SelfHash = hex.EncodeToString(hash[:]) } diff --git a/validator/valnode/redis/consumer.go b/validator/valnode/redis/consumer.go index fb7db1e870..13bf19ac43 100644 --- a/validator/valnode/redis/consumer.go +++ b/validator/valnode/redis/consumer.go @@ -99,7 +99,7 @@ func (s *ValidationServer) Start(ctx_in context.Context) { log.Error("Error validating", "request value", req.Value, "error", err) return 0 } - if err := c.SetResult(ctx, req.ID, res); err != nil { + if err := c.SetResult(ctx, req.Value.SelfHash, req.ID, res); err != nil { log.Error("Error setting result for request", "id", req.ID, "result", res, "error", err) return 0 } From e155ebb60c157af9ea7188a419b1495085262036 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Thu, 15 Aug 2024 21:27:47 +0530 Subject: [PATCH 034/489] trim acknotifiers map and use previous keepalive timeouts --- pubsub/consumer.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pubsub/consumer.go b/pubsub/consumer.go index 3adb571343..410c3c75f4 100644 --- a/pubsub/consumer.go +++ b/pubsub/consumer.go @@ -23,12 +23,12 @@ type ConsumerConfig struct { var DefaultConsumerConfig = ConsumerConfig{ ResponseEntryTimeout: time.Hour, - IdletimeToAutoclaim: 30 * time.Minute, + IdletimeToAutoclaim: 5 * time.Minute, } var TestConsumerConfig = ConsumerConfig{ ResponseEntryTimeout: time.Minute, - IdletimeToAutoclaim: time.Second, + IdletimeToAutoclaim: 30 * time.Millisecond, } func ConsumerConfigAddOptions(prefix string, f *pflag.FlagSet) { @@ -149,7 +149,7 @@ func (c *Consumer[Request, Response]) Consume(ctx context.Context) (*Message[Req case <-ctx.Done(): log.Info("Context done while claiming message to indicate hearbeat", "error", ctx.Err().Error()) return - case <-time.After(c.cfg.IdletimeToAutoclaim / 3): + case <-time.After(c.cfg.IdletimeToAutoclaim / 10): } } }) @@ -177,6 +177,9 @@ func (c *Consumer[Request, Response]) SetResult(ctx context.Context, id string, if _, err := c.client.XAck(ctx, c.redisStream, c.redisGroup, messageID).Result(); err != nil { return fmt.Errorf("acking message: %v, error: %w", messageID, err) } - close(c.ackNotifiers[messageID]) + if ackNotifier, found := c.ackNotifiers[messageID]; found { + close(ackNotifier) + delete(c.ackNotifiers, messageID) + } return nil } From 1473c602064d8818e7a7985630fabc1aa41714de Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Fri, 16 Aug 2024 10:58:13 +0530 Subject: [PATCH 035/489] Use faster hash function --- go.mod | 2 +- go.sum | 4 +-- pubsub/consumer.go | 41 ++++++++++++----------------- pubsub/pubsub_test.go | 6 +++-- validator/validation_entry.go | 7 +++-- validator/valnode/redis/consumer.go | 8 ++++-- 6 files changed, 33 insertions(+), 35 deletions(-) diff --git a/go.mod b/go.mod index 5453205c5d..3ba779ae62 100644 --- a/go.mod +++ b/go.mod @@ -80,7 +80,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.10.0 // indirect github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 github.com/cockroachdb/errors v1.9.1 // indirect github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect github.com/cockroachdb/redact v1.1.3 // indirect diff --git a/go.sum b/go.sum index bf0b385631..79d9263076 100644 --- a/go.sum +++ b/go.sum @@ -151,8 +151,8 @@ github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= diff --git a/pubsub/consumer.go b/pubsub/consumer.go index 410c3c75f4..9c3785ee3f 100644 --- a/pubsub/consumer.go +++ b/pubsub/consumer.go @@ -40,12 +40,11 @@ func ConsumerConfigAddOptions(prefix string, f *pflag.FlagSet) { // indicate it is alive. type Consumer[Request any, Response any] struct { stopwaiter.StopWaiter - id string - client redis.UniversalClient - redisStream string - redisGroup string - cfg *ConsumerConfig - ackNotifiers map[string]chan struct{} + id string + client redis.UniversalClient + redisStream string + redisGroup string + cfg *ConsumerConfig } type Message[Request any] struct { @@ -58,12 +57,11 @@ func NewConsumer[Request any, Response any](client redis.UniversalClient, stream return nil, fmt.Errorf("redis stream name cannot be empty") } return &Consumer[Request, Response]{ - id: uuid.NewString(), - client: client, - redisStream: streamName, - redisGroup: streamName, // There is 1-1 mapping of redis stream and consumer group. - cfg: cfg, - ackNotifiers: make(map[string]chan struct{}), + id: uuid.NewString(), + client: client, + redisStream: streamName, + redisGroup: streamName, // There is 1-1 mapping of redis stream and consumer group. + cfg: cfg, }, nil } @@ -86,7 +84,7 @@ func (c *Consumer[Request, Response]) StreamName() string { // Consumer first checks it there exists pending message that is claimed by // unresponsive consumer, if not then reads from the stream. -func (c *Consumer[Request, Response]) Consume(ctx context.Context) (*Message[Request], error) { +func (c *Consumer[Request, Response]) Consume(ctx context.Context) (*Message[Request], chan struct{}, error) { // First try to XAUTOCLAIM, this prioritizes processing PEL messages // that have been waiting for more than IdletimeToAutoclaim duration messages, _, err := c.client.XAutoClaim(ctx, &redis.XAutoClaimArgs{ @@ -109,13 +107,13 @@ func (c *Consumer[Request, Response]) Consume(ctx context.Context) (*Message[Req Block: time.Millisecond, // 0 seems to block the read instead of immediately returning }).Result() if errors.Is(err, redis.Nil) { - return nil, nil + return nil, nil, nil } if err != nil { - return nil, fmt.Errorf("reading message for consumer: %q: %w", c.id, err) + return nil, nil, fmt.Errorf("reading message for consumer: %q: %w", c.id, err) } if len(res) != 1 || len(res[0].Messages) != 1 { - return nil, fmt.Errorf("redis returned entries: %+v, for querying single message", res) + return nil, nil, fmt.Errorf("redis returned entries: %+v, for querying single message", res) } messages = res[0].Messages } @@ -125,11 +123,11 @@ func (c *Consumer[Request, Response]) Consume(ctx context.Context) (*Message[Req data, ok = (value).(string) ) if !ok { - return nil, fmt.Errorf("casting request to string: %w", err) + return nil, nil, fmt.Errorf("casting request to string: %w", err) } var req Request if err := json.Unmarshal([]byte(data), &req); err != nil { - return nil, fmt.Errorf("unmarshaling value: %v, error: %w", value, err) + return nil, nil, fmt.Errorf("unmarshaling value: %v, error: %w", value, err) } ackNotifier := make(chan struct{}) c.StopWaiter.LaunchThread(func(ctx context.Context) { @@ -153,12 +151,11 @@ func (c *Consumer[Request, Response]) Consume(ctx context.Context) (*Message[Req } } }) - c.ackNotifiers[messages[0].ID] = ackNotifier log.Debug("Redis stream consuming", "consumer_id", c.id, "message_id", messages[0].ID) return &Message[Request]{ ID: messages[0].ID, Value: req, - }, nil + }, ackNotifier, nil } func (c *Consumer[Request, Response]) SetResult(ctx context.Context, id string, messageID string, result Response) error { @@ -177,9 +174,5 @@ func (c *Consumer[Request, Response]) SetResult(ctx context.Context, id string, if _, err := c.client.XAck(ctx, c.redisStream, c.redisGroup, messageID).Result(); err != nil { return fmt.Errorf("acking message: %v, error: %w", messageID, err) } - if ackNotifier, found := c.ackNotifiers[messageID]; found { - close(ackNotifier) - delete(c.ackNotifiers, messageID) - } return nil } diff --git a/pubsub/pubsub_test.go b/pubsub/pubsub_test.go index 69839737e3..b1ffdca0fd 100644 --- a/pubsub/pubsub_test.go +++ b/pubsub/pubsub_test.go @@ -182,7 +182,7 @@ func consume(ctx context.Context, t *testing.T, consumers []*Consumer[testReques func(ctx context.Context) { for { - res, err := c.Consume(ctx) + res, ackNotifier, err := c.Consume(ctx) if err != nil { if !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, context.Canceled) { t.Errorf("Consume() unexpected error: %v", err) @@ -198,6 +198,7 @@ func consume(ctx context.Context, t *testing.T, consumers []*Consumer[testReques if err := c.SetResult(ctx, res.Value.SelfHash, res.ID, testResponse{Response: resp}); err != nil { t.Errorf("Error setting a result: %v", err) } + close(ackNotifier) wantResponses[idx] = append(wantResponses[idx], resp) } }) @@ -291,13 +292,14 @@ func TestRedisProduceComplex(t *testing.T) { // that other consumers will claim ownership on those messages. for i := 0; i < len(consumers); i += 3 { consumers[i].Start(ctx) - req, err := consumers[i].Consume(ctx) + req, _, err := consumers[i].Consume(ctx) if err != nil { t.Errorf("Error consuming message: %v", err) } if req == nil { t.Error("Didn't consume any message") } + // Kills the actnotifier hence allowing XAUTOCLAIM consumers[i].StopAndWait() } diff --git a/validator/validation_entry.go b/validator/validation_entry.go index 326bbc355b..dc102a4055 100644 --- a/validator/validation_entry.go +++ b/validator/validation_entry.go @@ -1,10 +1,10 @@ package validator import ( - "crypto/sha256" - "encoding/hex" "encoding/json" + "fmt" + "github.com/cespare/xxhash/v2" "github.com/ethereum/go-ethereum/common" "github.com/offchainlabs/nitro/arbutil" ) @@ -35,6 +35,5 @@ func (v *ValidationInput) SetSelfHash() { if err != nil { return } - hash := sha256.Sum256(jsonData) - v.SelfHash = hex.EncodeToString(hash[:]) + v.SelfHash = fmt.Sprintf("%d", xxhash.Sum64(jsonData)) } diff --git a/validator/valnode/redis/consumer.go b/validator/valnode/redis/consumer.go index 13bf19ac43..e5a5dae1df 100644 --- a/validator/valnode/redis/consumer.go +++ b/validator/valnode/redis/consumer.go @@ -84,7 +84,7 @@ func (s *ValidationServer) Start(ctx_in context.Context) { case <-ready: // Wait until the stream exists and start consuming iteratively. } s.StopWaiter.CallIteratively(func(ctx context.Context) time.Duration { - req, err := c.Consume(ctx) + req, ackNotifier, err := c.Consume(ctx) if err != nil { log.Error("Consuming request", "error", err) return 0 @@ -97,9 +97,13 @@ func (s *ValidationServer) Start(ctx_in context.Context) { res, err := valRun.Await(ctx) if err != nil { log.Error("Error validating", "request value", req.Value, "error", err) + close(ackNotifier) return 0 } - if err := c.SetResult(ctx, req.Value.SelfHash, req.ID, res); err != nil { + err = c.SetResult(ctx, req.Value.SelfHash, req.ID, res) + // Even in error we close ackNotifier as there's no retry mechanism here and closing it will alow other consumers to autoclaim + close(ackNotifier) + if err != nil { log.Error("Error setting result for request", "id", req.ID, "result", res, "error", err) return 0 } From e9bf293ea30261981efe6d565dfd40457214530c Mon Sep 17 00:00:00 2001 From: Pepper Lebeck-Jobe Date: Tue, 20 Aug 2024 10:42:26 -0400 Subject: [PATCH 036/489] Use the strongly-typed rawdb.Target --- staker/stateless_block_validator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/staker/stateless_block_validator.go b/staker/stateless_block_validator.go index 56e41a9c20..0a979ce96b 100644 --- a/staker/stateless_block_validator.go +++ b/staker/stateless_block_validator.go @@ -473,7 +473,7 @@ func (v *StatelessBlockValidator) RecordValidationInput(ctx context.Context, pos if validator.SpawnerSupportsModule(spawner, moduleRoot) { found = true // Hardcoded to use wavm so that it can be read by the prover. - input, err := entry.ToInput([]string{"wavm"}) + input, err := entry.ToInput([]rawdb.Target{rawdb.TargetWavm}) if err != nil { return err } From 04413ab92e1eff8b92675ecd3bab9b3aef9bba5a Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Thu, 22 Aug 2024 09:14:09 +0530 Subject: [PATCH 037/489] minor fix --- arbos/programs/api.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/arbos/programs/api.go b/arbos/programs/api.go index f520317cf0..504289322f 100644 --- a/arbos/programs/api.go +++ b/arbos/programs/api.go @@ -63,7 +63,6 @@ func newApiClosures( actingAddress := contract.Address() // not necessarily WASM readOnly := interpreter.ReadOnly() evm := interpreter.Evm() - depth := evm.Depth() db := evm.StateDB chainConfig := evm.ChainConfig() @@ -129,11 +128,6 @@ func newApiClosures( startGas := am.SaturatingUSub(gasLeft, baseCost) * 63 / 64 gas := am.MinInt(startGas, gasReq) - // Tracing: emit the call (value transfer is done later in evm.Call) - if tracingInfo != nil && tracingInfo.Tracer.OnOpcode != nil { - tracingInfo.Tracer.OnOpcode(0, byte(opcode), startGas, baseCost+gas, scope, []byte{}, depth, nil) - } - // EVM rule: calls that pay get a stipend (opCall) if value.Sign() != 0 { gas = am.SaturatingUAdd(gas, params.CallStipend) @@ -199,11 +193,6 @@ func newApiClosures( one64th := gas / 64 gas -= one64th - // Tracing: emit the create - if tracingInfo != nil && tracingInfo.Tracer.OnOpcode != nil { - tracingInfo.Tracer.OnOpcode(0, byte(opcode), startGas, baseCost+gas, scope, []byte{}, depth, nil) - } - var res []byte var addr common.Address // zero on failure var returnGas uint64 From 282d73c99fd4c00a00f4cfc400ae5819ff7b30a1 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Thu, 22 Aug 2024 09:17:30 +0530 Subject: [PATCH 038/489] Changes based on PR comments --- arbos/util/transfer.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/arbos/util/transfer.go b/arbos/util/transfer.go index 1240928eb6..98ba0ea0bc 100644 --- a/arbos/util/transfer.go +++ b/arbos/util/transfer.go @@ -8,13 +8,13 @@ import ( "errors" "fmt" "github.com/ethereum/go-ethereum/core/tracing" + "github.com/offchainlabs/nitro/util/arbmath" "math/big" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/log" "github.com/holiman/uint256" - "github.com/offchainlabs/nitro/util/arbmath" ) // TransferBalance represents a balance change occurring aside from a call. @@ -29,20 +29,6 @@ func TransferBalance( if amount.Sign() < 0 { panic(fmt.Sprintf("Tried to transfer negative amount %v from %v to %v", amount, from, to)) } - if from != nil { - balance := evm.StateDB.GetBalance(*from) - if arbmath.BigLessThan(balance.ToBig(), amount) { - return fmt.Errorf("%w: addr %v have %v want %v", vm.ErrInsufficientBalance, *from, balance, amount) - } - evm.StateDB.SubBalance(*from, uint256.MustFromBig(amount), tracing.BalanceChangeTransfer) - if evm.Context.ArbOSVersion >= 30 { - // ensure the from account is "touched" for EIP-161 - evm.StateDB.AddBalance(*from, &uint256.Int{}, tracing.BalanceChangeTransfer) - } - } - if to != nil { - evm.StateDB.AddBalance(*to, uint256.MustFromBig(amount), tracing.BalanceChangeTransfer) - } if tracer := evm.Config.Tracer; tracer != nil { if evm.Depth() != 0 && scenario != TracingDuringEVM { // A non-zero depth implies this transfer is occurring inside EVM execution @@ -70,6 +56,20 @@ func TransferBalance( } info.MockCall([]byte{}, 0, *from, *to, amount) } + if from != nil { + balance := evm.StateDB.GetBalance(*from) + if arbmath.BigLessThan(balance.ToBig(), amount) { + return fmt.Errorf("%w: addr %v have %v want %v", vm.ErrInsufficientBalance, *from, balance, amount) + } + evm.StateDB.SubBalance(*from, uint256.MustFromBig(amount), tracing.BalanceChangeTransfer) + if evm.Context.ArbOSVersion >= 30 { + // ensure the from account is "touched" for EIP-161 + evm.StateDB.AddBalance(*from, &uint256.Int{}, tracing.BalanceChangeTransfer) + } + } + if to != nil { + evm.StateDB.AddBalance(*to, uint256.MustFromBig(amount), tracing.BalanceChangeTransfer) + } return nil } From 605a137414289a02111fd4c66e4a3696e0744cfd Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Thu, 22 Aug 2024 16:23:08 +0530 Subject: [PATCH 039/489] Changes based on PR comments --- arbos/util/transfer.go | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/arbos/util/transfer.go b/arbos/util/transfer.go index 98ba0ea0bc..32ef4d1bd0 100644 --- a/arbos/util/transfer.go +++ b/arbos/util/transfer.go @@ -7,14 +7,14 @@ package util import ( "errors" "fmt" - "github.com/ethereum/go-ethereum/core/tracing" - "github.com/offchainlabs/nitro/util/arbmath" "math/big" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/log" "github.com/holiman/uint256" + "github.com/offchainlabs/nitro/util/arbmath" ) // TransferBalance represents a balance change occurring aside from a call. @@ -38,23 +38,24 @@ func TransferBalance( if scenario != TracingDuringEVM { tracer.CaptureArbitrumTransfer(from, to, amount, scenario == TracingBeforeEVM, purpose) - return nil - } - - if from == nil { - from = &common.Address{} - } - if to == nil { - to = &common.Address{} - } + } else { + fromCopy := from + toCopy := to + if fromCopy == nil { + fromCopy = &common.Address{} + } + if toCopy == nil { + toCopy = &common.Address{} + } - info := &TracingInfo{ - Tracer: evm.Config.Tracer, - Scenario: scenario, - Contract: vm.NewContract(addressHolder{*to}, addressHolder{*from}, uint256.NewInt(0), 0), - Depth: evm.Depth(), + info := &TracingInfo{ + Tracer: evm.Config.Tracer, + Scenario: scenario, + Contract: vm.NewContract(addressHolder{*toCopy}, addressHolder{*fromCopy}, uint256.NewInt(0), 0), + Depth: evm.Depth(), + } + info.MockCall([]byte{}, 0, *fromCopy, *toCopy, amount) } - info.MockCall([]byte{}, 0, *from, *to, amount) } if from != nil { balance := evm.StateDB.GetBalance(*from) From 39cd0a7a89159680b63fc07d30fa4831c55895b3 Mon Sep 17 00:00:00 2001 From: Pepper Lebeck-Jobe Date: Thu, 22 Aug 2024 11:04:16 -0400 Subject: [PATCH 040/489] Finish wiring the json file writing everywhere. At this point the location of the output file is hardcoded to be in the `$HOME/.arbitrum` directory. Hopefully, this will be okay. --- arbnode/api.go | 16 ++++++++++++++ staker/block_validator.go | 15 ++++++------- staker/stateless_block_validator.go | 27 ++++++----------------- system_tests/common_test.go | 17 ++++++++++++++ system_tests/program_norace_test.go | 13 ----------- system_tests/validation_mock_test.go | 4 ---- validator/client/validation_client.go | 8 ------- validator/interface.go | 1 - validator/server_api/json.go | 24 ++++++++++++++++++-- validator/server_arb/validator_spawner.go | 16 -------------- validator/valnode/validation_api.go | 9 -------- 11 files changed, 69 insertions(+), 81 deletions(-) diff --git a/arbnode/api.go b/arbnode/api.go index 228ad51cf8..27d3509d59 100644 --- a/arbnode/api.go +++ b/arbnode/api.go @@ -10,6 +10,7 @@ import ( "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/staker" "github.com/offchainlabs/nitro/validator" + "github.com/offchainlabs/nitro/validator/server_api" ) type BlockValidatorAPI struct { @@ -54,3 +55,18 @@ func (a *BlockValidatorDebugAPI) ValidateMessageNumber( result.Valid = valid return result, err } + +func (a *BlockValidatorDebugAPI) ValidationInputsAt(ctx context.Context, msgNum hexutil.Uint64, moduleRootOptional *common.Hash, +) (server_api.InputJSON, error) { + var moduleRoot common.Hash + if moduleRootOptional != nil { + moduleRoot = *moduleRootOptional + } else { + var err error + moduleRoot, err = a.val.GetLatestWasmModuleRoot(ctx) + if err != nil { + return server_api.InputJSON{}, fmt.Errorf("no latest WasmModuleRoot configured, must provide parameter: %w", err) + } + } + return a.val.ValidationInputsAt(ctx, arbutil.MessageIndex(msgNum), moduleRoot) +} diff --git a/staker/block_validator.go b/staker/block_validator.go index 4be65c8d1d..f91d91d2e5 100644 --- a/staker/block_validator.go +++ b/staker/block_validator.go @@ -29,6 +29,7 @@ import ( "github.com/offchainlabs/nitro/util/stopwaiter" "github.com/offchainlabs/nitro/validator" "github.com/offchainlabs/nitro/validator/client/redis" + "github.com/offchainlabs/nitro/validator/server_api" "github.com/spf13/pflag" ) @@ -504,18 +505,16 @@ func (v *BlockValidator) sendRecord(s *validationStatus) error { } //nolint:gosec -func (v *BlockValidator) writeToFile(validationEntry *validationEntry, moduleRoot common.Hash) error { +func (v *BlockValidator) writeToFile(validationEntry *validationEntry) error { input, err := validationEntry.ToInput([]rawdb.Target{rawdb.TargetWavm}) if err != nil { return err } - for _, spawner := range v.execSpawners { - if validator.SpawnerSupportsModule(spawner, moduleRoot) { - _, err = spawner.WriteToFile(input, moduleRoot).Await(v.GetContext()) - return err - } + inputJson := server_api.ValidationInputToJson(input) + if err := inputJson.WriteToFile("BlockValidator"); err != nil { + return err } - return errors.New("did not find exec spawner for wasmModuleRoot") + return nil } func (v *BlockValidator) SetCurrentWasmModuleRoot(hash common.Hash) error { @@ -783,7 +782,7 @@ validationsLoop: runEnd, err := run.Current() if err == nil && runEnd != validationStatus.Entry.End { err = fmt.Errorf("validation failed: expected %v got %v", validationStatus.Entry.End, runEnd) - writeErr := v.writeToFile(validationStatus.Entry, run.WasmModuleRoot()) + writeErr := v.writeToFile(validationStatus.Entry) if writeErr != nil { log.Warn("failed to write debug results file", "err", writeErr) } diff --git a/staker/stateless_block_validator.go b/staker/stateless_block_validator.go index 0a979ce96b..bb96e6f8c2 100644 --- a/staker/stateless_block_validator.go +++ b/staker/stateless_block_validator.go @@ -24,6 +24,7 @@ import ( "github.com/offchainlabs/nitro/util/rpcclient" "github.com/offchainlabs/nitro/validator" "github.com/offchainlabs/nitro/validator/client/redis" + "github.com/offchainlabs/nitro/validator/server_api" validatorclient "github.com/offchainlabs/nitro/validator/client" ) @@ -463,30 +464,16 @@ func (v *StatelessBlockValidator) ValidateResult( return true, &entry.End, nil } -func (v *StatelessBlockValidator) RecordValidationInput(ctx context.Context, pos arbutil.MessageIndex, moduleRoot common.Hash) error { +func (v *StatelessBlockValidator) ValidationInputsAt(ctx context.Context, pos arbutil.MessageIndex, moduleRoot common.Hash) (server_api.InputJSON, error) { entry, err := v.CreateReadyValidationEntry(ctx, pos) if err != nil { - return err + return server_api.InputJSON{}, err } - found := false - for _, spawner := range v.execSpawners { - if validator.SpawnerSupportsModule(spawner, moduleRoot) { - found = true - // Hardcoded to use wavm so that it can be read by the prover. - input, err := entry.ToInput([]rawdb.Target{rawdb.TargetWavm}) - if err != nil { - return err - } - _, err = spawner.WriteToFile(input, moduleRoot).Await(ctx) - if err != nil { - return err - } - } - } - if !found { - return fmt.Errorf("validation with WasmModuleRoot %v not supported by node", moduleRoot) + input, err := entry.ToInput([]rawdb.Target{rawdb.TargetWavm}) + if err != nil { + return server_api.InputJSON{}, err } - return nil + return *server_api.ValidationInputToJson(input), nil } func (v *StatelessBlockValidator) OverrideRecorder(t *testing.T, recorder execution.ExecutionRecorder) { diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 6e7375a921..53d17c762c 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -1459,6 +1459,23 @@ func logParser[T any](t *testing.T, source string, name string) func(*types.Log) } } +// recordBlock writes a json file with all of the data needed to validate a block. +// +// This can be used as an input to the arbitrator prover to validate a block. +func recordBlock(t *testing.T, block uint64, builder *NodeBuilder) { + t.Helper() + ctx := builder.ctx + wasmModuleRoot := currentRootModule(t) + inboxPos := arbutil.MessageIndex(block) + inputJson, err := builder.L2.ConsensusNode.StatelessBlockValidator.ValidationInputsAt(ctx, inboxPos, wasmModuleRoot) + if err != nil { + Fatal(t, "failed to get validation inputs", block, err) + } + if err := inputJson.WriteToFile(t.Name()); err != nil { + Fatal(t, "failed to write validation inputs", block, err) + } +} + func populateMachineDir(t *testing.T, cr *github.ConsensusRelease) string { baseDir := t.TempDir() machineDir := baseDir + "/machines" diff --git a/system_tests/program_norace_test.go b/system_tests/program_norace_test.go index 2ccd77c5b5..56b2046716 100644 --- a/system_tests/program_norace_test.go +++ b/system_tests/program_norace_test.go @@ -104,19 +104,6 @@ func validateBlockRange( } } -// recordBlock writes a json file with all of the data needed to validate a block. -// -// This can be used as an input to the arbitrator prover to validate a block. -func recordBlock(t *testing.T, block uint64, builder *NodeBuilder) { - t.Helper() - ctx := builder.ctx - wasmModuleRoot := currentRootModule(t) - inboxPos := arbutil.MessageIndex(block) - if err := builder.L2.ConsensusNode.StatelessBlockValidator.RecordValidationInput(ctx, inboxPos, wasmModuleRoot); err != nil { - Fatal(t, "failed to record block", block, err) - } -} - func TestProgramEvmData(t *testing.T) { t.Parallel() testEvmData(t, true) diff --git a/system_tests/validation_mock_test.go b/system_tests/validation_mock_test.go index b854ec1cb3..ddf0f4c3aa 100644 --- a/system_tests/validation_mock_test.go +++ b/system_tests/validation_mock_test.go @@ -96,10 +96,6 @@ func (s *mockSpawner) LatestWasmModuleRoot() containers.PromiseInterface[common. return containers.NewReadyPromise[common.Hash](mockWasmModuleRoots[0], nil) } -func (s *mockSpawner) WriteToFile(input *validator.ValidationInput, moduleRoot common.Hash) containers.PromiseInterface[struct{}] { - return containers.NewReadyPromise[struct{}](struct{}{}, nil) -} - type mockValRun struct { containers.Promise[validator.GoGlobalState] root common.Hash diff --git a/validator/client/validation_client.go b/validator/client/validation_client.go index 01e0684d70..8372731e06 100644 --- a/validator/client/validation_client.go +++ b/validator/client/validation_client.go @@ -186,14 +186,6 @@ func (c *ExecutionClient) LatestWasmModuleRoot() containers.PromiseInterface[com }) } -func (c *ExecutionClient) WriteToFile(input *validator.ValidationInput, moduleRoot common.Hash) containers.PromiseInterface[struct{}] { - jsonInput := server_api.ValidationInputToJson(input) - return stopwaiter.LaunchPromiseThread[struct{}](c, func(ctx context.Context) (struct{}, error) { - err := c.client.CallContext(ctx, nil, server_api.Namespace+"_writeToFile", jsonInput, moduleRoot) - return struct{}{}, err - }) -} - func (r *ExecutionClientRun) SendKeepAlive(ctx context.Context) time.Duration { err := r.client.client.CallContext(ctx, nil, server_api.Namespace+"_execKeepAlive", r.id) if err != nil { diff --git a/validator/interface.go b/validator/interface.go index 8c2991364a..4e759545e0 100644 --- a/validator/interface.go +++ b/validator/interface.go @@ -27,7 +27,6 @@ type ExecutionSpawner interface { ValidationSpawner CreateExecutionRun(wasmModuleRoot common.Hash, input *ValidationInput) containers.PromiseInterface[ExecutionRun] LatestWasmModuleRoot() containers.PromiseInterface[common.Hash] - WriteToFile(input *ValidationInput, moduleRoot common.Hash) containers.PromiseInterface[struct{}] } type ExecutionRun interface { diff --git a/validator/server_api/json.go b/validator/server_api/json.go index dbe2bb1fee..069ab014d9 100644 --- a/validator/server_api/json.go +++ b/validator/server_api/json.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "os" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" @@ -68,12 +69,31 @@ type InputJSON struct { DebugChain bool } -func (i *InputJSON) WriteToFile() error { +// WriteToFile writes the InputJSON to a file in JSON format. +// +// The path to the file is determined in part by the slug parameter so +// callers can provide a recognizable name to differentiate various +// contexts in which the InputJSON is being written. +// +// The file is created at a path +// +// $HOME/.arbuitrum/validation-inputs///block_inputs_.json +func (i *InputJSON) WriteToFile(slug string) error { + homeDir, err := os.UserHomeDir() + if err != nil { + return err + } + t := time.Now() + tStr := t.Format("20060102_150405") + dir := fmt.Sprintf("%s/.arbitrum/validation-inputs/%s/%s", homeDir, slug, tStr) + if err = os.MkdirAll(dir, 0700); err != nil { + return err + } contents, err := json.MarshalIndent(i, "", " ") if err != nil { return err } - if err = os.WriteFile(fmt.Sprintf("block_inputs_%d.json", i.Id), contents, 0600); err != nil { + if err = os.WriteFile(fmt.Sprintf("%s/block_inputs_%d.json", dir, i.Id), contents, 0600); err != nil { return err } return nil diff --git a/validator/server_arb/validator_spawner.go b/validator/server_arb/validator_spawner.go index ae8c8be9ab..74cf77fdf5 100644 --- a/validator/server_arb/validator_spawner.go +++ b/validator/server_arb/validator_spawner.go @@ -14,7 +14,6 @@ import ( "github.com/offchainlabs/nitro/util/containers" "github.com/offchainlabs/nitro/util/stopwaiter" "github.com/offchainlabs/nitro/validator" - "github.com/offchainlabs/nitro/validator/server_api" "github.com/offchainlabs/nitro/validator/server_common" "github.com/offchainlabs/nitro/validator/valnode/redis" @@ -203,21 +202,6 @@ func (v *ArbitratorSpawner) Room() int { return avail } -func (v *ArbitratorSpawner) writeToFile(_ context.Context, input *validator.ValidationInput, _ common.Hash) error { - jsonInput := server_api.ValidationInputToJson(input) - if err := jsonInput.WriteToFile(); err != nil { - return err - } - return nil -} - -func (v *ArbitratorSpawner) WriteToFile(input *validator.ValidationInput, moduleRoot common.Hash) containers.PromiseInterface[struct{}] { - return stopwaiter.LaunchPromiseThread[struct{}](v, func(ctx context.Context) (struct{}, error) { - err := v.writeToFile(ctx, input, moduleRoot) - return struct{}{}, err - }) -} - func (v *ArbitratorSpawner) CreateExecutionRun(wasmModuleRoot common.Hash, input *validator.ValidationInput) containers.PromiseInterface[validator.ExecutionRun] { getMachine := func(ctx context.Context) (MachineInterface, error) { initialFrozenMachine, err := v.machineLoader.GetZeroStepMachine(ctx, wasmModuleRoot) diff --git a/validator/valnode/validation_api.go b/validator/valnode/validation_api.go index 68d357695b..c36d8ee62c 100644 --- a/validator/valnode/validation_api.go +++ b/validator/valnode/validation_api.go @@ -118,15 +118,6 @@ func (a *ExecServerAPI) Start(ctx_in context.Context) { a.CallIteratively(a.removeOldRuns) } -func (a *ExecServerAPI) WriteToFile(ctx context.Context, jsonInput *server_api.InputJSON, moduleRoot common.Hash) error { - input, err := server_api.ValidationInputFromJson(jsonInput) - if err != nil { - return err - } - _, err = a.execSpawner.WriteToFile(input, moduleRoot).Await(ctx) - return err -} - var errRunNotFound error = errors.New("run not found") func (a *ExecServerAPI) getRun(id uint64) (validator.ExecutionRun, error) { From 5f40b1c58653120c151d329b7a3fc91818e7d6ec Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 12 Aug 2024 13:46:44 -0300 Subject: [PATCH 041/489] Creates wallet contract through Staker's DataPoster --- cmd/nitro/nitro.go | 2 +- staker/validatorwallet/contract.go | 90 ++++++++++++++++++++++-------- system_tests/fast_confirm_test.go | 8 +-- system_tests/staker_test.go | 4 +- 4 files changed, 74 insertions(+), 30 deletions(-) diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index 39f204980d..8c26b95a92 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -372,7 +372,7 @@ func mainImpl() int { log.Crit("error getting rollup addresses config", "err", err) } // #nosec G115 - addr, err := validatorwallet.GetValidatorWalletContract(ctx, deployInfo.ValidatorWalletCreator, int64(deployInfo.DeployedAt), l1TransactionOptsValidator, l1Reader, true) + addr, err := validatorwallet.GetValidatorWalletContract(ctx, deployInfo.ValidatorWalletCreator, int64(deployInfo.DeployedAt), l1TransactionOptsValidator, l1Reader, true, nil) if err != nil { log.Crit("error creating validator wallet contract", "error", err, "address", l1TransactionOptsValidator.From.Hex()) } diff --git a/staker/validatorwallet/contract.go b/staker/validatorwallet/contract.go index 77b403b669..89d4fef11d 100644 --- a/staker/validatorwallet/contract.go +++ b/staker/validatorwallet/contract.go @@ -26,8 +26,11 @@ import ( "github.com/offchainlabs/nitro/util/headerreader" ) -var validatorABI abi.ABI -var walletCreatedID common.Hash +var ( + validatorABI abi.ABI + validatorWalletCreatorABI abi.ABI + walletCreatedID common.Hash +) func init() { parsedValidator, err := abi.JSON(strings.NewReader(rollupgen.ValidatorWalletABI)) @@ -40,6 +43,7 @@ func init() { if err != nil { panic(err) } + validatorWalletCreatorABI = parsedValidatorWalletCreator walletCreatedID = parsedValidatorWalletCreator.Events["WalletCreated"].ID } @@ -179,6 +183,37 @@ func (v *Contract) executeTransaction(ctx context.Context, tx *types.Transaction return v.dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), *v.Address(), data, gas, auth.Value) } +// Exposed for tests +func (v *Contract) CreateWalletContract( + ctx context.Context, + validatorWalletFactoryAddr common.Address, +) (*types.Transaction, error) { + var initialExecutorAllowedDests []common.Address + txData, err := validatorWalletCreatorABI.Pack("createWallet", initialExecutorAllowedDests) + if err != nil { + return nil, err + } + + auth, err := v.getAuth(ctx, nil) + if err != nil { + return nil, err + } + + gas, err := gasForTxData( + ctx, + v.l1Reader, + auth, + &validatorWalletFactoryAddr, + txData, + v.getExtraGas, + ) + if err != nil { + return nil, fmt.Errorf("getting gas for tx data: %w", err) + } + + return v.dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), validatorWalletFactoryAddr, txData, gas, auth.Value) +} + func (v *Contract) populateWallet(ctx context.Context, createIfMissing bool) error { if v.con != nil { return nil @@ -194,7 +229,7 @@ func (v *Contract) populateWallet(ctx context.Context, createIfMissing bool) err if err != nil { return err } - addr, err := GetValidatorWalletContract(ctx, v.walletFactoryAddr, v.rollupFromBlock, auth, v.l1Reader, createIfMissing) + addr, err := GetValidatorWalletContract(ctx, v.walletFactoryAddr, v.rollupFromBlock, auth, v.l1Reader, createIfMissing, v.CreateWalletContract) if err != nil { return err } @@ -295,25 +330,29 @@ func (v *Contract) ExecuteTransactions(ctx context.Context, builder *txbuilder.B return arbTx, nil } -func (v *Contract) estimateGas(ctx context.Context, value *big.Int, data []byte) (uint64, error) { - h, err := v.l1Reader.LastHeader(ctx) +func gasForTxData(ctx context.Context, l1Reader *headerreader.HeaderReader, auth *bind.TransactOpts, to *common.Address, data []byte, getExtraGas func() uint64) (uint64, error) { + if auth.GasLimit != 0 { + return auth.GasLimit, nil + } + + h, err := l1Reader.LastHeader(ctx) if err != nil { return 0, fmt.Errorf("getting the last header: %w", err) } gasFeeCap := new(big.Int).Mul(h.BaseFee, big.NewInt(2)) gasFeeCap = arbmath.BigMax(gasFeeCap, arbmath.FloatToBig(params.GWei)) - gasTipCap, err := v.l1Reader.Client().SuggestGasTipCap(ctx) + gasTipCap, err := l1Reader.Client().SuggestGasTipCap(ctx) if err != nil { return 0, fmt.Errorf("getting suggested gas tip cap: %w", err) } gasFeeCap.Add(gasFeeCap, gasTipCap) - g, err := v.l1Reader.Client().EstimateGas( + g, err := l1Reader.Client().EstimateGas( ctx, ethereum.CallMsg{ - From: v.auth.From, - To: v.Address(), - Value: value, + From: auth.From, + To: to, + Value: auth.Value, Data: data, GasFeeCap: gasFeeCap, GasTipCap: gasTipCap, @@ -322,7 +361,11 @@ func (v *Contract) estimateGas(ctx context.Context, value *big.Int, data []byte) if err != nil { return 0, fmt.Errorf("estimating gas: %w", err) } - return g + v.getExtraGas(), nil + return g + getExtraGas(), nil +} + +func (v *Contract) gasForTxData(ctx context.Context, auth *bind.TransactOpts, data []byte) (uint64, error) { + return gasForTxData(ctx, v.l1Reader, auth, v.Address(), data, v.getExtraGas) } func (v *Contract) TimeoutChallenges(ctx context.Context, challenges []uint64) (*types.Transaction, error) { @@ -341,14 +384,6 @@ func (v *Contract) TimeoutChallenges(ctx context.Context, challenges []uint64) ( return v.dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), *v.Address(), data, gas, auth.Value) } -// gasForTxData returns auth.GasLimit if it's nonzero, otherwise returns estimate. -func (v *Contract) gasForTxData(ctx context.Context, auth *bind.TransactOpts, data []byte) (uint64, error) { - if auth.GasLimit != 0 { - return auth.GasLimit, nil - } - return v.estimateGas(ctx, auth.Value, data) -} - func (v *Contract) L1Client() arbutil.L1Interface { return v.l1Reader.Client() } @@ -407,6 +442,7 @@ func GetValidatorWalletContract( transactAuth *bind.TransactOpts, l1Reader *headerreader.HeaderReader, createIfMissing bool, + createWalletFunc func(context.Context, common.Address) (*types.Transaction, error), ) (*common.Address, error) { client := l1Reader.Client() @@ -443,10 +479,18 @@ func GetValidatorWalletContract( return nil, nil } - var initialExecutorAllowedDests []common.Address - tx, err := walletCreator.CreateWallet(transactAuth, initialExecutorAllowedDests) - if err != nil { - return nil, err + var tx *types.Transaction + if createWalletFunc == nil { + var initialExecutorAllowedDests []common.Address + tx, err = walletCreator.CreateWallet(transactAuth, initialExecutorAllowedDests) + if err != nil { + return nil, err + } + } else { + tx, err = createWalletFunc(ctx, validatorWalletFactoryAddr) + if err != nil { + return nil, err + } } receipt, err := l1Reader.WaitForTxApproval(ctx, tx) diff --git a/system_tests/fast_confirm_test.go b/system_tests/fast_confirm_test.go index 4a679e5077..79b77524a2 100644 --- a/system_tests/fast_confirm_test.go +++ b/system_tests/fast_confirm_test.go @@ -79,10 +79,10 @@ func TestFastConfirmation(t *testing.T) { builder.L1.TransferBalance(t, "Faucet", "Validator", balance, builder.L1Info) l1auth := builder.L1Info.GetDefaultTransactOpts("Validator", ctx) - valWalletAddrPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true) + valWalletAddrPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, nil) Require(t, err) valWalletAddr := *valWalletAddrPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, nil) Require(t, err) if valWalletAddr == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddr.String(), "vs", valWalletAddrCheck.String()) @@ -278,10 +278,10 @@ func TestFastConfirmationWithSafe(t *testing.T) { builder.L1.TransferBalance(t, "Faucet", "ValidatorB", balance, builder.L1Info) l1authB := builder.L1Info.GetDefaultTransactOpts("ValidatorB", ctx) - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true) + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, nil) Require(t, err) valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, nil) Require(t, err) if valWalletAddrA == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) diff --git a/system_tests/staker_test.go b/system_tests/staker_test.go index 03c9fd3628..13530adfd5 100644 --- a/system_tests/staker_test.go +++ b/system_tests/staker_test.go @@ -132,10 +132,10 @@ func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) builder.L1.TransferBalance(t, "Faucet", "ValidatorB", balance, builder.L1Info) l1authB := builder.L1Info.GetDefaultTransactOpts("ValidatorB", ctx) - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true) + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, nil) Require(t, err) valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, nil) Require(t, err) if valWalletAddrA == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) From 26bd0bd59e0ed1d0948c197937fd601809bf5342 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 12 Aug 2024 15:23:46 -0300 Subject: [PATCH 042/489] Staker tests creates wallet contract through Staker's DataPoster --- system_tests/staker_test.go | 42 +++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/system_tests/staker_test.go b/system_tests/staker_test.go index 13530adfd5..ea34eb8a96 100644 --- a/system_tests/staker_test.go +++ b/system_tests/staker_test.go @@ -37,6 +37,7 @@ import ( "github.com/offchainlabs/nitro/util" "github.com/offchainlabs/nitro/util/arbmath" "github.com/offchainlabs/nitro/util/colors" + "github.com/offchainlabs/nitro/util/testhelpers" "github.com/offchainlabs/nitro/validator/valnode" ) @@ -57,7 +58,8 @@ func makeBackgroundTxs(ctx context.Context, builder *NodeBuilder) error { } func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) { - t.Parallel() + logHandler := testhelpers.InitTestLog(t, log.LvlTrace) + ctx, cancelCtx := context.WithCancel(context.Background()) defer cancelCtx() srv := externalsignertest.NewServer(t) @@ -132,15 +134,6 @@ func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) builder.L1.TransferBalance(t, "Faucet", "ValidatorB", balance, builder.L1Info) l1authB := builder.L1Info.GetDefaultTransactOpts("ValidatorB", ctx) - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, nil) - Require(t, err) - valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, nil) - Require(t, err) - if valWalletAddrA == *valWalletAddrCheck { - Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) - } - rollup, err := rollupgen.NewRollupAdminLogic(l2nodeA.DeployInfo.Rollup, builder.L1.Client) Require(t, err) @@ -149,16 +142,9 @@ func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) rollupABI, err := abi.JSON(strings.NewReader(rollupgen.RollupAdminLogicABI)) Require(t, err, "unable to parse rollup ABI") - setValidatorCalldata, err := rollupABI.Pack("setValidator", []common.Address{valWalletAddrA, l1authB.From, srv.Address}, []bool{true, true, true}) - Require(t, err, "unable to generate setValidator calldata") - tx, err := upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setValidatorCalldata) - Require(t, err, "unable to set validators") - _, err = builder.L1.EnsureTxSucceeded(tx) - Require(t, err) - setMinAssertPeriodCalldata, err := rollupABI.Pack("setMinimumAssertionPeriod", big.NewInt(1)) Require(t, err, "unable to generate setMinimumAssertionPeriod calldata") - tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setMinAssertPeriodCalldata) + tx, err := upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setMinAssertPeriodCalldata) Require(t, err, "unable to set minimum assertion period") _, err = builder.L1.EnsureTxSucceeded(tx) Require(t, err) @@ -190,6 +176,22 @@ func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) valConfigA.Strategy = "MakeNodes" } + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.CreateWalletContract) + Require(t, err) + valWalletAddrA := *valWalletAddrAPtr + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.CreateWalletContract) + Require(t, err) + if valWalletAddrA == *valWalletAddrCheck { + Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) + } + + setValidatorCalldata, err := rollupABI.Pack("setValidator", []common.Address{valWalletAddrA, l1authB.From, srv.Address}, []bool{true, true, true}) + Require(t, err, "unable to generate setValidator calldata") + tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setValidatorCalldata) + Require(t, err, "unable to set validators") + _, err = builder.L1.EnsureTxSucceeded(tx) + Require(t, err) + _, valStack := createTestValidationNode(t, ctx, &valnode.TestValidationConfig) blockValidatorConfig := staker.TestBlockValidatorConfig @@ -464,6 +466,10 @@ func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) if !stakerBWasStaked { Fatal(t, "staker B was never staked") } + + if logHandler.WasLogged("data poster expected next transaction to have nonce \\d+ but was requested to post transaction with nonce \\d+") { + Fatal(t, "Staker's DataPoster inferred nonce incorrectly") + } } func TestStakersCooperative(t *testing.T) { From c3ee555bc6c3679aed7c1347fc8c771d3d02caca Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 12 Aug 2024 15:24:02 -0300 Subject: [PATCH 043/489] TestGetValidatorWalletContractWithoutDataPoster --- system_tests/staker_test.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/system_tests/staker_test.go b/system_tests/staker_test.go index ea34eb8a96..3fdf82ca74 100644 --- a/system_tests/staker_test.go +++ b/system_tests/staker_test.go @@ -475,3 +475,29 @@ func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) func TestStakersCooperative(t *testing.T) { stakerTestImpl(t, false, false) } + +func TestGetValidatorWalletContractWithoutDataPoster(t *testing.T) { + t.Parallel() + + ctx, cancelCtx := context.WithCancel(context.Background()) + defer cancelCtx() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, true) + cleanup := builder.Build(t) + defer cleanup() + + balance := big.NewInt(params.Ether) + balance.Mul(balance, big.NewInt(100)) + builder.L1Info.GenerateAccount("ValidatorA") + builder.L1.TransferBalance(t, "Faucet", "ValidatorA", balance, builder.L1Info) + l1auth := builder.L1Info.GetDefaultTransactOpts("ValidatorA", ctx) + + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, &l1auth, builder.L2.ConsensusNode.L1Reader, true, nil) + Require(t, err) + valWalletAddrA := *valWalletAddrAPtr + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, &l1auth, builder.L2.ConsensusNode.L1Reader, true, nil) + Require(t, err) + if valWalletAddrA == *valWalletAddrCheck { + Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) + } +} From 2fde94e651c991df1e4a3efe858d758bd784e4e4 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 12 Aug 2024 15:52:10 -0300 Subject: [PATCH 044/489] Adds comment related to v.CreateWalletContract --- staker/validatorwallet/contract.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/staker/validatorwallet/contract.go b/staker/validatorwallet/contract.go index 89d4fef11d..495e796cd2 100644 --- a/staker/validatorwallet/contract.go +++ b/staker/validatorwallet/contract.go @@ -229,6 +229,10 @@ func (v *Contract) populateWallet(ctx context.Context, createIfMissing bool) err if err != nil { return err } + + // By passing v.CreateWalletContract as a parameter to GetValidatorWalletContract we force to create a validator wallet through the Staker's DataPoster object. + // DataPoster keeps in its internal state information related to the transactions sent through it, which is used to infer the expected nonce in a transaction for example. + // If a transaction is sent using the Staker's DataPoster key, but not through the Staker's DataPoster object, DataPoster's internal state will be outdated, which can compromise the expected nonce inference. addr, err := GetValidatorWalletContract(ctx, v.walletFactoryAddr, v.rollupFromBlock, auth, v.l1Reader, createIfMissing, v.CreateWalletContract) if err != nil { return err From 04e3f12507ceff95812d0b72901d84ed643954a1 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 12 Aug 2024 16:16:12 -0300 Subject: [PATCH 045/489] Fast confirmation test creates wallet contract through Staker's DataPoster --- system_tests/fast_confirm_test.go | 48 +++++++++++++++---------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/system_tests/fast_confirm_test.go b/system_tests/fast_confirm_test.go index 79b77524a2..ae624be1e9 100644 --- a/system_tests/fast_confirm_test.go +++ b/system_tests/fast_confirm_test.go @@ -79,15 +79,6 @@ func TestFastConfirmation(t *testing.T) { builder.L1.TransferBalance(t, "Faucet", "Validator", balance, builder.L1Info) l1auth := builder.L1Info.GetDefaultTransactOpts("Validator", ctx) - valWalletAddrPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, nil) - Require(t, err) - valWalletAddr := *valWalletAddrPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, nil) - Require(t, err) - if valWalletAddr == *valWalletAddrCheck { - Require(t, err, "didn't cache validator wallet address", valWalletAddr.String(), "vs", valWalletAddrCheck.String()) - } - rollup, err := rollupgen.NewRollupAdminLogic(l2node.DeployInfo.Rollup, builder.L1.Client) Require(t, err) @@ -96,27 +87,13 @@ func TestFastConfirmation(t *testing.T) { rollupABI, err := abi.JSON(strings.NewReader(rollupgen.RollupAdminLogicABI)) Require(t, err, "unable to parse rollup ABI") - setValidatorCalldata, err := rollupABI.Pack("setValidator", []common.Address{valWalletAddr, srv.Address}, []bool{true, true}) - Require(t, err, "unable to generate setValidator calldata") - tx, err := upgradeExecutor.ExecuteCall(&deployAuth, l2node.DeployInfo.Rollup, setValidatorCalldata) - Require(t, err, "unable to set validators") - _, err = builder.L1.EnsureTxSucceeded(tx) - Require(t, err) - setMinAssertPeriodCalldata, err := rollupABI.Pack("setMinimumAssertionPeriod", big.NewInt(1)) Require(t, err, "unable to generate setMinimumAssertionPeriod calldata") - tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2node.DeployInfo.Rollup, setMinAssertPeriodCalldata) + tx, err := upgradeExecutor.ExecuteCall(&deployAuth, l2node.DeployInfo.Rollup, setMinAssertPeriodCalldata) Require(t, err, "unable to set minimum assertion period") _, err = builder.L1.EnsureTxSucceeded(tx) Require(t, err) - setAnyTrustFastConfirmerCalldata, err := rollupABI.Pack("setAnyTrustFastConfirmer", valWalletAddr) - Require(t, err, "unable to generate setAnyTrustFastConfirmer calldata") - tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2node.DeployInfo.Rollup, setAnyTrustFastConfirmerCalldata) - Require(t, err, "unable to set anytrust fast confirmer") - _, err = builder.L1.EnsureTxSucceeded(tx) - Require(t, err) - valConfig := staker.TestL1ValidatorConfig valConfig.EnableFastConfirmation = true parentChainID, err := builder.L1.Client.ChainID(ctx) @@ -138,6 +115,29 @@ func TestFastConfirmation(t *testing.T) { Require(t, err) valConfig.Strategy = "MakeNodes" + valWalletAddrPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, valWallet.CreateWalletContract) + Require(t, err) + valWalletAddr := *valWalletAddrPtr + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, valWallet.CreateWalletContract) + Require(t, err) + if valWalletAddr == *valWalletAddrCheck { + Require(t, err, "didn't cache validator wallet address", valWalletAddr.String(), "vs", valWalletAddrCheck.String()) + } + + setValidatorCalldata, err := rollupABI.Pack("setValidator", []common.Address{valWalletAddr, srv.Address}, []bool{true, true}) + Require(t, err, "unable to generate setValidator calldata") + tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2node.DeployInfo.Rollup, setValidatorCalldata) + Require(t, err, "unable to set validators") + _, err = builder.L1.EnsureTxSucceeded(tx) + Require(t, err) + + setAnyTrustFastConfirmerCalldata, err := rollupABI.Pack("setAnyTrustFastConfirmer", valWalletAddr) + Require(t, err, "unable to generate setAnyTrustFastConfirmer calldata") + tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2node.DeployInfo.Rollup, setAnyTrustFastConfirmerCalldata) + Require(t, err, "unable to set anytrust fast confirmer") + _, err = builder.L1.EnsureTxSucceeded(tx) + Require(t, err) + _, valStack := createTestValidationNode(t, ctx, &valnode.TestValidationConfig) blockValidatorConfig := staker.TestBlockValidatorConfig From b933b74725e2dd34df1f472ab22fbdfcff277698 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Thu, 22 Aug 2024 22:19:16 +0200 Subject: [PATCH 046/489] pass targets to rawdb.WrapDatabaseWithWasm, change rawdb.Target to ethdb.WasmTarget --- arbnode/inbox_test.go | 4 +- arbos/programs/native.go | 11 ++--- arbos/programs/wasmstorehelper.go | 3 +- cmd/nitro/init.go | 6 +-- cmd/nitro/init_test.go | 4 ++ cmd/nitro/nitro.go | 2 +- execution/gethexec/executionengine.go | 35 ++++++++++------ execution/gethexec/node.go | 50 +++++++++++++++++++++-- go-ethereum | 2 +- staker/block_validator.go | 2 +- staker/challenge_manager.go | 3 +- staker/stateless_block_validator.go | 5 +-- system_tests/common_test.go | 47 +++++++++++---------- system_tests/forwarder_test.go | 4 +- system_tests/recreatestate_rpc_test.go | 18 ++++---- system_tests/retryable_test.go | 2 +- system_tests/validation_mock_test.go | 6 +-- validator/client/redis/producer.go | 9 ++-- validator/client/validation_client.go | 13 +++--- validator/interface.go | 4 +- validator/server_api/json.go | 8 ++-- validator/server_arb/validator_spawner.go | 5 ++- validator/server_jit/spawner.go | 5 ++- validator/validation_entry.go | 4 +- validator/valnode/validation_api.go | 4 +- 25 files changed, 161 insertions(+), 95 deletions(-) diff --git a/arbnode/inbox_test.go b/arbnode/inbox_test.go index 1c46c593b9..d579b7c278 100644 --- a/arbnode/inbox_test.go +++ b/arbnode/inbox_test.go @@ -72,7 +72,9 @@ func NewTransactionStreamerForTest(t *testing.T, ownerAddress common.Address) (* if err != nil { Fail(t, err) } - if err := execEngine.Initialize(gethexec.DefaultCachingConfig.StylusLRUCache, &gethexec.DefaultStylusTargetConfig); err != nil { + stylusTargetConfig := &gethexec.DefaultStylusTargetConfig + Require(t, stylusTargetConfig.Validate()) // pre-processes config (i.a. parses wasmTargets) + if err := execEngine.Initialize(gethexec.DefaultCachingConfig.StylusLRUCache, stylusTargetConfig); err != nil { Fail(t, err) } execSeq := &execClientWrapper{execEngine, t} diff --git a/arbos/programs/native.go b/arbos/programs/native.go index fd3dec25a0..36e200af4d 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/offchainlabs/nitro/arbos/burn" "github.com/offchainlabs/nitro/arbos/util" @@ -71,7 +72,7 @@ func activateProgramInternal( version uint16, debug bool, gasLeft *uint64, -) (*activationInfo, map[rawdb.Target][]byte, error) { +) (*activationInfo, map[ethdb.WasmTarget][]byte, error) { output := &rustBytes{} moduleHash := &bytes32{} stylusData := &C.StylusData{} @@ -111,7 +112,7 @@ func activateProgramInternal( if status_asm != 0 { return nil, nil, fmt.Errorf("%w: %s", ErrProgramActivation, string(asm)) } - asmMap := map[rawdb.Target][]byte{ + asmMap := map[ethdb.WasmTarget][]byte{ rawdb.TargetWavm: module, target: asm, } @@ -171,7 +172,7 @@ func getLocalAsm(statedb vm.StateDB, moduleHash common.Hash, addressForLogging c } asm, exists := asmMap[localTarget] if !exists { - var availableTargets []rawdb.Target + var availableTargets []ethdb.WasmTarget for target := range asmMap { availableTargets = append(availableTargets, target) } @@ -203,7 +204,7 @@ func callProgram( } if db, ok := db.(*state.StateDB); ok { - targets := []rawdb.Target{ + targets := []ethdb.WasmTarget{ rawdb.TargetWavm, rawdb.LocalTarget(), } @@ -291,7 +292,7 @@ func ResizeWasmLruCache(size uint32) { const DefaultTargetDescriptionArm = "arm64-linux-unknown+neon" const DefaultTargetDescriptionX86 = "x86_64-linux-unknown+sse4.2" -func SetTarget(name rawdb.Target, description string, native bool) error { +func SetTarget(name ethdb.WasmTarget, description string, native bool) error { output := &rustBytes{} status := userStatus(C.stylus_target_set( goSlice([]byte(name)), diff --git a/arbos/programs/wasmstorehelper.go b/arbos/programs/wasmstorehelper.go index 4f82d80282..95daaf544a 100644 --- a/arbos/programs/wasmstorehelper.go +++ b/arbos/programs/wasmstorehelper.go @@ -12,6 +12,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" ) @@ -44,7 +45,7 @@ func (p Programs) SaveActiveProgramToWasmStore(statedb *state.StateDB, codeHash } // If already in wasm store then return early - _, err = statedb.TryGetActivatedAsmMap([]rawdb.Target{rawdb.TargetWavm, rawdb.LocalTarget()}, moduleHash) + _, err = statedb.TryGetActivatedAsmMap([]ethdb.WasmTarget{rawdb.TargetWavm, rawdb.LocalTarget()}, moduleHash) if err == nil { return nil } diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index d9ae0df3b0..14d9a52203 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -475,7 +475,7 @@ func validateOrUpgradeWasmStoreSchemaVersion(db ethdb.Database) error { return nil } -func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeConfig, chainId *big.Int, cacheConfig *core.CacheConfig, persistentConfig *conf.PersistentConfig, l1Client arbutil.L1Interface, rollupAddrs chaininfo.RollupAddresses) (ethdb.Database, *core.BlockChain, error) { +func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeConfig, chainId *big.Int, cacheConfig *core.CacheConfig, targetConfig *gethexec.StylusTargetConfig, persistentConfig *conf.PersistentConfig, l1Client arbutil.L1Interface, rollupAddrs chaininfo.RollupAddresses) (ethdb.Database, *core.BlockChain, error) { if !config.Init.Force { if readOnlyDb, err := stack.OpenDatabaseWithFreezerWithExtraOptions("l2chaindata", 0, 0, config.Persistent.Ancient, "l2chaindata/", true, persistentConfig.Pebble.ExtraOptions("l2chaindata")); err == nil { if chainConfig := gethexec.TryReadStoredChainConfig(readOnlyDb); chainConfig != nil { @@ -500,7 +500,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo if err := dbutil.UnfinishedConversionCheck(wasmDb); err != nil { return nil, nil, fmt.Errorf("wasm unfinished database conversion check error: %w", err) } - chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmDb, 1) + chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmDb, 1, targetConfig.WasmTargets()) _, err = rawdb.ParseStateScheme(cacheConfig.StateScheme, chainDb) if err != nil { return nil, nil, err @@ -617,7 +617,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo if err := validateOrUpgradeWasmStoreSchemaVersion(wasmDb); err != nil { return nil, nil, err } - chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmDb, 1) + chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmDb, 1, targetConfig.WasmTargets()) _, err = rawdb.ParseStateScheme(cacheConfig.StateScheme, chainDb) if err != nil { return nil, nil, err diff --git a/cmd/nitro/init_test.go b/cmd/nitro/init_test.go index b2773ed861..af2242af78 100644 --- a/cmd/nitro/init_test.go +++ b/cmd/nitro/init_test.go @@ -377,6 +377,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), + &gethexec.DefaultStylusTargetConfig, &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, @@ -393,6 +394,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), + &gethexec.DefaultStylusTargetConfig, &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, @@ -410,6 +412,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), + &gethexec.DefaultStylusTargetConfig, &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, @@ -545,6 +548,7 @@ func TestOpenInitializeChainDbEmptyInit(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), + &gethexec.DefaultStylusTargetConfig, &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index a052c146d1..052755d8a3 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -481,7 +481,7 @@ func mainImpl() int { } } - chainDb, l2BlockChain, err := openInitializeChainDb(ctx, stack, nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), &nodeConfig.Persistent, l1Client, rollupAddrs) + chainDb, l2BlockChain, err := openInitializeChainDb(ctx, stack, nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), &nodeConfig.Execution.StylusTarget, &nodeConfig.Persistent, l1Client, rollupAddrs) if l2BlockChain != nil { deferFuncs = append(deferFuncs, func() { l2BlockChain.Stop() }) } diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index 806355b2c6..4e0217f527 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -154,20 +154,31 @@ func (s *ExecutionEngine) Initialize(rustCacheSize uint32, targetConfig *StylusT if rustCacheSize != 0 { programs.ResizeWasmLruCache(rustCacheSize) } - var effectiveStylusTarget string - target := rawdb.LocalTarget() - switch target { - case rawdb.TargetArm64: - effectiveStylusTarget = targetConfig.Arm64 - case rawdb.TargetAmd64: - effectiveStylusTarget = targetConfig.Amd64 - case rawdb.TargetHost: - effectiveStylusTarget = targetConfig.Host + + localTarget := rawdb.LocalTarget() + targets := targetConfig.WasmTargets() + var nativeSet bool + for _, target := range targets { + var effectiveStylusTarget string + switch target { + case rawdb.TargetArm64: + effectiveStylusTarget = targetConfig.Arm64 + case rawdb.TargetAmd64: + effectiveStylusTarget = targetConfig.Amd64 + case rawdb.TargetHost: + effectiveStylusTarget = targetConfig.Host + } + isNative := target == localTarget + err := programs.SetTarget(target, effectiveStylusTarget, isNative) + if err != nil { + return fmt.Errorf("Failed to set stylus target: %w", err) + } + nativeSet = nativeSet || isNative } - err := programs.SetTarget(target, effectiveStylusTarget, true) - if err != nil { - return fmt.Errorf("Failed to set stylus target: %w", err) + if !nativeSet { + return fmt.Errorf("local target %v missing in list of archs %v", localTarget, targets) } + return nil } diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index b864332e83..f519cce6ed 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -11,6 +11,7 @@ import ( "github.com/ethereum/go-ethereum/arbitrum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/filters" @@ -29,21 +30,61 @@ import ( ) type StylusTargetConfig struct { - Arm64 string `koanf:"arm64"` - Amd64 string `koanf:"amd64"` - Host string `koanf:"host"` + Arm64 string `koanf:"arm64"` + Amd64 string `koanf:"amd64"` + Host string `koanf:"host"` + Archs []string `koanf:"archs"` + + wasmTargets []ethdb.WasmTarget +} + +func (c *StylusTargetConfig) WasmTargets() []ethdb.WasmTarget { + return c.wasmTargets +} + +func (c *StylusTargetConfig) Validate() error { + localTarget := rawdb.LocalTarget() + var nativeFound bool + targets := make([]ethdb.WasmTarget, 0, len(c.Archs)) + for _, arch := range c.Archs { + target := ethdb.WasmTarget(arch) + if !rawdb.IsSupportedWasmTarget(target) { + return fmt.Errorf("unsupported architecture: %v, possible values: %s, %s, %s, %s", arch, rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost) + } + var alreadyInList bool + for _, t := range targets { + if target == t { + alreadyInList = true + break + } + } + if alreadyInList { + continue + } + if target == localTarget { + nativeFound = true + } + targets = append(targets, target) + } + if !nativeFound { + return fmt.Errorf("native target not found in archs list, native target: %v, archs: %v", localTarget, c.Archs) + } + c.wasmTargets = targets + return nil } var DefaultStylusTargetConfig = StylusTargetConfig{ Arm64: programs.DefaultTargetDescriptionArm, Amd64: programs.DefaultTargetDescriptionX86, Host: "", + Archs: []string{string(rawdb.TargetWavm), string(rawdb.LocalTarget())}, } func StylusTargetConfigAddOptions(prefix string, f *flag.FlagSet) { f.String(prefix+".arm64", DefaultStylusTargetConfig.Arm64, "stylus programs compilation target for arm64 linux") f.String(prefix+".amd64", DefaultStylusTargetConfig.Amd64, "stylus programs compilation target for amd64 linux") f.String(prefix+".host", DefaultStylusTargetConfig.Host, "stylus programs compilation target for system other than 64-bit ARM or 64-bit x86") + f.StringSlice(prefix+".archs", DefaultStylusTargetConfig.Archs, fmt.Sprintf("Comma separated architectures list to compile stylus program to and cache in wasm store (available targets: %s, %s, %s, %s)", rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost)) } type Config struct { @@ -82,6 +123,9 @@ func (c *Config) Validate() error { if c.forwardingTarget != "" && c.Sequencer.Enable { return errors.New("ForwardingTarget set and sequencer enabled") } + if err := c.StylusTarget.Validate(); err != nil { + return err + } return nil } diff --git a/go-ethereum b/go-ethereum index 575062fad7..cb03c12f7f 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 575062fad7ff4db9d7c235f49472f658be29e2fe +Subproject commit cb03c12f7f02d47344d2d089eef748e92b99c95e diff --git a/staker/block_validator.go b/staker/block_validator.go index 8f5724beac..5cfce84c99 100644 --- a/staker/block_validator.go +++ b/staker/block_validator.go @@ -499,7 +499,7 @@ func (v *BlockValidator) sendRecord(s *validationStatus) error { //nolint:gosec func (v *BlockValidator) writeToFile(validationEntry *validationEntry, moduleRoot common.Hash) error { - input, err := validationEntry.ToInput([]rawdb.Target{rawdb.TargetWavm}) + input, err := validationEntry.ToInput([]ethdb.WasmTarget{rawdb.TargetWavm}) if err != nil { return err } diff --git a/staker/challenge_manager.go b/staker/challenge_manager.go index b1421d7e41..978a876fd0 100644 --- a/staker/challenge_manager.go +++ b/staker/challenge_manager.go @@ -16,6 +16,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" "github.com/offchainlabs/nitro/arbutil" @@ -468,7 +469,7 @@ func (m *ChallengeManager) createExecutionBackend(ctx context.Context, step uint if err != nil { return fmt.Errorf("error creating validation entry for challenge %v msg %v for execution challenge: %w", m.challengeIndex, initialCount, err) } - input, err := entry.ToInput([]rawdb.Target{rawdb.TargetWavm}) + input, err := entry.ToInput([]ethdb.WasmTarget{rawdb.TargetWavm}) if err != nil { return fmt.Errorf("error getting validation entry input of challenge %v msg %v: %w", m.challengeIndex, initialCount, err) } diff --git a/staker/stateless_block_validator.go b/staker/stateless_block_validator.go index d5eeb8eb69..f54ec8b58c 100644 --- a/staker/stateless_block_validator.go +++ b/staker/stateless_block_validator.go @@ -12,7 +12,6 @@ import ( "github.com/offchainlabs/nitro/arbstate/daprovider" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" @@ -134,7 +133,7 @@ type validationEntry struct { DelayedMsg []byte } -func (e *validationEntry) ToInput(stylusArchs []rawdb.Target) (*validator.ValidationInput, error) { +func (e *validationEntry) ToInput(stylusArchs []ethdb.WasmTarget) (*validator.ValidationInput, error) { if e.Stage != Ready { return nil, errors.New("cannot create input from non-ready entry") } @@ -143,7 +142,7 @@ func (e *validationEntry) ToInput(stylusArchs []rawdb.Target) (*validator.Valida HasDelayedMsg: e.HasDelayedMsg, DelayedMsgNr: e.DelayedMsgNr, Preimages: e.Preimages, - UserWasms: make(map[rawdb.Target]map[common.Hash][]byte, len(e.UserWasms)), + UserWasms: make(map[ethdb.WasmTarget]map[common.Hash][]byte, len(e.UserWasms)), BatchInfo: e.BatchInfo, DelayedMsg: e.DelayedMsg, StartState: e.Start, diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 6e7375a921..392f6de805 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -197,7 +197,7 @@ var TestSequencerConfig = gethexec.SequencerConfig{ EnableProfiling: false, } -func ExecConfigDefaultNonSequencerTest() *gethexec.Config { +func ExecConfigDefaultNonSequencerTest(t *testing.T) *gethexec.Config { config := gethexec.ConfigDefault config.Caching = TestCachingConfig config.ParentChainReader = headerreader.TestConfig @@ -206,12 +206,12 @@ func ExecConfigDefaultNonSequencerTest() *gethexec.Config { config.ForwardingTarget = "null" config.TxPreChecker.Strictness = gethexec.TxPreCheckerStrictnessNone - _ = config.Validate() + Require(t, config.Validate()) return &config } -func ExecConfigDefaultTest() *gethexec.Config { +func ExecConfigDefaultTest(t *testing.T) *gethexec.Config { config := gethexec.ConfigDefault config.Caching = TestCachingConfig config.Sequencer = TestSequencerConfig @@ -219,7 +219,7 @@ func ExecConfigDefaultTest() *gethexec.Config { config.ForwardingTarget = "null" config.TxPreChecker.Strictness = gethexec.TxPreCheckerStrictnessNone - _ = config.Validate() + Require(t, config.Validate()) return &config } @@ -272,7 +272,7 @@ func (b *NodeBuilder) DefaultConfig(t *testing.T, withL1 bool) *NodeBuilder { b.l2StackConfig = testhelpers.CreateStackConfigForTest(b.dataDir) cp := valnode.TestValidationConfig b.valnodeConfig = &cp - b.execConfig = ExecConfigDefaultTest() + b.execConfig = ExecConfigDefaultTest(t) return b } @@ -310,7 +310,7 @@ func (b *NodeBuilder) CheckConfig(t *testing.T) { b.nodeConfig = arbnode.ConfigDefaultL1Test() } if b.execConfig == nil { - b.execConfig = ExecConfigDefaultTest() + b.execConfig = ExecConfigDefaultTest(t) } if b.L1Info == nil { b.L1Info = NewL1TestInfo(t) @@ -346,7 +346,7 @@ func (b *NodeBuilder) BuildL2OnL1(t *testing.T) func() { var l2arbDb ethdb.Database var l2blockchain *core.BlockChain _, b.L2.Stack, l2chainDb, l2arbDb, l2blockchain = createL2BlockChainWithStackConfig( - t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, &b.execConfig.Caching) + t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, b.execConfig) var sequencerTxOptsPtr *bind.TransactOpts var dataSigner signature.DataSignerFunc @@ -409,7 +409,7 @@ func (b *NodeBuilder) BuildL2(t *testing.T) func() { var arbDb ethdb.Database var blockchain *core.BlockChain b.L2Info, b.L2.Stack, chainDb, arbDb, blockchain = createL2BlockChain( - t, b.L2Info, b.dataDir, b.chainConfig, &b.execConfig.Caching) + t, b.L2Info, b.dataDir, b.chainConfig, b.execConfig) Require(t, b.execConfig.Validate()) execConfig := b.execConfig @@ -460,7 +460,7 @@ func (b *NodeBuilder) RestartL2Node(t *testing.T) { } b.L2.cleanup() - l2info, stack, chainDb, arbDb, blockchain := createL2BlockChainWithStackConfig(t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, &b.execConfig.Caching) + l2info, stack, chainDb, arbDb, blockchain := createL2BlockChainWithStackConfig(t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, b.execConfig) execConfigFetcher := func() *gethexec.Config { return b.execConfig } execNode, err := gethexec.CreateExecutionNode(b.ctx, stack, chainDb, blockchain, nil, execConfigFetcher) @@ -1047,30 +1047,32 @@ func DeployOnTestL1( } func createL2BlockChain( - t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, cacheConfig *gethexec.CachingConfig, + t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, execConfig *gethexec.Config, ) (*BlockchainTestInfo, *node.Node, ethdb.Database, ethdb.Database, *core.BlockChain) { - return createL2BlockChainWithStackConfig(t, l2info, dataDir, chainConfig, nil, nil, cacheConfig) + return createL2BlockChainWithStackConfig(t, l2info, dataDir, chainConfig, nil, nil, execConfig) } func createL2BlockChainWithStackConfig( - t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, initMessage *arbostypes.ParsedInitMessage, stackConfig *node.Config, cacheConfig *gethexec.CachingConfig, + t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, initMessage *arbostypes.ParsedInitMessage, stackConfig *node.Config, execConfig *gethexec.Config, ) (*BlockchainTestInfo, *node.Node, ethdb.Database, ethdb.Database, *core.BlockChain) { if l2info == nil { l2info = NewArbTestInfo(t, chainConfig.ChainID) } - var stack *node.Node - var err error if stackConfig == nil { stackConfig = testhelpers.CreateStackConfigForTest(dataDir) } - stack, err = node.New(stackConfig) + if execConfig == nil { + execConfig = ExecConfigDefaultTest(t) + } + + stack, err := node.New(stackConfig) Require(t, err) chainData, err := stack.OpenDatabaseWithExtraOptions("l2chaindata", 0, 0, "l2chaindata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("l2chaindata")) Require(t, err) wasmData, err := stack.OpenDatabaseWithExtraOptions("wasm", 0, 0, "wasm/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("wasm")) Require(t, err) - chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmData, 0) + chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmData, 0, execConfig.StylusTarget.WasmTargets()) arbDb, err := stack.OpenDatabaseWithExtraOptions("arbitrumdata", 0, 0, "arbitrumdata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("arbitrumdata")) Require(t, err) @@ -1085,11 +1087,8 @@ func createL2BlockChainWithStackConfig( SerializedChainConfig: serializedChainConfig, } } - var coreCacheConfig *core.CacheConfig - if cacheConfig != nil { - coreCacheConfig = gethexec.DefaultCacheConfigFor(stack, cacheConfig) - } - blockchain, err := gethexec.WriteOrTestBlockChain(chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest().TxLookupLimit, 0) + coreCacheConfig := gethexec.DefaultCacheConfigFor(stack, &execConfig.Caching) + blockchain, err := gethexec.WriteOrTestBlockChain(chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest(t).TxLookupLimit, 0) Require(t, err) return l2info, stack, chainDb, arbDb, blockchain @@ -1149,7 +1148,7 @@ func Create2ndNodeWithConfig( nodeConfig = arbnode.ConfigDefaultL1NonSequencerTest() } if execConfig == nil { - execConfig = ExecConfigDefaultNonSequencerTest() + execConfig = ExecConfigDefaultNonSequencerTest(t) } feedErrChan := make(chan error, 10) l1rpcClient := l1stack.Attach() @@ -1165,7 +1164,7 @@ func Create2ndNodeWithConfig( Require(t, err) wasmData, err := l2stack.OpenDatabaseWithExtraOptions("wasm", 0, 0, "wasm/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("wasm")) Require(t, err) - l2chainDb := rawdb.WrapDatabaseWithWasm(l2chainData, wasmData, 0) + l2chainDb := rawdb.WrapDatabaseWithWasm(l2chainData, wasmData, 0, execConfig.StylusTarget.WasmTargets()) l2arbDb, err := l2stack.OpenDatabaseWithExtraOptions("arbitrumdata", 0, 0, "arbitrumdata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("arbitrumdata")) Require(t, err) @@ -1179,7 +1178,7 @@ func Create2ndNodeWithConfig( chainConfig := firstExec.ArbInterface.BlockChain().Config() coreCacheConfig := gethexec.DefaultCacheConfigFor(l2stack, &execConfig.Caching) - l2blockchain, err := gethexec.WriteOrTestBlockChain(l2chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest().TxLookupLimit, 0) + l2blockchain, err := gethexec.WriteOrTestBlockChain(l2chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest(t).TxLookupLimit, 0) Require(t, err) AddValNodeIfNeeded(t, ctx, nodeConfig, true, "", valnodeConfig.Wasm.RootPath) diff --git a/system_tests/forwarder_test.go b/system_tests/forwarder_test.go index 9fe419593e..1b083d2089 100644 --- a/system_tests/forwarder_test.go +++ b/system_tests/forwarder_test.go @@ -38,7 +38,7 @@ func TestStaticForwarder(t *testing.T) { clientA := builder.L2.Client nodeConfigB := arbnode.ConfigDefaultL1Test() - execConfigB := ExecConfigDefaultTest() + execConfigB := ExecConfigDefaultTest(t) execConfigB.Sequencer.Enable = false nodeConfigB.Sequencer = false nodeConfigB.DelayedSequencer.Enable = false @@ -109,7 +109,7 @@ func createForwardingNode(t *testing.T, builder *NodeBuilder, ipcPath string, re nodeConfig.Sequencer = false nodeConfig.DelayedSequencer.Enable = false nodeConfig.BatchPoster.Enable = false - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.Sequencer.Enable = false execConfig.Forwarder.RedisUrl = redisUrl execConfig.ForwardingTarget = fallbackPath diff --git a/system_tests/recreatestate_rpc_test.go b/system_tests/recreatestate_rpc_test.go index 09d53669ee..d5a6311cb4 100644 --- a/system_tests/recreatestate_rpc_test.go +++ b/system_tests/recreatestate_rpc_test.go @@ -95,7 +95,7 @@ func removeStatesFromDb(t *testing.T, bc *core.BlockChain, db ethdb.Database, fr func TestRecreateStateForRPCNoDepthLimit(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = arbitrum.InfiniteMaxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -133,7 +133,7 @@ func TestRecreateStateForRPCBigEnoughDepthLimit(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() depthGasLimit := int64(256 * util.NormalizeL2GasForL1GasInitial(800_000, params.GWei)) - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = depthGasLimit execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -170,7 +170,7 @@ func TestRecreateStateForRPCBigEnoughDepthLimit(t *testing.T) { func TestRecreateStateForRPCDepthLimitExceeded(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = int64(200) execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -207,7 +207,7 @@ func TestRecreateStateForRPCMissingBlockParent(t *testing.T) { var headerCacheLimit uint64 = 512 ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = arbitrum.InfiniteMaxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -255,7 +255,7 @@ func TestRecreateStateForRPCBeyondGenesis(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = arbitrum.InfiniteMaxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -293,7 +293,7 @@ func TestRecreateStateForRPCBlockNotFoundWhileRecreating(t *testing.T) { var blockCacheLimit uint64 = 256 ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = arbitrum.InfiniteMaxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -341,7 +341,7 @@ func testSkippingSavingStateAndRecreatingAfterRestart(t *testing.T, cacheConfig ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = maxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -480,7 +480,7 @@ func TestSkippingSavingStateAndRecreatingAfterRestart(t *testing.T) { func TestGettingStateForRPCFullNode(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.Caching.SnapshotCache = 0 // disable snapshots execConfig.Caching.BlockAge = 0 // use only Caching.BlockCount to keep only last N blocks in dirties cache, no matter how new they are execConfig.Sequencer.MaxBlockSpeed = 0 @@ -522,7 +522,7 @@ func TestGettingStateForRPCFullNode(t *testing.T) { func TestGettingStateForRPCHybridArchiveNode(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.Caching.Archive = true // For now Archive node should use HashScheme execConfig.Caching.StateScheme = rawdb.HashScheme diff --git a/system_tests/retryable_test.go b/system_tests/retryable_test.go index 106dfc6d46..aa9fbfd72e 100644 --- a/system_tests/retryable_test.go +++ b/system_tests/retryable_test.go @@ -1042,7 +1042,7 @@ func elevateL2Basefee(t *testing.T, ctx context.Context, builder *NodeBuilder) { _, err = precompilesgen.NewArbosTest(common.HexToAddress("0x69"), builder.L2.Client) Require(t, err, "failed to deploy ArbosTest") - burnAmount := ExecConfigDefaultTest().RPC.RPCGasCap + burnAmount := ExecConfigDefaultTest(t).RPC.RPCGasCap burnTarget := uint64(5 * l2pricing.InitialSpeedLimitPerSecondV6 * l2pricing.InitialBacklogTolerance) for i := uint64(0); i < (burnTarget+burnAmount)/burnAmount; i++ { burnArbGas := arbosTestAbi.Methods["burnArbGas"] diff --git a/system_tests/validation_mock_test.go b/system_tests/validation_mock_test.go index 88421e4c4b..2739c7545e 100644 --- a/system_tests/validation_mock_test.go +++ b/system_tests/validation_mock_test.go @@ -8,9 +8,9 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rpc" "github.com/offchainlabs/nitro/arbnode" @@ -61,8 +61,8 @@ func (s *mockSpawner) WasmModuleRoots() ([]common.Hash, error) { return mockWasmModuleRoots, nil } -func (s *mockSpawner) StylusArchs() []rawdb.Target { - return []rawdb.Target{"mock"} +func (s *mockSpawner) StylusArchs() []ethdb.WasmTarget { + return []ethdb.WasmTarget{"mock"} } func (s *mockSpawner) Launch(entry *validator.ValidationInput, moduleRoot common.Hash) validator.ValidationRun { diff --git a/validator/client/redis/producer.go b/validator/client/redis/producer.go index f98c246d0e..adc2f34af5 100644 --- a/validator/client/redis/producer.go +++ b/validator/client/redis/producer.go @@ -7,6 +7,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/go-redis/redis/v8" "github.com/offchainlabs/nitro/pubsub" @@ -35,7 +36,7 @@ func (c ValidationClientConfig) Enabled() bool { func (c ValidationClientConfig) Validate() error { for _, arch := range c.StylusArchs { - if !rawdb.Target(arch).IsValid() { + if !rawdb.IsSupportedWasmTarget(ethdb.WasmTarget(arch)) { return fmt.Errorf("Invalid stylus arch: %v", arch) } } @@ -162,10 +163,10 @@ func (c *ValidationClient) Name() string { return c.config.Name } -func (c *ValidationClient) StylusArchs() []rawdb.Target { - stylusArchs := make([]rawdb.Target, 0, len(c.config.StylusArchs)) +func (c *ValidationClient) StylusArchs() []ethdb.WasmTarget { + stylusArchs := make([]ethdb.WasmTarget, 0, len(c.config.StylusArchs)) for _, arch := range c.config.StylusArchs { - stylusArchs = append(stylusArchs, rawdb.Target(arch)) + stylusArchs = append(stylusArchs, ethdb.WasmTarget(arch)) } return stylusArchs } diff --git a/validator/client/validation_client.go b/validator/client/validation_client.go index 80cff66675..9dd7780fbc 100644 --- a/validator/client/validation_client.go +++ b/validator/client/validation_client.go @@ -22,6 +22,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rpc" @@ -31,7 +32,7 @@ type ValidationClient struct { stopwaiter.StopWaiter client *rpcclient.RpcClient name string - stylusArchs []rawdb.Target + stylusArchs []ethdb.WasmTarget room atomic.Int32 wasmModuleRoots []common.Hash } @@ -40,7 +41,7 @@ func NewValidationClient(config rpcclient.ClientConfigFetcher, stack *node.Node) return &ValidationClient{ client: rpcclient.NewRpcClient(config, stack), name: "not started", - stylusArchs: []rawdb.Target{"not started"}, + stylusArchs: []ethdb.WasmTarget{"not started"}, } } @@ -67,14 +68,14 @@ func (c *ValidationClient) Start(ctx context.Context) error { if len(name) == 0 { return errors.New("couldn't read name from server") } - var stylusArchs []rawdb.Target + var stylusArchs []ethdb.WasmTarget if err := c.client.CallContext(ctx, &stylusArchs, server_api.Namespace+"_stylusArchs"); err != nil { var rpcError rpc.Error ok := errors.As(err, &rpcError) if !ok || rpcError.ErrorCode() != -32601 { return fmt.Errorf("could not read stylus arch from server: %w", err) } - stylusArchs = []rawdb.Target{rawdb.Target("pre-stylus")} // invalid, will fail if trying to validate block with stylus + stylusArchs = []ethdb.WasmTarget{ethdb.WasmTarget("pre-stylus")} // invalid, will fail if trying to validate block with stylus } else { if len(stylusArchs) == 0 { return fmt.Errorf("could not read stylus archs from validation server") @@ -117,11 +118,11 @@ func (c *ValidationClient) WasmModuleRoots() ([]common.Hash, error) { return nil, errors.New("not started") } -func (c *ValidationClient) StylusArchs() []rawdb.Target { +func (c *ValidationClient) StylusArchs() []ethdb.WasmTarget { if c.Started() { return c.stylusArchs } - return []rawdb.Target{"not started"} + return []ethdb.WasmTarget{"not started"} } func (c *ValidationClient) Stop() { diff --git a/validator/interface.go b/validator/interface.go index 81b40ae5cf..af08629137 100644 --- a/validator/interface.go +++ b/validator/interface.go @@ -4,7 +4,7 @@ import ( "context" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/util/containers" ) @@ -14,7 +14,7 @@ type ValidationSpawner interface { Start(context.Context) error Stop() Name() string - StylusArchs() []rawdb.Target + StylusArchs() []ethdb.WasmTarget Room() int } diff --git a/validator/server_api/json.go b/validator/server_api/json.go index dbe2bb1fee..6fe936e17d 100644 --- a/validator/server_api/json.go +++ b/validator/server_api/json.go @@ -11,7 +11,7 @@ import ( "os" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/arbcompress" "github.com/offchainlabs/nitro/arbutil" @@ -64,7 +64,7 @@ type InputJSON struct { BatchInfo []BatchInfoJson DelayedMsgB64 string StartState validator.GoGlobalState - UserWasms map[rawdb.Target]map[common.Hash]string + UserWasms map[ethdb.WasmTarget]map[common.Hash]string DebugChain bool } @@ -96,7 +96,7 @@ func ValidationInputToJson(entry *validator.ValidationInput) *InputJSON { DelayedMsgB64: base64.StdEncoding.EncodeToString(entry.DelayedMsg), StartState: entry.StartState, PreimagesB64: jsonPreimagesMap, - UserWasms: make(map[rawdb.Target]map[common.Hash]string), + UserWasms: make(map[ethdb.WasmTarget]map[common.Hash]string), DebugChain: entry.DebugChain, } for _, binfo := range entry.BatchInfo { @@ -128,7 +128,7 @@ func ValidationInputFromJson(entry *InputJSON) (*validator.ValidationInput, erro DelayedMsgNr: entry.DelayedMsgNr, StartState: entry.StartState, Preimages: preimages, - UserWasms: make(map[rawdb.Target]map[common.Hash][]byte), + UserWasms: make(map[ethdb.WasmTarget]map[common.Hash][]byte), DebugChain: entry.DebugChain, } delayed, err := base64.StdEncoding.DecodeString(entry.DelayedMsgB64) diff --git a/validator/server_arb/validator_spawner.go b/validator/server_arb/validator_spawner.go index 844a988d28..96bc5d619d 100644 --- a/validator/server_arb/validator_spawner.go +++ b/validator/server_arb/validator_spawner.go @@ -13,6 +13,7 @@ import ( "github.com/spf13/pflag" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/util/containers" "github.com/offchainlabs/nitro/util/stopwaiter" @@ -89,8 +90,8 @@ func (s *ArbitratorSpawner) WasmModuleRoots() ([]common.Hash, error) { return s.locator.ModuleRoots(), nil } -func (s *ArbitratorSpawner) StylusArchs() []rawdb.Target { - return []rawdb.Target{rawdb.TargetWavm} +func (s *ArbitratorSpawner) StylusArchs() []ethdb.WasmTarget { + return []ethdb.WasmTarget{rawdb.TargetWavm} } func (s *ArbitratorSpawner) Name() string { diff --git a/validator/server_jit/spawner.go b/validator/server_jit/spawner.go index 92b50b17cb..d77317d218 100644 --- a/validator/server_jit/spawner.go +++ b/validator/server_jit/spawner.go @@ -10,6 +10,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/util/stopwaiter" "github.com/offchainlabs/nitro/validator" @@ -72,8 +73,8 @@ func (v *JitSpawner) WasmModuleRoots() ([]common.Hash, error) { return v.locator.ModuleRoots(), nil } -func (v *JitSpawner) StylusArchs() []rawdb.Target { - return []rawdb.Target{rawdb.LocalTarget()} +func (v *JitSpawner) StylusArchs() []ethdb.WasmTarget { + return []ethdb.WasmTarget{rawdb.LocalTarget()} } func (v *JitSpawner) execute( diff --git a/validator/validation_entry.go b/validator/validation_entry.go index 2c357659ad..d340993fa2 100644 --- a/validator/validation_entry.go +++ b/validator/validation_entry.go @@ -2,7 +2,7 @@ package validator import ( "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/arbutil" ) @@ -17,7 +17,7 @@ type ValidationInput struct { HasDelayedMsg bool DelayedMsgNr uint64 Preimages map[arbutil.PreimageType]map[common.Hash][]byte - UserWasms map[rawdb.Target]map[common.Hash][]byte + UserWasms map[ethdb.WasmTarget]map[common.Hash][]byte BatchInfo []BatchInfo DelayedMsg []byte StartState GoGlobalState diff --git a/validator/valnode/validation_api.go b/validator/valnode/validation_api.go index a79ac7fa55..a10d931dfc 100644 --- a/validator/valnode/validation_api.go +++ b/validator/valnode/validation_api.go @@ -12,7 +12,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/util/stopwaiter" "github.com/offchainlabs/nitro/validator" @@ -45,7 +45,7 @@ func (a *ValidationServerAPI) WasmModuleRoots() ([]common.Hash, error) { return a.spawner.WasmModuleRoots() } -func (a *ValidationServerAPI) StylusArchs() ([]rawdb.Target, error) { +func (a *ValidationServerAPI) StylusArchs() ([]ethdb.WasmTarget, error) { return a.spawner.StylusArchs(), nil } From eefd7294777775f69689c316ca97c4b9d1338172 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 23 Aug 2024 00:52:30 +0200 Subject: [PATCH 047/489] compile to targets in WasmTargets list --- arbos/programs/native.go | 61 +++++++++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 20 deletions(-) diff --git a/arbos/programs/native.go b/arbos/programs/native.go index 36e200af4d..98abf9443c 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -100,25 +100,50 @@ func activateProgramInternal( } return nil, nil, err } - target := rawdb.LocalTarget() - status_asm := C.stylus_compile( - goSlice(wasm), - u16(version), - cbool(debug), - goSlice([]byte(target)), - output, - ) - asm := output.intoBytes() - if status_asm != 0 { - return nil, nil, fmt.Errorf("%w: %s", ErrProgramActivation, string(asm)) + targets := db.Database().WasmTargets() + type result struct { + target ethdb.WasmTarget + asm []byte + err error } - asmMap := map[ethdb.WasmTarget][]byte{ - rawdb.TargetWavm: module, - target: asm, + results := make(chan result, len(targets)) + for _, target := range targets { + if target == rawdb.TargetWavm { + results <- result{target, module, nil} + } else { + target := target + go func() { + output := &rustBytes{} + status_asm := C.stylus_compile( + goSlice(wasm), + u16(version), + cbool(debug), + goSlice([]byte(target)), + output, + ) + asm := output.intoBytes() + if status_asm != 0 { + results <- result{target, nil, fmt.Errorf("%w: %s", ErrProgramActivation, string(asm))} + return + } + results <- result{target, asm, nil} + }() + } + } + asmMap := make(map[ethdb.WasmTarget][]byte, len(targets)) + for range targets { + res := <-results + if res.err != nil { + err = errors.Join(res.err, err) + } else { + asmMap[res.target] = res.asm + } + } + if err != nil { + return nil, nil, fmt.Errorf("compilation failed for one or more targets: %w", err) } hash := moduleHash.toHash() - info := &activationInfo{ moduleHash: hash, initGas: uint16(stylusData.init_cost), @@ -204,11 +229,7 @@ func callProgram( } if db, ok := db.(*state.StateDB); ok { - targets := []ethdb.WasmTarget{ - rawdb.TargetWavm, - rawdb.LocalTarget(), - } - db.RecordProgram(targets, moduleHash) + db.RecordProgram(moduleHash) } evmApi := newApi(interpreter, tracingInfo, scope, memoryModel) From 2832703599b4749c32e9af9760b4d3ffdacbba9c Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 23 Aug 2024 01:21:02 +0200 Subject: [PATCH 048/489] use wasm targets list in SaveActiveProgramToWasmStore --- arbos/programs/wasmstorehelper.go | 4 ++-- go-ethereum | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arbos/programs/wasmstorehelper.go b/arbos/programs/wasmstorehelper.go index 95daaf544a..434820dd9c 100644 --- a/arbos/programs/wasmstorehelper.go +++ b/arbos/programs/wasmstorehelper.go @@ -12,7 +12,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" ) @@ -44,8 +43,9 @@ func (p Programs) SaveActiveProgramToWasmStore(statedb *state.StateDB, codeHash return err } + targets := statedb.Database().WasmTargets() // If already in wasm store then return early - _, err = statedb.TryGetActivatedAsmMap([]ethdb.WasmTarget{rawdb.TargetWavm, rawdb.LocalTarget()}, moduleHash) + _, err = statedb.TryGetActivatedAsmMap(targets, moduleHash) if err == nil { return nil } diff --git a/go-ethereum b/go-ethereum index cb03c12f7f..c7d753f2c4 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit cb03c12f7f02d47344d2d089eef748e92b99c95e +Subproject commit c7d753f2c4d5eb41c94e2e2499219a72fbf241a0 From 10f9b159550b59d97304cd935db25b076adf6443 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 23 Aug 2024 01:48:47 +0200 Subject: [PATCH 049/489] update geth pin --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index c7d753f2c4..7fc4c64607 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit c7d753f2c4d5eb41c94e2e2499219a72fbf241a0 +Subproject commit 7fc4c646075eb9b6e26374886ea6fe9463b106aa From 17319ad2d1bb00d0c63a22693844652f6b739b1e Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 23 Aug 2024 16:18:01 +0200 Subject: [PATCH 050/489] update geth pin --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 7fc4c64607..0a376604c9 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 7fc4c646075eb9b6e26374886ea6fe9463b106aa +Subproject commit 0a376604c9ce0c588de0d943184380acd7e25db4 From 1a353b21ccc4a52f35ee0df69ba135a304a1ba17 Mon Sep 17 00:00:00 2001 From: Pepper Lebeck-Jobe Date: Fri, 23 Aug 2024 15:18:23 +0000 Subject: [PATCH 051/489] Add a hacky sleep Reviewers, please tell me the "right" way to ensure that the transaction is visible in the L1. I don't like the idea of adding sleeps in tests to ensure that some event has occurred. Is there some RPC I can call or event I can subscribe to that I can then wait for before proceeding? --- system_tests/program_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/system_tests/program_test.go b/system_tests/program_test.go index bb6e34ec4f..9392de0c4d 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -394,6 +394,9 @@ func storageTest(t *testing.T, jit bool) { assertStorageAt(t, ctx, l2client, programAddress, key, value) validateBlocks(t, 2, jit, builder) + // Without this sleep, this test fails in the "-race" CI builds complaining that + // the block has not been seen in L1 yet. + time.Sleep(10 * time.Second) // Captures a block_input_.json file for the block that included the // storage write transaction. recordBlock(t, receipt.BlockNumber.Uint64(), builder) From cdb43d77e5ef976c6ad3e09ac63d07034c85bfcf Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 23 Aug 2024 17:57:18 +0200 Subject: [PATCH 052/489] update validation client stylus archs sanity check --- validator/client/validation_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/client/validation_client.go b/validator/client/validation_client.go index 36316a8328..3b18ad1851 100644 --- a/validator/client/validation_client.go +++ b/validator/client/validation_client.go @@ -81,7 +81,7 @@ func (c *ValidationClient) Start(ctx context.Context) error { return fmt.Errorf("could not read stylus archs from validation server") } for _, stylusArch := range stylusArchs { - if stylusArch != rawdb.TargetWavm && stylusArch != rawdb.LocalTarget() && stylusArch != "mock" { + if !rawdb.IsSupportedWasmTarget(ethdb.WasmTarget(stylusArch)) && stylusArch != "mock" { return fmt.Errorf("unsupported stylus architecture: %v", stylusArch) } } From 71ab928cb42b66247ac9fdc3acd82280255de712 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 23 Aug 2024 19:27:15 +0200 Subject: [PATCH 053/489] change archs list option to extra-archs that must include wavm and is appended with local target if not in list --- execution/gethexec/node.go | 47 ++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index f519cce6ed..bc29e9b7f2 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -30,10 +30,10 @@ import ( ) type StylusTargetConfig struct { - Arm64 string `koanf:"arm64"` - Amd64 string `koanf:"amd64"` - Host string `koanf:"host"` - Archs []string `koanf:"archs"` + Arm64 string `koanf:"arm64"` + Amd64 string `koanf:"amd64"` + Host string `koanf:"host"` + ExtraArchs []string `koanf:"extra-archs"` wasmTargets []ethdb.WasmTarget } @@ -44,47 +44,40 @@ func (c *StylusTargetConfig) WasmTargets() []ethdb.WasmTarget { func (c *StylusTargetConfig) Validate() error { localTarget := rawdb.LocalTarget() - var nativeFound bool - targets := make([]ethdb.WasmTarget, 0, len(c.Archs)) - for _, arch := range c.Archs { + targets := make([]ethdb.WasmTarget, 0, len(c.ExtraArchs)) + targetsSet := make(map[ethdb.WasmTarget]struct{}, len(c.ExtraArchs)+1) + for _, arch := range c.ExtraArchs { target := ethdb.WasmTarget(arch) if !rawdb.IsSupportedWasmTarget(target) { return fmt.Errorf("unsupported architecture: %v, possible values: %s, %s, %s, %s", arch, rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost) } - var alreadyInList bool - for _, t := range targets { - if target == t { - alreadyInList = true - break - } - } - if alreadyInList { - continue + if _, duplicate := targetsSet[target]; !duplicate { + targets = append(targets, target) + targetsSet[target] = struct{}{} } - if target == localTarget { - nativeFound = true - } - targets = append(targets, target) } - if !nativeFound { - return fmt.Errorf("native target not found in archs list, native target: %v, archs: %v", localTarget, c.Archs) + if _, has := targetsSet[rawdb.TargetWavm]; !has { + return fmt.Errorf("%s target not found in archs list, archs: %v", rawdb.TargetWavm, c.ExtraArchs) + } + if _, has := targetsSet[localTarget]; !has { + targets = append(targets, localTarget) } c.wasmTargets = targets return nil } var DefaultStylusTargetConfig = StylusTargetConfig{ - Arm64: programs.DefaultTargetDescriptionArm, - Amd64: programs.DefaultTargetDescriptionX86, - Host: "", - Archs: []string{string(rawdb.TargetWavm), string(rawdb.LocalTarget())}, + Arm64: programs.DefaultTargetDescriptionArm, + Amd64: programs.DefaultTargetDescriptionX86, + Host: "", + ExtraArchs: []string{string(rawdb.TargetWavm)}, } func StylusTargetConfigAddOptions(prefix string, f *flag.FlagSet) { f.String(prefix+".arm64", DefaultStylusTargetConfig.Arm64, "stylus programs compilation target for arm64 linux") f.String(prefix+".amd64", DefaultStylusTargetConfig.Amd64, "stylus programs compilation target for amd64 linux") f.String(prefix+".host", DefaultStylusTargetConfig.Host, "stylus programs compilation target for system other than 64-bit ARM or 64-bit x86") - f.StringSlice(prefix+".archs", DefaultStylusTargetConfig.Archs, fmt.Sprintf("Comma separated architectures list to compile stylus program to and cache in wasm store (available targets: %s, %s, %s, %s)", rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost)) + f.StringSlice(prefix+".extra-archs", DefaultStylusTargetConfig.ExtraArchs, fmt.Sprintf("Comma separated list of extra architectures to cross-compile stylus program to and cache in wasm store (additionally to local target). Currently must include at least %s. (supported targets: %s, %s, %s, %s)", rawdb.TargetWavm, rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost)) } type Config struct { From 40e6b9bce33b2af048561889a7ed914e113def77 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Mon, 26 Aug 2024 13:43:32 +0530 Subject: [PATCH 054/489] address PR comments, handle memory better and add test to cover incorrect request scenario --- pubsub/consumer.go | 18 ++++-- pubsub/producer.go | 129 ++++++++++++++++++++++++++++-------------- pubsub/pubsub_test.go | 91 +++++++++++++++++------------ 3 files changed, 155 insertions(+), 83 deletions(-) diff --git a/pubsub/consumer.go b/pubsub/consumer.go index 9c3785ee3f..2c4787101d 100644 --- a/pubsub/consumer.go +++ b/pubsub/consumer.go @@ -18,7 +18,7 @@ type ConsumerConfig struct { // Timeout of result entry in Redis. ResponseEntryTimeout time.Duration `koanf:"response-entry-timeout"` // Minimum idle time after which messages will be autoclaimed - IdletimeToAutoclaim time.Duration `koanf:"Idletime-to-autoclaim"` + IdletimeToAutoclaim time.Duration `koanf:"idletime-to-autoclaim"` } var DefaultConsumerConfig = ConsumerConfig{ @@ -33,7 +33,7 @@ var TestConsumerConfig = ConsumerConfig{ func ConsumerConfigAddOptions(prefix string, f *pflag.FlagSet) { f.Duration(prefix+".response-entry-timeout", DefaultConsumerConfig.ResponseEntryTimeout, "timeout for response entry") - f.Duration(prefix+".Idletime-to-autoclaim", DefaultConsumerConfig.IdletimeToAutoclaim, "After a message spends this amount of time in PEL (Pending Entries List i.e claimed by another consumer but not Acknowledged) it will be allowed to be autoclaimed by other consumers") + f.Duration(prefix+".idletime-to-autoclaim", DefaultConsumerConfig.IdletimeToAutoclaim, "After a message spends this amount of time in PEL (Pending Entries List i.e claimed by another consumer but not Acknowledged) it will be allowed to be autoclaimed by other consumers") } // Consumer implements a consumer for redis stream provides heartbeat to @@ -93,9 +93,12 @@ func (c *Consumer[Request, Response]) Consume(ctx context.Context) (*Message[Req MinIdle: c.cfg.IdletimeToAutoclaim, // Minimum idle time for messages to claim (in milliseconds) Stream: c.redisStream, Start: "0", - Count: 1, // Limit the number of messages to claim + Count: 5, // Try looking for 50 entries in PEL, this assumes there are a maximum of 50 consumers in this redisGroup }).Result() - if len(messages) != 1 || err != nil { + if len(messages) == 0 || err != nil { + if err != nil { + log.Error("error from xautoclaim", "err", err) + } // Fallback to reading new messages res, err := c.client.XReadGroup(ctx, &redis.XReadGroupArgs{ Group: c.redisGroup, @@ -132,7 +135,9 @@ func (c *Consumer[Request, Response]) Consume(ctx context.Context) (*Message[Req ackNotifier := make(chan struct{}) c.StopWaiter.LaunchThread(func(ctx context.Context) { for { - if err := c.client.XClaim(ctx, &redis.XClaimArgs{ + // Use XClaimJustID so that we would have clear difference between invalid requests that are claimed multiple times due to xautoclaim and + // valid requests that are just being claimed in regular intervals to indicate heartbeat + if err := c.client.XClaimJustID(ctx, &redis.XClaimArgs{ Stream: c.redisStream, Group: c.redisGroup, Consumer: c.id, @@ -174,5 +179,8 @@ func (c *Consumer[Request, Response]) SetResult(ctx context.Context, id string, if _, err := c.client.XAck(ctx, c.redisStream, c.redisGroup, messageID).Result(); err != nil { return fmt.Errorf("acking message: %v, error: %w", messageID, err) } + if _, err := c.client.XDel(ctx, c.redisStream, messageID).Result(); err != nil { + return fmt.Errorf("deleting message: %v, error: %w", messageID, err) + } return nil } diff --git a/pubsub/producer.go b/pubsub/producer.go index df6e7d5a28..cf5dfdbd36 100644 --- a/pubsub/producer.go +++ b/pubsub/producer.go @@ -57,21 +57,26 @@ type ProducerConfig struct { CheckResultInterval time.Duration `koanf:"check-result-interval"` // Timeout of entry's written to redis by producer ResponseEntryTimeout time.Duration `koanf:"response-entry-timeout"` + // RequestTimeout is a TTL for any message sent to the redis stream + RequestTimeout time.Duration `koanf:"request-timeout"` } var DefaultProducerConfig = ProducerConfig{ CheckResultInterval: 5 * time.Second, ResponseEntryTimeout: time.Hour, + RequestTimeout: time.Hour, // should we increase this? } var TestProducerConfig = ProducerConfig{ CheckResultInterval: 5 * time.Millisecond, ResponseEntryTimeout: time.Minute, + RequestTimeout: 2 * time.Second, } func ProducerAddConfigAddOptions(prefix string, f *pflag.FlagSet) { f.Duration(prefix+".check-result-interval", DefaultProducerConfig.CheckResultInterval, "interval in which producer checks pending messages whether consumer processing them is inactive") f.Duration(prefix+".response-entry-timeout", DefaultProducerConfig.ResponseEntryTimeout, "timeout after which responses written from producer to the redis are cleared. Currently used for the key mapping unique request id to redis stream message id") + f.Duration(prefix+".request-timeout", DefaultProducerConfig.RequestTimeout, "timeout after which the message in redis stream is considered as errored, this prevents workers from working on wrong requests indefinitely") } func NewProducer[Request any, Response any](client redis.UniversalClient, streamName string, cfg *ProducerConfig) (*Producer[Request, Response], error) { @@ -91,37 +96,58 @@ func NewProducer[Request any, Response any](client redis.UniversalClient, stream }, nil } -func setMaxMsgIdInt(maxMsgIdInt *[2]uint64, msgId string) error { - idParts := strings.Split(msgId, "-") - if len(idParts) != 2 { - return fmt.Errorf("invalid i.d: %v", msgId) +// cmpMsgId compares two msgid's and returns (0) if equal, (-1) if msgId1 < msgId2, (1) if msgId1 > msgId2, (-2) if not comparable (or error) +func cmpMsgId(msgId1, msgId2 string) int { + getUintParts := func(msgId string) ([2]uint64, error) { + idParts := strings.Split(msgId, "-") + if len(idParts) != 2 { + return [2]uint64{}, fmt.Errorf("invalid i.d: %v", msgId) + } + idTimeStamp, err := strconv.ParseUint(idParts[0], 10, 64) + if err != nil { + return [2]uint64{}, fmt.Errorf("invalid i.d: %v err: %w", msgId, err) + } + idSerial, err := strconv.ParseUint(idParts[1], 10, 64) + if err != nil { + return [2]uint64{}, fmt.Errorf("invalid i.d serial: %v err: %w", msgId, err) + } + return [2]uint64{idTimeStamp, idSerial}, nil } - idTimeStamp, err := strconv.ParseUint(idParts[0], 10, 64) + id1, err := getUintParts(msgId1) if err != nil { - return fmt.Errorf("invalid i.d: %v err: %w", msgId, err) - } - if idTimeStamp < maxMsgIdInt[0] { - return nil + log.Trace("error comparing msgIds", "msgId1", msgId1, "msgId2", msgId2) + return -2 } - idSerial, err := strconv.ParseUint(idParts[1], 10, 64) + id2, err := getUintParts(msgId2) if err != nil { - return fmt.Errorf("invalid i.d serial: %v err: %w", msgId, err) + log.Trace("error comparing msgIds", "msgId1", msgId1, "msgId2", msgId2) + return -2 } - if idTimeStamp > maxMsgIdInt[0] { - maxMsgIdInt[0] = idTimeStamp - maxMsgIdInt[1] = idSerial - return nil + if id1[0] < id2[0] { + return -1 + } else if id1[0] > id2[0] { + return 1 + } else if id1[1] < id2[1] { + return -1 + } else if id1[1] > id2[1] { + return 1 } - // idTimeStamp == maxMsgIdInt[0] - if idSerial > maxMsgIdInt[1] { - maxMsgIdInt[1] = idSerial - } - return nil + return 0 } // checkResponses checks iteratively whether response for the promise is ready. func (p *Producer[Request, Response]) checkResponses(ctx context.Context) time.Duration { - maxMsgIdInt := [2]uint64{0, 0} + pelData, err := p.client.XPending(ctx, p.redisStream, p.redisGroup).Result() + if err != nil { + log.Error("error getting PEL data from xpending, xtrimming is disabled", "err", err) + } + deletePromise := func(id string) { + // Try deleting UNIQUEID_MSGID_MAP_KEY corresponding to this id from redis + if err := p.client.Del(ctx, MessageKeyFor(p.redisStream, id)+UNIQUEID_MSGID_MAP_KEY).Err(); err != nil { + log.Error("Error deleting key from redis that flags that a request is being processed", "err", err) + } + delete(p.promises, id) + } p.promisesLock.Lock() defer p.promisesLock.Unlock() responded := 0 @@ -135,16 +161,22 @@ func (p *Producer[Request, Response]) checkResponses(ctx context.Context) time.D if err != nil { if !errors.Is(err, redis.Nil) { log.Error("Error reading value in redis", "key", id, "error", err) + } else { + // The request this producer is waiting for has been past its TTL or is older than current PEL's lower, + // so safe to error and stop tracking this promise + allowedOldestID := fmt.Sprintf("%d-0", time.Now().Add(-p.cfg.RequestTimeout).UnixMilli()) + if pelData != nil && pelData.Lower != "" { + allowedOldestID = pelData.Lower + } + if cmpMsgId(msgIDAndPromise.msgID, allowedOldestID) == -1 { + msgIDAndPromise.promise.ProduceError(errors.New("error getting response, request has been waiting for too long")) + log.Error("error getting response, request has been waiting past its TTL") + errored++ + deletePromise(id) + } } continue } - // We keep track of a maxMsgId of a successfully solved request, because messages - // with id lower than this are either ack-ed or in PEL, so its safe to call XTRIMMINID on maxMsgId - errSetId := setMaxMsgIdInt(&maxMsgIdInt, msgIDAndPromise.msgID) - if errSetId != nil { - log.Error("error setting maxMsgId", "err", err) - return p.cfg.CheckResultInterval - } var resp Response if err := json.Unmarshal([]byte(res), &resp); err != nil { msgIDAndPromise.promise.ProduceError(fmt.Errorf("error unmarshalling: %w", err)) @@ -154,21 +186,36 @@ func (p *Producer[Request, Response]) checkResponses(ctx context.Context) time.D msgIDAndPromise.promise.Produce(resp) responded++ } - // Try deleting UNIQUEID_MSGID_MAP_KEY corresponding to this id from redis - if err := p.client.Del(ctx, msgKey+UNIQUEID_MSGID_MAP_KEY).Err(); err != nil { - log.Error("Error deleting key from redis that flags that a request is being processed", "err", err) - } - delete(p.promises, id) + deletePromise(id) } - var trimmed int64 - var trimErr error - maxMsgId := "+" - // If at least response for one promise was found, find the maximum of the found ones and XTRIMMINID from that msg id + 1 - if maxMsgIdInt[0] > 0 { - maxMsgId = fmt.Sprintf("%d-%d", maxMsgIdInt[0], maxMsgIdInt[1]+1) - trimmed, trimErr = p.client.XTrimMinID(ctx, p.redisStream, maxMsgId).Result() + // XDEL on consumer side already deletes acked messages (mark as deleted) but doesnt claim the memory back, XTRIM helps in claiming this memory in normal conditions + // pelData might be outdated when we do the xtrim, but thats ok as the messages are also being trimmed by other producers + if pelData != nil && pelData.Lower != "" { + trimmed, trimErr := p.client.XTrimMinID(ctx, p.redisStream, pelData.Lower).Result() + log.Trace("trimming", "xTrimMinID", pelData.Lower, "trimmed", trimmed, "responded", responded, "errored", errored, "trim-err", trimErr) + // Check if pelData.Lower has been past its TTL and if it is then ack it to remove from PEL and delete it, once + // its taken out from PEL the producer that sent this request will handle the corresponding promise accordingly (if PEL is non-empty) + allowedOldestID := fmt.Sprintf("%d-0", time.Now().Add(-p.cfg.RequestTimeout).UnixMilli()) + if cmpMsgId(pelData.Lower, allowedOldestID) == -1 { + if err := p.client.XClaim(ctx, &redis.XClaimArgs{ + Stream: p.redisStream, + Group: p.redisGroup, + Consumer: p.id, + MinIdle: 0, + Messages: []string{pelData.Lower}, + }).Err(); err != nil { + log.Error("error claiming PEL's lower message thats past its TTL", "msgID", pelData.Lower, "err", err) + return p.cfg.CheckResultInterval + } + if _, err := p.client.XAck(ctx, p.redisStream, p.redisGroup, pelData.Lower).Result(); err != nil { + log.Error("error acking PEL's lower message thats past its TTL", "msgID", pelData.Lower, "err", err) + return p.cfg.CheckResultInterval + } + if _, err := p.client.XDel(ctx, p.redisStream, pelData.Lower).Result(); err != nil { + log.Error("error deleting PEL's lower message thats past its TTL", "msgID", pelData.Lower, "err", err) + } + } } - log.Trace("trimming", "xTrimMinID", maxMsgId, "trimmed", trimmed, "responded", responded, "errored", errored, "trim-err", trimErr) return p.cfg.CheckResultInterval } diff --git a/pubsub/pubsub_test.go b/pubsub/pubsub_test.go index b1ffdca0fd..3883420f4e 100644 --- a/pubsub/pubsub_test.go +++ b/pubsub/pubsub_test.go @@ -26,8 +26,9 @@ var ( ) type testRequest struct { - Request string - SelfHash string // Is a unique identifier which can be used to compare any two validationInputs + Request string + IsInvalid bool + SelfHash string // Is a unique identifier which can be used to compare any two validationInputs } // SetSelfHash should be only called once. In the context of redis streams- by the producer @@ -63,6 +64,7 @@ func producerCfg() *ProducerConfig { return &ProducerConfig{ CheckResultInterval: TestProducerConfig.CheckResultInterval, ResponseEntryTimeout: TestProducerConfig.ResponseEntryTimeout, + RequestTimeout: TestProducerConfig.RequestTimeout, } } @@ -136,10 +138,13 @@ func flatten(responses [][]string) []string { return ret } -func produceMessages(ctx context.Context, msgs []string, producer *Producer[testRequest, testResponse], useUniqueIdentifier bool) ([]*containers.Promise[testResponse], error) { +func produceMessages(ctx context.Context, msgs []string, producer *Producer[testRequest, testResponse], useUniqueIdentifier, withInvalidEntries bool) ([]*containers.Promise[testResponse], error) { var promises []*containers.Promise[testResponse] for i := 0; i < len(msgs); i++ { req := testRequest{Request: msgs[i]} + if withInvalidEntries && i%50 == 0 { + req.IsInvalid = true + } if useUniqueIdentifier { req.SetSelfHash() } @@ -194,12 +199,14 @@ func consume(ctx context.Context, t *testing.T, consumers []*Consumer[testReques continue } gotMessages[idx][res.ID] = res.Value.Request - resp := fmt.Sprintf("result for: %v", res.ID) - if err := c.SetResult(ctx, res.Value.SelfHash, res.ID, testResponse{Response: resp}); err != nil { - t.Errorf("Error setting a result: %v", err) + if !res.Value.IsInvalid { + resp := fmt.Sprintf("result for: %v", res.ID) + if err := c.SetResult(ctx, res.Value.SelfHash, res.ID, testResponse{Response: resp}); err != nil { + t.Errorf("Error setting a result: %v", err) + } + wantResponses[idx] = append(wantResponses[idx], resp) } close(ackNotifier) - wantResponses[idx] = append(wantResponses[idx], resp) } }) } @@ -210,45 +217,50 @@ func TestRedisProduceComplex(t *testing.T) { log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelTrace, true))) t.Parallel() for _, tc := range []struct { - name string - entries1Count int - entries2Count int - numProducers int - withDuplicates bool // If this is set, then every fourth entry (while generation) of each entries list is equal - killConsumers bool + name string + entriesCount []int + numProducers int + withDuplicates bool // If this is set, then every fourth entry (while generation) of each entries list is equal + killConsumers bool + withInvalidEntries bool // If this is set, then every 50th entry is invalid (requests that can't be solved by any consumer) }{ { - name: "one producer, all consumers are active", - entries1Count: messagesCount, - numProducers: 1, + name: "one producer, all consumers are active", + entriesCount: []int{messagesCount}, + numProducers: 1, }, { name: "one producer, some consumers killed, others should take over their work", - entries1Count: messagesCount, + entriesCount: []int{messagesCount}, numProducers: 1, killConsumers: true, }, { - name: "two producers, all consumers are active, all unique entries", - entries1Count: 20, - entries2Count: 20, - numProducers: 2, + name: "two producers, all consumers are active, all unique entries", + entriesCount: []int{20, 20}, + numProducers: 2, }, { name: "two producers, all consumers are active, some duplicate entries", - entries1Count: 20, - entries2Count: 20, + entriesCount: []int{20, 20}, numProducers: 2, withDuplicates: true, }, { name: "two producers, some consumers killed, others should take over their work, some duplicate entries, unequal number of requests from producers", - entries1Count: messagesCount, - entries2Count: 2 * messagesCount, + entriesCount: []int{messagesCount, 2 * messagesCount}, numProducers: 2, withDuplicates: true, killConsumers: true, }, + { + name: "two producers, some consumers killed, others should take over their work, some duplicate entries, some invalid entries, unequal number of requests from producers", + entriesCount: []int{messagesCount, 2 * messagesCount}, + numProducers: 2, + withDuplicates: true, + killConsumers: true, + withInvalidEntries: true, + }, } { t.Run(tc.name, func(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) @@ -271,15 +283,15 @@ func TestRedisProduceComplex(t *testing.T) { var entries [][]string if tc.numProducers == 2 { - entries = append(entries, wantMessages(tc.entries1Count, "1.", tc.withDuplicates)) - entries = append(entries, wantMessages(tc.entries2Count, "2.", tc.withDuplicates)) + entries = append(entries, wantMessages(tc.entriesCount[0], "1.", tc.withDuplicates)) + entries = append(entries, wantMessages(tc.entriesCount[1], "2.", tc.withDuplicates)) } else { - entries = append(entries, wantMessages(tc.entries1Count, "", tc.withDuplicates)) + entries = append(entries, wantMessages(tc.entriesCount[0], "", tc.withDuplicates)) } var promises [][]*containers.Promise[testResponse] for i := 0; i < tc.numProducers; i++ { - prs, err := produceMessages(ctx, entries[i], producers[i], tc.numProducers == 2) + prs, err := produceMessages(ctx, entries[i], producers[i], tc.numProducers == 2, tc.withInvalidEntries) if err != nil { t.Fatalf("Error producing messages from producer%d: %v", i, err) } @@ -311,8 +323,17 @@ func TestRedisProduceComplex(t *testing.T) { var gotResponses []string for i := 0; i < tc.numProducers; i++ { grs, errIndexes := awaitResponses(ctx, promises[i]) - if len(errIndexes) != 0 { - t.Fatalf("Error awaiting responses from promises%d: %v", i, errIndexes) + if tc.withInvalidEntries { + if errIndexes[len(errIndexes)-1]+50 <= len(entries[i]) { + t.Fatalf("Unexpected number of invalid requests while awaiting responses") + } + for j, idx := range errIndexes { + if idx != j*50 { + t.Fatalf("Invalid request' index mismatch want: %d got %d", j*50, idx) + } + } + } else if len(errIndexes) != 0 { + t.Fatalf("Error awaiting responses from promises %d: %v", i, errIndexes) } gotResponses = append(gotResponses, grs...) } @@ -325,6 +346,7 @@ func TestRedisProduceComplex(t *testing.T) { if err != nil { t.Fatalf("mergeMaps() unexpected error: %v", err) } + got = removeDuplicates(got) var combinedEntries []string for i := 0; i < tc.numProducers; i++ { @@ -384,14 +406,9 @@ func removeDuplicates(list []string) []string { // mergeValues merges maps from the slice and returns their values. // Returns and error if there exists duplicate key. func mergeValues(messages []map[string]string) ([]string, error) { - res := make(map[string]any) var ret []string for _, m := range messages { - for k, v := range m { - if _, found := res[k]; found { - return nil, fmt.Errorf("duplicate key: %v", k) - } - res[k] = v + for _, v := range m { ret = append(ret, v) } } From b5fff687e82b90d97f347b47be5695edb5032e4b Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Mon, 26 Aug 2024 13:48:46 +0530 Subject: [PATCH 055/489] increase TestProducerConfig requestTimeout --- pubsub/producer.go | 2 +- pubsub/pubsub_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pubsub/producer.go b/pubsub/producer.go index cf5dfdbd36..ee5d4de528 100644 --- a/pubsub/producer.go +++ b/pubsub/producer.go @@ -70,7 +70,7 @@ var DefaultProducerConfig = ProducerConfig{ var TestProducerConfig = ProducerConfig{ CheckResultInterval: 5 * time.Millisecond, ResponseEntryTimeout: time.Minute, - RequestTimeout: 2 * time.Second, + RequestTimeout: time.Minute, } func ProducerAddConfigAddOptions(prefix string, f *pflag.FlagSet) { diff --git a/pubsub/pubsub_test.go b/pubsub/pubsub_test.go index 3883420f4e..3e03af3f48 100644 --- a/pubsub/pubsub_test.go +++ b/pubsub/pubsub_test.go @@ -64,7 +64,7 @@ func producerCfg() *ProducerConfig { return &ProducerConfig{ CheckResultInterval: TestProducerConfig.CheckResultInterval, ResponseEntryTimeout: TestProducerConfig.ResponseEntryTimeout, - RequestTimeout: TestProducerConfig.RequestTimeout, + RequestTimeout: 2 * time.Second, } } From cee4620308d8c3d735e35f9ef9eccd2c2a4eb321 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Mon, 26 Aug 2024 14:16:34 +0530 Subject: [PATCH 056/489] fix tests --- pubsub/pubsub_test.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/pubsub/pubsub_test.go b/pubsub/pubsub_test.go index 3e03af3f48..c4e11b8a72 100644 --- a/pubsub/pubsub_test.go +++ b/pubsub/pubsub_test.go @@ -342,11 +342,14 @@ func TestRedisProduceComplex(t *testing.T) { c.StopAndWait() } - got, err := mergeValues(gotMessages) + got, err := mergeValues(gotMessages, tc.withInvalidEntries) if err != nil { t.Fatalf("mergeMaps() unexpected error: %v", err) } - got = removeDuplicates(got) + // Only when there are invalid entries got will have duplicates + if tc.withInvalidEntries { + got = removeDuplicates(got) + } var combinedEntries []string for i := 0; i < tc.numProducers; i++ { @@ -405,10 +408,15 @@ func removeDuplicates(list []string) []string { // mergeValues merges maps from the slice and returns their values. // Returns and error if there exists duplicate key. -func mergeValues(messages []map[string]string) ([]string, error) { +func mergeValues(messages []map[string]string, withInvalidEntries bool) ([]string, error) { + res := make(map[string]any) var ret []string for _, m := range messages { - for _, v := range m { + for k, v := range m { + if _, found := res[k]; found && !withInvalidEntries { + return nil, fmt.Errorf("duplicate key: %v", k) + } + res[k] = v ret = append(ret, v) } } From 304f2c1127a4def9d292b617b3b52a437e9891b2 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Mon, 26 Aug 2024 16:53:34 +0200 Subject: [PATCH 057/489] add cross compilation to progam test --- system_tests/common_test.go | 5 ++ system_tests/program_test.go | 92 +++++++++++++++++++++++++++++++++--- 2 files changed, 90 insertions(+), 7 deletions(-) diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 392f6de805..457dae0910 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -293,6 +293,11 @@ func (b *NodeBuilder) WithWasmRootDir(wasmRootDir string) *NodeBuilder { return b } +func (b *NodeBuilder) WithExtraArchs(targets []string) *NodeBuilder { + b.execConfig.StylusTarget.ExtraArchs = targets + return b +} + func (b *NodeBuilder) Build(t *testing.T) func() { b.CheckConfig(t) if b.withL1 { diff --git a/system_tests/program_test.go b/system_tests/program_test.go index ed640809db..8c4ca2e438 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -45,18 +45,31 @@ import ( var oneEth = arbmath.UintToBig(1e18) +var allWasmTargets = []string{string(rawdb.TargetWavm), string(rawdb.TargetArm64), string(rawdb.TargetAmd64), string(rawdb.TargetHost)} + func TestProgramKeccak(t *testing.T) { t.Parallel() - keccakTest(t, true) + t.Run("WithDefaultWasmTargets", func(t *testing.T) { + keccakTest(t, true) + }) + + t.Run("WithAllWasmTargets", func(t *testing.T) { + keccakTest(t, true, func(builder *NodeBuilder) { + builder.WithExtraArchs(allWasmTargets) + }) + }) } -func keccakTest(t *testing.T, jit bool) { - builder, auth, cleanup := setupProgramTest(t, jit) +func keccakTest(t *testing.T, jit bool, builderOpts ...func(*NodeBuilder)) { + builder, auth, cleanup := setupProgramTest(t, jit, builderOpts...) ctx := builder.ctx l2client := builder.L2.Client defer cleanup() programAddress := deployWasm(t, ctx, auth, l2client, rustFile("keccak")) + wasmDb := builder.L2.ExecNode.Backend.ArbInterface().BlockChain().StateCache().WasmStore() + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) + wasm, _ := readWasmFile(t, rustFile("keccak")) otherAddressSameCode := deployContract(t, ctx, auth, l2client, wasm) arbWasm, err := pgen.NewArbWasm(types.ArbWasmAddress, l2client) @@ -68,6 +81,7 @@ func keccakTest(t *testing.T, jit bool) { Fatal(t, "activate should have failed with ProgramUpToDate", err) } }) + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) if programAddress == otherAddressSameCode { Fatal(t, "expected to deploy at two separate program addresses") @@ -141,11 +155,18 @@ func keccakTest(t *testing.T, jit bool) { func TestProgramActivateTwice(t *testing.T) { t.Parallel() - testActivateTwice(t, true) + t.Run("WithDefaultWasmTargets", func(t *testing.T) { + testActivateTwice(t, true) + }) + t.Run("WithAllWasmTargets", func(t *testing.T) { + testActivateTwice(t, true, func(builder *NodeBuilder) { + builder.WithExtraArchs(allWasmTargets) + }) + }) } -func testActivateTwice(t *testing.T, jit bool) { - builder, auth, cleanup := setupProgramTest(t, jit) +func testActivateTwice(t *testing.T, jit bool, builderOpts ...func(*NodeBuilder)) { + builder, auth, cleanup := setupProgramTest(t, jit, builderOpts...) ctx := builder.ctx l2info := builder.L2Info l2client := builder.L2.Client @@ -171,6 +192,10 @@ func testActivateTwice(t *testing.T, jit bool) { colors.PrintBlue("keccak program B deployed to ", keccakB) multiAddr := deployWasm(t, ctx, auth, l2client, rustFile("multicall")) + + wasmDb := builder.L2.ExecNode.Backend.ArbInterface().BlockChain().StateCache().WasmStore() + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) + preimage := []byte("it's time to du-du-du-du d-d-d-d-d-d-d de-duplicate") keccakArgs := []byte{0x01} // keccak the preimage once @@ -194,6 +219,7 @@ func testActivateTwice(t *testing.T, jit bool) { // Calling the contract pre-activation should fail. checkReverts() + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) // mechanisms for creating calldata activateProgram, _ := util.NewCallParser(pgen.ArbWasmABI, "activateProgram") @@ -216,6 +242,7 @@ func testActivateTwice(t *testing.T, jit bool) { // Ensure the revert also reverted keccak's activation checkReverts() + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) // Activate keccak program A, then call into B, which should succeed due to being the same codehash args = argsForMulticall(vm.CALL, types.ArbWasmAddress, oneEth, pack(activateProgram(keccakA))) @@ -223,6 +250,7 @@ func testActivateTwice(t *testing.T, jit bool) { tx = l2info.PrepareTxTo("Owner", &multiAddr, 1e9, oneEth, args) ensure(tx, l2client.SendTransaction(ctx, tx)) + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 2) validateBlocks(t, 7, jit, builder) } @@ -1834,7 +1862,9 @@ func createMapFromDb(db ethdb.KeyValueStore) (map[string][]byte, error) { } func TestWasmStoreRebuilding(t *testing.T) { - builder, auth, cleanup := setupProgramTest(t, true) + builder, auth, cleanup := setupProgramTest(t, true, func(b *NodeBuilder) { + b.WithExtraArchs(allWasmTargets) + }) ctx := builder.ctx l2info := builder.L2Info l2client := builder.L2.Client @@ -1871,6 +1901,7 @@ func TestWasmStoreRebuilding(t *testing.T) { storeMap, err := createMapFromDb(wasmDb) Require(t, err) + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) // close nodeB cleanupB() @@ -1926,5 +1957,52 @@ func TestWasmStoreRebuilding(t *testing.T) { } } + checkWasmStoreContent(t, wasmDbAfterRebuild, builder.execConfig.StylusTarget.ExtraArchs, 1) cleanupB() } + +func readModuleHashes(t *testing.T, wasmDb ethdb.KeyValueStore) []common.Hash { + modulesSet := make(map[common.Hash]struct{}) + asmPrefix := []byte{0x00, 'w'} + it := wasmDb.NewIterator(asmPrefix, nil) + defer it.Release() + for it.Next() { + key := it.Key() + if len(key) != rawdb.WasmKeyLen { + t.Fatalf("unexpected activated module key length, len: %d, key: %v", len(key), key) + } + moduleHash := key[rawdb.WasmPrefixLen:] + if len(moduleHash) != common.HashLength { + t.Fatalf("Invalid moduleHash length in key: %v, moduleHash: %v", key, moduleHash) + } + modulesSet[common.BytesToHash(moduleHash)] = struct{}{} + } + modules := make([]common.Hash, 0, len(modulesSet)) + for module := range modulesSet { + modules = append(modules, module) + } + return modules +} + +func checkWasmStoreContent(t *testing.T, wasmDb ethdb.KeyValueStore, targets []string, numModules int) { + modules := readModuleHashes(t, wasmDb) + if len(modules) != numModules { + t.Fatalf("Unexpected number of module hashes found in wasm store, want: %d, have: %d", numModules, len(modules)) + } + for _, module := range modules { + for _, target := range targets { + wasmTarget := ethdb.WasmTarget(target) + if !rawdb.IsSupportedWasmTarget(wasmTarget) { + t.Fatalf("internal test error - unsupported target passed to checkWasmStoreContent: %v", target) + } + func() { + defer func() { + if r := recover(); r != nil { + t.Fatalf("Failed to read activated asm for target: %v, module: %v", target, module) + } + }() + _ = rawdb.ReadActivatedAsm(wasmDb, wasmTarget, module) + }() + } + } +} From ea0cbc672d82802e91be407b595463d6d9415f8c Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Tue, 27 Aug 2024 15:05:59 +0200 Subject: [PATCH 058/489] update geth pin --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 0a376604c9..4ca4b4d734 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 0a376604c9ce0c588de0d943184380acd7e25db4 +Subproject commit 4ca4b4d734021df09cb42a796ca8f263d47383f9 From 2a3dc1d6a805b77f08d6f8e48a1286d0e2c11d44 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Tue, 27 Aug 2024 23:35:10 +0530 Subject: [PATCH 059/489] rectify xautoclaim logic and address PR comments --- pubsub/consumer.go | 66 +++++++++++++++++++++++++++-------- pubsub/producer.go | 31 ++++++++-------- validator/validation_entry.go | 5 +++ 3 files changed, 72 insertions(+), 30 deletions(-) diff --git a/pubsub/consumer.go b/pubsub/consumer.go index 2c4787101d..20bfccb6a8 100644 --- a/pubsub/consumer.go +++ b/pubsub/consumer.go @@ -5,6 +5,9 @@ import ( "encoding/json" "errors" "fmt" + "math" + "math/rand" + "strconv" "time" "github.com/ethereum/go-ethereum/log" @@ -82,23 +85,54 @@ func (c *Consumer[Request, Response]) StreamName() string { return c.redisStream } +func decrementMsgIdByOne(msgId string) string { + id, err := getUintParts(msgId) + if err != nil { + log.Error("Error decrementing start of XAutoClaim by one, defaulting to 0", "err", err) + return "0" + } + if id[1] > 0 { + return strconv.FormatUint(id[0], 10) + "-" + strconv.FormatUint(id[1]-1, 10) + } else if id[0] > 0 { + return strconv.FormatUint(id[0]-1, 10) + "-" + strconv.FormatUint(math.MaxUint64, 10) + } else { + log.Error("Error decrementing start of XAutoClaim by one, defaulting to 0", "err", err) + return "0" + } +} + // Consumer first checks it there exists pending message that is claimed by // unresponsive consumer, if not then reads from the stream. func (c *Consumer[Request, Response]) Consume(ctx context.Context) (*Message[Request], chan struct{}, error) { - // First try to XAUTOCLAIM, this prioritizes processing PEL messages - // that have been waiting for more than IdletimeToAutoclaim duration - messages, _, err := c.client.XAutoClaim(ctx, &redis.XAutoClaimArgs{ - Group: c.redisGroup, - Consumer: c.id, - MinIdle: c.cfg.IdletimeToAutoclaim, // Minimum idle time for messages to claim (in milliseconds) - Stream: c.redisStream, - Start: "0", - Count: 5, // Try looking for 50 entries in PEL, this assumes there are a maximum of 50 consumers in this redisGroup - }).Result() - if len(messages) == 0 || err != nil { + // First try to XAUTOCLAIM, with start as a random messageID from PEL with MinIdle as IdletimeToAutoclaim + // this prioritizes processing PEL messages that have been waiting for more than IdletimeToAutoclaim duration + var messages []redis.XMessage + if pendingMsgs, err := c.client.XPendingExt(ctx, &redis.XPendingExtArgs{ + Stream: c.redisStream, + Group: c.redisGroup, + Start: "-", + End: "+", + Count: math.MaxInt64, + Idle: c.cfg.IdletimeToAutoclaim, + }).Result(); err != nil { + if !errors.Is(err, redis.Nil) { + log.Error("Error from XpendingExt in getting PEL for auto claim", "err", err, "penindlen", len(pendingMsgs)) + } + } else if len(pendingMsgs) > 0 { + idx := rand.Intn(len(pendingMsgs)) + messages, _, err = c.client.XAutoClaim(ctx, &redis.XAutoClaimArgs{ + Group: c.redisGroup, + Consumer: c.id, + MinIdle: c.cfg.IdletimeToAutoclaim, // Minimum idle time for messages to claim (in milliseconds) + Stream: c.redisStream, + Start: decrementMsgIdByOne(pendingMsgs[idx].ID), + Count: 1, + }).Result() if err != nil { log.Error("error from xautoclaim", "err", err) } + } + if len(messages) == 0 { // Fallback to reading new messages res, err := c.client.XReadGroup(ctx, &redis.XReadGroupArgs{ Group: c.redisGroup, @@ -126,7 +160,7 @@ func (c *Consumer[Request, Response]) Consume(ctx context.Context) (*Message[Req data, ok = (value).(string) ) if !ok { - return nil, nil, fmt.Errorf("casting request to string: %w", err) + return nil, nil, errors.New("error casting request to string") } var req Request if err := json.Unmarshal([]byte(data), &req); err != nil { @@ -137,14 +171,16 @@ func (c *Consumer[Request, Response]) Consume(ctx context.Context) (*Message[Req for { // Use XClaimJustID so that we would have clear difference between invalid requests that are claimed multiple times due to xautoclaim and // valid requests that are just being claimed in regular intervals to indicate heartbeat - if err := c.client.XClaimJustID(ctx, &redis.XClaimArgs{ + if ids, err := c.client.XClaimJustID(ctx, &redis.XClaimArgs{ Stream: c.redisStream, Group: c.redisGroup, Consumer: c.id, MinIdle: 0, Messages: []string{messages[0].ID}, - }).Err(); err != nil { - log.Error("error claiming message, it might be possible that other consumers might pick this request", "msgID", messages[0].ID) + }).Result(); err != nil { + log.Error("Error claiming message, it might be possible that other consumers might pick this request", "msgID", messages[0].ID) + } else if len(ids) != 1 { + log.Warn("XClaimJustID returned empty response when indicating hearbeat", "msgID", messages[0].ID) } select { case <-ackNotifier: diff --git a/pubsub/producer.go b/pubsub/producer.go index ee5d4de528..74023ad5b0 100644 --- a/pubsub/producer.go +++ b/pubsub/producer.go @@ -96,23 +96,24 @@ func NewProducer[Request any, Response any](client redis.UniversalClient, stream }, nil } +func getUintParts(msgId string) ([2]uint64, error) { + idParts := strings.Split(msgId, "-") + if len(idParts) != 2 { + return [2]uint64{}, fmt.Errorf("invalid i.d: %v", msgId) + } + idTimeStamp, err := strconv.ParseUint(idParts[0], 10, 64) + if err != nil { + return [2]uint64{}, fmt.Errorf("invalid i.d: %v err: %w", msgId, err) + } + idSerial, err := strconv.ParseUint(idParts[1], 10, 64) + if err != nil { + return [2]uint64{}, fmt.Errorf("invalid i.d serial: %v err: %w", msgId, err) + } + return [2]uint64{idTimeStamp, idSerial}, nil +} + // cmpMsgId compares two msgid's and returns (0) if equal, (-1) if msgId1 < msgId2, (1) if msgId1 > msgId2, (-2) if not comparable (or error) func cmpMsgId(msgId1, msgId2 string) int { - getUintParts := func(msgId string) ([2]uint64, error) { - idParts := strings.Split(msgId, "-") - if len(idParts) != 2 { - return [2]uint64{}, fmt.Errorf("invalid i.d: %v", msgId) - } - idTimeStamp, err := strconv.ParseUint(idParts[0], 10, 64) - if err != nil { - return [2]uint64{}, fmt.Errorf("invalid i.d: %v err: %w", msgId, err) - } - idSerial, err := strconv.ParseUint(idParts[1], 10, 64) - if err != nil { - return [2]uint64{}, fmt.Errorf("invalid i.d serial: %v err: %w", msgId, err) - } - return [2]uint64{idTimeStamp, idSerial}, nil - } id1, err := getUintParts(msgId1) if err != nil { log.Trace("error comparing msgIds", "msgId1", msgId1, "msgId2", msgId2) diff --git a/validator/validation_entry.go b/validator/validation_entry.go index be98698208..05bbe50729 100644 --- a/validator/validation_entry.go +++ b/validator/validation_entry.go @@ -7,6 +7,7 @@ import ( "github.com/cespare/xxhash/v2" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/log" "github.com/offchainlabs/nitro/arbutil" ) @@ -32,6 +33,10 @@ type ValidationInput struct { // SetSelfHash should be only called once. In the context of redis streams- by the producer, before submitting a request func (v *ValidationInput) SetSelfHash() { + if v.SelfHash != "" { + log.Error("SetSelfHash called more then once") + return // exiting early as hash has already been set + } jsonData, err := json.Marshal(v) if err != nil { return From bd234b8f67a799562dfea6c6aea16ddda3f07968 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Tue, 27 Aug 2024 20:51:54 -0500 Subject: [PATCH 060/489] Fix spurious refund error log to just be debug --- arbos/tx_processor.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/arbos/tx_processor.go b/arbos/tx_processor.go index b08c7c5d30..ec37a61e04 100644 --- a/arbos/tx_processor.go +++ b/arbos/tx_processor.go @@ -532,6 +532,16 @@ func (p *TxProcessor) EndTxHook(gasLeft uint64, success bool) { refund := func(refundFrom common.Address, amount *big.Int) { const errLog = "fee address doesn't have enough funds to give user refund" + logMissingRefund := func(err error) { + logLevel := log.Error + isContract := p.evm.StateDB.GetCodeSize(refundFrom) > 0 + if isContract { + // It's expected that the balance might not still be in this address if it's a contract. + logLevel = log.Debug + } + logLevel(errLog, "err", err, "feeAddress", refundFrom) + } + // Refund funds to the fee refund address without overdrafting the L1 deposit. toRefundAddr := takeFunds(maxRefund, amount) err = util.TransferBalance(&refundFrom, &inner.RefundTo, toRefundAddr, p.evm, scenario, "refund") @@ -539,13 +549,13 @@ func (p *TxProcessor) EndTxHook(gasLeft uint64, success bool) { // Normally the network fee address should be holding any collected fees. // However, in theory, they could've been transferred out during the redeem attempt. // If the network fee address doesn't have the necessary balance, log an error and don't give a refund. - log.Error(errLog, "err", err, "feeAddress", refundFrom) + logMissingRefund(err) } // Any extra refund can't be given to the fee refund address if it didn't come from the L1 deposit. // Instead, give the refund to the retryable from address. err = util.TransferBalance(&refundFrom, &inner.From, arbmath.BigSub(amount, toRefundAddr), p.evm, scenario, "refund") if err != nil { - log.Error(errLog, "err", err, "feeAddress", refundFrom) + logMissingRefund(err) } } From 7b4834facf7e1c5834ea630551b9908a15183321 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Wed, 28 Aug 2024 13:24:35 +0200 Subject: [PATCH 061/489] pass WasmTargets to RecordProgram call --- arbos/programs/native.go | 4 ++-- go-ethereum | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arbos/programs/native.go b/arbos/programs/native.go index 98abf9443c..08431b1209 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -228,8 +228,8 @@ func callProgram( panic("missing asm") } - if db, ok := db.(*state.StateDB); ok { - db.RecordProgram(moduleHash) + if stateDb, ok := db.(*state.StateDB); ok { + stateDb.RecordProgram(db.Database().WasmTargets(), moduleHash) } evmApi := newApi(interpreter, tracingInfo, scope, memoryModel) diff --git a/go-ethereum b/go-ethereum index 4ca4b4d734..81114dde8a 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 4ca4b4d734021df09cb42a796ca8f263d47383f9 +Subproject commit 81114dde8a26bae90c188605c4a36d5919a4a265 From 99b34b301af91888b32b3395b58aad7eb0ad46b9 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Wed, 28 Aug 2024 13:39:09 +0200 Subject: [PATCH 062/489] init test: call Validate on StylusTargetConfig --- cmd/nitro/init_test.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cmd/nitro/init_test.go b/cmd/nitro/init_test.go index af2242af78..eadc6234c0 100644 --- a/cmd/nitro/init_test.go +++ b/cmd/nitro/init_test.go @@ -350,6 +350,12 @@ func TestEmptyDatabaseDir(t *testing.T) { } } +func defaultStylusTargetConfigForTest(t *testing.T) *gethexec.StylusTargetConfig { + targetConfig := gethexec.DefaultStylusTargetConfig + Require(t, targetConfig.Validate()) + return &targetConfig +} + func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { t.Parallel() @@ -377,7 +383,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), - &gethexec.DefaultStylusTargetConfig, + defaultStylusTargetConfigForTest(t), &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, @@ -394,7 +400,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), - &gethexec.DefaultStylusTargetConfig, + defaultStylusTargetConfigForTest(t), &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, @@ -412,7 +418,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), - &gethexec.DefaultStylusTargetConfig, + defaultStylusTargetConfigForTest(t), &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, @@ -548,7 +554,7 @@ func TestOpenInitializeChainDbEmptyInit(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), - &gethexec.DefaultStylusTargetConfig, + defaultStylusTargetConfigForTest(t), &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, From bf735f466ce13e25006ce2f87f6362692df51659 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Wed, 28 Aug 2024 14:15:46 +0200 Subject: [PATCH 063/489] don't call programs.SetTarget for TargetWavm --- execution/gethexec/executionengine.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index 192408794b..56b52ed986 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -157,6 +157,9 @@ func populateStylusTargetCache(targetConfig *StylusTargetConfig) error { for _, target := range targets { var effectiveStylusTarget string switch target { + case rawdb.TargetWavm: + // skip wavm target + continue case rawdb.TargetArm64: effectiveStylusTarget = targetConfig.Arm64 case rawdb.TargetAmd64: From 29327a8be174e5c582caef4fbe8d4fd55839688e Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Wed, 28 Aug 2024 14:16:57 +0200 Subject: [PATCH 064/489] refactor StylusTargetConfig.Validate --- execution/gethexec/node.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index bc29e9b7f2..01464123fd 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -44,22 +44,24 @@ func (c *StylusTargetConfig) WasmTargets() []ethdb.WasmTarget { func (c *StylusTargetConfig) Validate() error { localTarget := rawdb.LocalTarget() - targets := make([]ethdb.WasmTarget, 0, len(c.ExtraArchs)) - targetsSet := make(map[ethdb.WasmTarget]struct{}, len(c.ExtraArchs)+1) + targets := make([]ethdb.WasmTarget, 0, len(c.ExtraArchs)+1) + targetsSet := make(map[ethdb.WasmTarget]bool, len(c.ExtraArchs)) for _, arch := range c.ExtraArchs { target := ethdb.WasmTarget(arch) + if targetsSet[target] { + // skip duplicate + continue + } if !rawdb.IsSupportedWasmTarget(target) { return fmt.Errorf("unsupported architecture: %v, possible values: %s, %s, %s, %s", arch, rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost) } - if _, duplicate := targetsSet[target]; !duplicate { - targets = append(targets, target) - targetsSet[target] = struct{}{} - } + targets = append(targets, target) + targetsSet[target] = true } - if _, has := targetsSet[rawdb.TargetWavm]; !has { + if !targetsSet[rawdb.TargetWavm] { return fmt.Errorf("%s target not found in archs list, archs: %v", rawdb.TargetWavm, c.ExtraArchs) } - if _, has := targetsSet[localTarget]; !has { + if !targetsSet[localTarget] { targets = append(targets, localTarget) } c.wasmTargets = targets From aba94150dbb0a1b4c6533762284c83948a299e89 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Wed, 28 Aug 2024 14:21:21 +0200 Subject: [PATCH 065/489] error for unsupported stylus target in populateStylusTargetCache --- execution/gethexec/executionengine.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index 56b52ed986..8594d5867d 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -166,11 +166,13 @@ func populateStylusTargetCache(targetConfig *StylusTargetConfig) error { effectiveStylusTarget = targetConfig.Amd64 case rawdb.TargetHost: effectiveStylusTarget = targetConfig.Host + default: + return fmt.Errorf("unsupported stylus target: %v", target) } isNative := target == localTarget err := programs.SetTarget(target, effectiveStylusTarget, isNative) if err != nil { - return fmt.Errorf("Failed to set stylus target: %w", err) + return fmt.Errorf("failed to set stylus target: %w", err) } nativeSet = nativeSet || isNative } From a5cccc49fd53fccf68682f62cdb7fc1e93b0b97f Mon Sep 17 00:00:00 2001 From: Maciej Kulawik <10907694+magicxyyz@users.noreply.github.com> Date: Wed, 28 Aug 2024 15:52:46 +0200 Subject: [PATCH 066/489] fix typo in system_tests/recreatestate_rpc_test.go Co-authored-by: Diego Ximenes Mendes --- system_tests/recreatestate_rpc_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system_tests/recreatestate_rpc_test.go b/system_tests/recreatestate_rpc_test.go index 1a2e00d7f4..b31e02fa89 100644 --- a/system_tests/recreatestate_rpc_test.go +++ b/system_tests/recreatestate_rpc_test.go @@ -524,7 +524,7 @@ func testGettingState(t *testing.T, execConfig *gethexec.Config) { blockCountRequiredToFlushDirties := builder.execConfig.Caching.BlockCount makeSomeTransfers(t, ctx, builder, blockCountRequiredToFlushDirties) - // force garbage callection to check if it won't break anything + // force garbage collection to check if it won't break anything runtime.GC() exists = state.Exist(addr) From 1e1f49c7bf5342f1d0d15948ee1507a23b62e8ad Mon Sep 17 00:00:00 2001 From: Maciej Kulawik <10907694+magicxyyz@users.noreply.github.com> Date: Wed, 28 Aug 2024 15:56:04 +0200 Subject: [PATCH 067/489] fix comment in system_tests/recreatestate_rpc_test.go Co-authored-by: Diego Ximenes Mendes --- system_tests/recreatestate_rpc_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system_tests/recreatestate_rpc_test.go b/system_tests/recreatestate_rpc_test.go index b31e02fa89..6c5d3153ad 100644 --- a/system_tests/recreatestate_rpc_test.go +++ b/system_tests/recreatestate_rpc_test.go @@ -616,7 +616,7 @@ func TestStateAndHeaderForRecentBlock(t *testing.T) { // 1. Before state trie node is referenced in core.BlockChain.writeBlockWithState, block body is written to database with key prefix `b` followed by block number and then block hash (see: rawdb.blockBodyKey) // 2. Each thread tries to read the block body entry to: a. extract recent block hash b. congest resource usage to slow down execution of core.BlockChain.writeBlockWithState // 3. After extracting the hash from block body entry key, StateAndHeaderByNumberOfHash is called for the hash. It is expected that it will: - // a. either fail with "ahead of current block" if we made it before rawdb.WriteCanonicalHash is called in core.BlockChain.writeHeadBlock, what is called after writeBlockWithState finishes, + // a. either fail with "ahead of current block" if we made it before rawdb.WriteCanonicalHash is called in core.BlockChain.writeHeadBlock, which is called after writeBlockWithState finishes, // b. or it will succeed if the canonical hash was written for the block meaning that writeBlockWithState was fully executed (i.a. state root trie node correctly referenced) - then the recentBlock is advanced go func() { defer wgCallers.Done() From e80c834463b001fbe1a6a6ed8f7bc0a268982738 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Wed, 28 Aug 2024 19:04:29 +0200 Subject: [PATCH 068/489] use errors.As when checking for MissingNodeError --- system_tests/recreatestate_rpc_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/system_tests/recreatestate_rpc_test.go b/system_tests/recreatestate_rpc_test.go index d28d923bbf..0a5ed38502 100644 --- a/system_tests/recreatestate_rpc_test.go +++ b/system_tests/recreatestate_rpc_test.go @@ -536,6 +536,10 @@ func testGettingState(t *testing.T, execConfig *gethexec.Config) { if err == nil { Fatal(t, "StateAndHeaderByNumber didn't failed as expected") } + expectedErr := &trie.MissingNodeError{} + if !errors.As(err, &expectedErr) { + Fatal(t, "StateAndHeaderByNumber failed with unexpected error:", err) + } } func TestGettingState(t *testing.T) { From 9a326e281871d3c29dd5a796972298c87e917235 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Wed, 28 Aug 2024 22:04:10 -0500 Subject: [PATCH 069/489] Disable preimage validation in JIT --- arbitrator/jit/src/wavmio.rs | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/arbitrator/jit/src/wavmio.rs b/arbitrator/jit/src/wavmio.rs index 062d18d8e9..0ca666d3b2 100644 --- a/arbitrator/jit/src/wavmio.rs +++ b/arbitrator/jit/src/wavmio.rs @@ -8,8 +8,6 @@ use crate::{ }; use arbutil::{Color, PreimageType}; use caller_env::{GuestPtr, MemAccess}; -use sha2::Sha256; -use sha3::{Digest, Keccak256}; use std::{ io, io::{BufReader, BufWriter, ErrorKind}, @@ -170,19 +168,25 @@ pub fn resolve_preimage_impl( error!("Missing requested preimage for hash {hash_hex} in {name}") }; - // Check if preimage rehashes to the provided hash. Exclude blob preimages - let calculated_hash: [u8; 32] = match preimage_type { - PreimageType::Keccak256 => Keccak256::digest(preimage).into(), - PreimageType::Sha2_256 => Sha256::digest(preimage).into(), - PreimageType::EthVersionedHash => *hash, - }; - if calculated_hash != *hash { - error!( - "Calculated hash {} of preimage {} does not match provided hash {}", - hex::encode(calculated_hash), - hex::encode(preimage), - hex::encode(*hash) - ); + #[cfg(debug_assertions)] + { + use sha2::Sha256; + use sha3::{Digest, Keccak256}; + + // Check if preimage rehashes to the provided hash. Exclude blob preimages + let calculated_hash: [u8; 32] = match preimage_type { + PreimageType::Keccak256 => Keccak256::digest(preimage).into(), + PreimageType::Sha2_256 => Sha256::digest(preimage).into(), + PreimageType::EthVersionedHash => *hash, + }; + if calculated_hash != *hash { + error!( + "Calculated hash {} of preimage {} does not match provided hash {}", + hex::encode(calculated_hash), + hex::encode(preimage), + hex::encode(*hash) + ); + } } if offset % 32 != 0 { From 25af47fcc1ffb06285a87d61a8cfe0cc0c5f92a3 Mon Sep 17 00:00:00 2001 From: Pepper Lebeck-Jobe Date: Thu, 29 Aug 2024 13:43:15 +0200 Subject: [PATCH 070/489] Ensure the inbox tracker has the desired message. If the inbox is lagging behind the message, the node won't be able to create a validation entry for it. --- system_tests/common_test.go | 10 ++++++++++ system_tests/program_test.go | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 53d17c762c..7f8864ff94 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -1467,6 +1467,16 @@ func recordBlock(t *testing.T, block uint64, builder *NodeBuilder) { ctx := builder.ctx wasmModuleRoot := currentRootModule(t) inboxPos := arbutil.MessageIndex(block) + for { + time.Sleep(250 * time.Millisecond) + batches, err := builder.L2.ConsensusNode.InboxTracker.GetBatchCount() + Require(t, err) + haveMessages, err := builder.L2.ConsensusNode.InboxTracker.GetBatchMessageCount(batches - 1) + Require(t, err) + if haveMessages >= inboxPos { + break + } + } inputJson, err := builder.L2.ConsensusNode.StatelessBlockValidator.ValidationInputsAt(ctx, inboxPos, wasmModuleRoot) if err != nil { Fatal(t, "failed to get validation inputs", block, err) diff --git a/system_tests/program_test.go b/system_tests/program_test.go index 023d88a380..c5de186978 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -396,7 +396,8 @@ func storageTest(t *testing.T, jit bool) { validateBlocks(t, 2, jit, builder) // Without this sleep, this test fails in the "-race" CI builds complaining that // the block has not been seen in L1 yet. - time.Sleep(10 * time.Second) + // time.Sleep(10 * time.Second) + // Captures a block_input_.json file for the block that included the // storage write transaction. recordBlock(t, receipt.BlockNumber.Uint64(), builder) From e0d70e6aeddb02606689812dff8a92ae16c009f8 Mon Sep 17 00:00:00 2001 From: Pepper Lebeck-Jobe Date: Thu, 29 Aug 2024 13:45:07 +0200 Subject: [PATCH 071/489] Remove commented out code. Meant to do this in the previous commit. --- system_tests/program_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/system_tests/program_test.go b/system_tests/program_test.go index c5de186978..9b3dd56ffb 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -394,9 +394,6 @@ func storageTest(t *testing.T, jit bool) { assertStorageAt(t, ctx, l2client, programAddress, key, value) validateBlocks(t, 2, jit, builder) - // Without this sleep, this test fails in the "-race" CI builds complaining that - // the block has not been seen in L1 yet. - // time.Sleep(10 * time.Second) // Captures a block_input_.json file for the block that included the // storage write transaction. From bc25dde884f0c473c72a41cff01ba2b248ef3415 Mon Sep 17 00:00:00 2001 From: Pepper Lebeck-Jobe Date: Thu, 29 Aug 2024 15:35:05 +0200 Subject: [PATCH 072/489] Separate the writer for validaiton inputs from the json. This change attempts to make the location for writing the json file more flexibly configurable by the clients without making it difficult to put the data in a resonable spot by default. --- staker/block_validator.go | 11 ++- system_tests/common_test.go | 6 +- validator/inputs/writer.go | 129 ++++++++++++++++++++++++++++++++ validator/inputs/writer_test.go | 87 +++++++++++++++++++++ validator/server_api/json.go | 33 +------- 5 files changed, 234 insertions(+), 32 deletions(-) create mode 100644 validator/inputs/writer.go create mode 100644 validator/inputs/writer_test.go diff --git a/staker/block_validator.go b/staker/block_validator.go index a7e475b4f0..a884ac81f7 100644 --- a/staker/block_validator.go +++ b/staker/block_validator.go @@ -29,6 +29,7 @@ import ( "github.com/offchainlabs/nitro/util/stopwaiter" "github.com/offchainlabs/nitro/validator" "github.com/offchainlabs/nitro/validator/client/redis" + "github.com/offchainlabs/nitro/validator/inputs" "github.com/offchainlabs/nitro/validator/server_api" "github.com/spf13/pflag" ) @@ -95,6 +96,9 @@ type BlockValidator struct { // for testing only testingProgressMadeChan chan struct{} + // For troubleshooting failed validations + validationInputsWriter *inputs.Writer + fatalErr chan<- error MemoryFreeLimitChecker resourcemanager.LimitChecker @@ -275,6 +279,11 @@ func NewBlockValidator( config: config, fatalErr: fatalErr, } + valInputsWriter, err := inputs.NewWriter() + if err != nil { + return nil, err + } + ret.validationInputsWriter = valInputsWriter.SetSlug("BlockValidator") if !config().Dangerous.ResetBlockValidation { validated, err := ret.ReadLastValidatedInfo() if err != nil { @@ -512,7 +521,7 @@ func (v *BlockValidator) writeToFile(validationEntry *validationEntry) error { return err } inputJson := server_api.ValidationInputToJson(input) - if err := inputJson.WriteToFile("BlockValidator"); err != nil { + if err := v.validationInputsWriter.Write(inputJson); err != nil { return err } return nil diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 7f8864ff94..17834b429d 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -36,6 +36,7 @@ import ( "github.com/offchainlabs/nitro/util/headerreader" "github.com/offchainlabs/nitro/util/redisutil" "github.com/offchainlabs/nitro/util/signature" + "github.com/offchainlabs/nitro/validator/inputs" "github.com/offchainlabs/nitro/validator/server_api" "github.com/offchainlabs/nitro/validator/server_common" "github.com/offchainlabs/nitro/validator/valnode" @@ -1477,11 +1478,14 @@ func recordBlock(t *testing.T, block uint64, builder *NodeBuilder) { break } } + validationInputsWriter, err := inputs.NewWriter() + Require(t, err) + validationInputsWriter.SetSlug(t.Name()) inputJson, err := builder.L2.ConsensusNode.StatelessBlockValidator.ValidationInputsAt(ctx, inboxPos, wasmModuleRoot) if err != nil { Fatal(t, "failed to get validation inputs", block, err) } - if err := inputJson.WriteToFile(t.Name()); err != nil { + if err := validationInputsWriter.Write(&inputJson); err != nil { Fatal(t, "failed to write validation inputs", block, err) } } diff --git a/validator/inputs/writer.go b/validator/inputs/writer.go new file mode 100644 index 0000000000..b8160a4fa5 --- /dev/null +++ b/validator/inputs/writer.go @@ -0,0 +1,129 @@ +package inputs + +import ( + "fmt" + "os" + "time" + + "github.com/offchainlabs/nitro/validator/server_api" +) + +// Writer is a configurable writer of InputJSON files. +// +// The default Writer will write to a path like: +// +// $HOME/.arbuitrum/validation-inputs//block_inputs_.json +// +// The path can be nested under a slug directory so callers can provide a +// recognizable name to differentiate various contexts in which the InputJSON +// is being written. If the Writer is configured by calling SetSlug, then the +// path will be like: +// +// $HOME/.arbuitrum/validation-inputs///block_inputs_.json +// +// The inclusion of a timestamp directory is on by default to avoid conflicts which +// would result in files being overwritten. However, the Writer can be configured +// to not use a timestamp directory. If the Writer is configured by calling +// SetUseTimestampDir(false), then the path will be like: +// +// $HOME/.arbuitrum/validation-inputs//block_inputs_.json +// +// Finally, to give complete control to the clients, the base directory can be +// set directly with SetBaseDir. In which case, the path will be like: +// +// /block_inputs_.json +// or +// //block_inputs_.json +// or +// ///block_inputs_.json +type Writer struct { + clock Clock + baseDir string + slug string + useTimestampDir bool +} + +// Clock is an interface for getting the current time. +type Clock interface { + Now() time.Time +} + +type realClock struct{} + +func (realClock) Now() time.Time { + return time.Now() +} + +// NewWriter creates a new Writer with default settings. +func NewWriter() (*Writer, error) { + homeDir, err := os.UserHomeDir() + if err != nil { + return nil, err + } + baseDir := fmt.Sprintf("%s/.arbitrum/validation-inputs", homeDir) + return &Writer{ + clock: realClock{}, + baseDir: baseDir, + slug: "", + useTimestampDir: true}, nil +} + +// SetClockForTesting sets the clock used by the Writer. +// +// This is only intended for testing. +func (w *Writer) SetClockForTesting(clock Clock) *Writer { + w.clock = clock + return w +} + +// SetSlug configures the Writer to use the given slug as a directory name. +func (w *Writer) SetSlug(slug string) *Writer { + w.slug = slug + return w +} + +// ClearSlug clears the slug configuration. +// +// This is equivalent to calling SetSlug("") but is more readable. +func (w *Writer) ClearSlug() *Writer { + w.slug = "" + return w +} + +// SetBaseDir configures the Writer to use the given base directory. +func (w *Writer) SetBaseDir(baseDir string) *Writer { + w.baseDir = baseDir + return w +} + +// SetUseTimestampDir controls the addition of a timestamp directory. +func (w *Writer) SetUseTimestampDir(useTimestampDir bool) *Writer { + w.useTimestampDir = useTimestampDir + return w +} + +// Write writes the given InputJSON to a file in JSON format. +func (w *Writer) Write(inputs *server_api.InputJSON) error { + dir := w.baseDir + if w.slug != "" { + dir = fmt.Sprintf("%s/%s", dir, w.slug) + } + if w.useTimestampDir { + t := w.clock.Now() + tStr := t.Format("20060102_150405") + dir = fmt.Sprintf("%s/%s", dir, tStr) + } + if err := os.MkdirAll(dir, 0700); err != nil { + return err + } + contents, err := inputs.Marshal() + if err != nil { + return err + } + if err = os.WriteFile( + fmt.Sprintf("%s/block_inputs_%d.json", dir, inputs.Id), + contents, 0600); err != nil { + return err + } + return nil +} diff --git a/validator/inputs/writer_test.go b/validator/inputs/writer_test.go new file mode 100644 index 0000000000..5e80b9aa3a --- /dev/null +++ b/validator/inputs/writer_test.go @@ -0,0 +1,87 @@ +package inputs + +import ( + "os" + "testing" + "time" + + "github.com/offchainlabs/nitro/validator/server_api" +) + +func TestDefaultBaseDir(t *testing.T) { + // Simply testing that the default baseDir is set relative to the user's home directory. + // This way, the other tests can all override the baseDir to a temporary directory. + w, err := NewWriter() + if err != nil { + t.Fatal(err) + } + homeDir, err := os.UserHomeDir() + if err != nil { + t.Fatal(err) + } + if w.baseDir != homeDir+"/.arbitrum/validation-inputs" { + t.Errorf("unexpected baseDir: %v", w.baseDir) + } +} + +type fakeClock struct { + now time.Time +} + +func (c fakeClock) Now() time.Time { + return c.now +} + +func TestWriting(t *testing.T) { + w, err := NewWriter() + if err != nil { + t.Fatal(err) + } + w.SetClockForTesting(fakeClock{now: time.Date(2021, 1, 2, 3, 4, 5, 0, time.UTC)}) + dir := t.TempDir() + w.SetBaseDir(dir) + err = w.Write(&server_api.InputJSON{Id: 24601}) + if err != nil { + t.Fatal(err) + } + // The file should exist. + if _, err := os.Stat(dir + "/20210102_030405/block_inputs_24601.json"); err != nil { + t.Error(err) + } +} + +func TestWritingWithSlug(t *testing.T) { + w, err := NewWriter() + if err != nil { + t.Fatal(err) + } + w.SetClockForTesting(fakeClock{now: time.Date(2021, 1, 2, 3, 4, 5, 0, time.UTC)}) + dir := t.TempDir() + w.SetBaseDir(dir).SetSlug("foo") + err = w.Write(&server_api.InputJSON{Id: 24601}) + if err != nil { + t.Fatal(err) + } + // The file should exist. + if _, err := os.Stat(dir + "/foo/20210102_030405/block_inputs_24601.json"); err != nil { + t.Error(err) + } +} + +func TestWritingWithoutTimestampDir(t *testing.T) { + w, err := NewWriter() + if err != nil { + t.Fatal(err) + } + w.SetClockForTesting(fakeClock{now: time.Date(2021, 1, 2, 3, 4, 5, 0, time.UTC)}) + dir := t.TempDir() + w.SetBaseDir(dir).SetUseTimestampDir(false) + err = w.Write(&server_api.InputJSON{Id: 24601}) + if err != nil { + t.Fatal(err) + } + // The file should exist. + if _, err := os.Stat(dir + "/block_inputs_24601.json"); err != nil { + t.Error(err) + } +} diff --git a/validator/server_api/json.go b/validator/server_api/json.go index 069ab014d9..bdb19bc0ba 100644 --- a/validator/server_api/json.go +++ b/validator/server_api/json.go @@ -8,8 +8,6 @@ import ( "encoding/json" "errors" "fmt" - "os" - "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" @@ -69,34 +67,9 @@ type InputJSON struct { DebugChain bool } -// WriteToFile writes the InputJSON to a file in JSON format. -// -// The path to the file is determined in part by the slug parameter so -// callers can provide a recognizable name to differentiate various -// contexts in which the InputJSON is being written. -// -// The file is created at a path -// -// $HOME/.arbuitrum/validation-inputs///block_inputs_.json -func (i *InputJSON) WriteToFile(slug string) error { - homeDir, err := os.UserHomeDir() - if err != nil { - return err - } - t := time.Now() - tStr := t.Format("20060102_150405") - dir := fmt.Sprintf("%s/.arbitrum/validation-inputs/%s/%s", homeDir, slug, tStr) - if err = os.MkdirAll(dir, 0700); err != nil { - return err - } - contents, err := json.MarshalIndent(i, "", " ") - if err != nil { - return err - } - if err = os.WriteFile(fmt.Sprintf("%s/block_inputs_%d.json", dir, i.Id), contents, 0600); err != nil { - return err - } - return nil +// Marshal returns the JSON encoding of the InputJSON. +func (i *InputJSON) Marshal() ([]byte, error) { + return json.MarshalIndent(i, "", " ") } type BatchInfoJson struct { From 7a4fe007e5e3edb198130265ad167dc39d531073 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Thu, 29 Aug 2024 21:21:50 +0200 Subject: [PATCH 073/489] simplify StylusTargetConfig.Validate --- execution/gethexec/node.go | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index 01464123fd..21c2b4bece 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -43,26 +43,21 @@ func (c *StylusTargetConfig) WasmTargets() []ethdb.WasmTarget { } func (c *StylusTargetConfig) Validate() error { - localTarget := rawdb.LocalTarget() - targets := make([]ethdb.WasmTarget, 0, len(c.ExtraArchs)+1) targetsSet := make(map[ethdb.WasmTarget]bool, len(c.ExtraArchs)) for _, arch := range c.ExtraArchs { target := ethdb.WasmTarget(arch) - if targetsSet[target] { - // skip duplicate - continue - } if !rawdb.IsSupportedWasmTarget(target) { return fmt.Errorf("unsupported architecture: %v, possible values: %s, %s, %s, %s", arch, rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost) } - targets = append(targets, target) targetsSet[target] = true } if !targetsSet[rawdb.TargetWavm] { return fmt.Errorf("%s target not found in archs list, archs: %v", rawdb.TargetWavm, c.ExtraArchs) } - if !targetsSet[localTarget] { - targets = append(targets, localTarget) + targetsSet[rawdb.LocalTarget()] = true + targets := make([]ethdb.WasmTarget, 0, len(c.ExtraArchs)+1) + for target := range targetsSet { + targets = append(targets, target) } c.wasmTargets = targets return nil From 1ae0aff868abd8d4347433d0eef6846dfa0a2a99 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Thu, 29 Aug 2024 23:06:37 -0500 Subject: [PATCH 074/489] Check error type --- arbos/tx_processor.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arbos/tx_processor.go b/arbos/tx_processor.go index ec37a61e04..d6c35339f6 100644 --- a/arbos/tx_processor.go +++ b/arbos/tx_processor.go @@ -533,6 +533,10 @@ func (p *TxProcessor) EndTxHook(gasLeft uint64, success bool) { const errLog = "fee address doesn't have enough funds to give user refund" logMissingRefund := func(err error) { + if !errors.Is(err, vm.ErrInsufficientBalance) { + log.Error("unexpected error refunding balance", "err", err, "feeAddress", refundFrom) + return + } logLevel := log.Error isContract := p.evm.StateDB.GetCodeSize(refundFrom) > 0 if isContract { From e31fcecbd6538b718f1596348df4f8852fb284b2 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Fri, 30 Aug 2024 17:33:07 +0530 Subject: [PATCH 075/489] Changes based on PR comments --- arbnode/seq_coordinator.go | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/arbnode/seq_coordinator.go b/arbnode/seq_coordinator.go index 431ea43512..521b17340e 100644 --- a/arbnode/seq_coordinator.go +++ b/arbnode/seq_coordinator.go @@ -752,17 +752,26 @@ func (c *SeqCoordinator) launchHealthcheckServer(ctx context.Context) { func (c *SeqCoordinator) Start(ctxIn context.Context) { c.StopWaiter.Start(ctxIn, c) - c.CallIteratively(c.chooseRedisAndUpdate) + var newRedisCoordinator *redisutil.RedisCoordinator + if c.config.NewRedisUrl != "" { + var err error + newRedisCoordinator, err = redisutil.NewRedisCoordinator(c.config.NewRedisUrl) + if err != nil { + log.Warn("failed to create new redis coordinator", "err", + err, "newRedisUrl", c.config.NewRedisUrl) + } + } + c.CallIteratively(func(ctx context.Context) time.Duration { return c.chooseRedisAndUpdate(ctx, newRedisCoordinator) }) if c.config.ChosenHealthcheckAddr != "" { c.StopWaiter.LaunchThread(c.launchHealthcheckServer) } } -func (c *SeqCoordinator) chooseRedisAndUpdate(ctx context.Context) time.Duration { +func (c *SeqCoordinator) chooseRedisAndUpdate(ctx context.Context, newRedisCoordinator *redisutil.RedisCoordinator) time.Duration { // If we have a new redis coordinator, and we haven't switched to it yet, try to switch. if c.config.NewRedisUrl != "" && c.prevRedisCoordinator == nil { // If we fail to try to switch, we'll retry soon. - if err := c.trySwitchingRedis(ctx); err != nil { + if err := c.trySwitchingRedis(ctx, newRedisCoordinator); err != nil { log.Warn("error while trying to switch redis coordinator", "err", err) return c.retryAfterRedisError() } @@ -770,7 +779,11 @@ func (c *SeqCoordinator) chooseRedisAndUpdate(ctx context.Context) time.Duration return c.update(ctx) } -func (c *SeqCoordinator) trySwitchingRedis(ctx context.Context) error { +func (c *SeqCoordinator) trySwitchingRedis(ctx context.Context, newRedisCoordinator *redisutil.RedisCoordinator) error { + err := c.wantsLockoutUpdate(ctx, newRedisCoordinator.Client) + if err != nil { + return err + } current, err := c.Client.Get(ctx, redisutil.CHOSENSEQ_KEY).Result() var wasEmpty bool if errors.Is(err, redis.Nil) { @@ -783,20 +796,10 @@ func (c *SeqCoordinator) trySwitchingRedis(ctx context.Context) error { } // If the chosen key is set to switch, we need to switch to the new redis coordinator. if !wasEmpty && (current == redisutil.SWITCHED_REDIS) { - newRedisCoordinator, err := redisutil.NewRedisCoordinator(c.config.NewRedisUrl) - if err != nil { - log.Warn("failed to create new redis coordinator", "err", - err, "newRedisUrl", c.config.NewRedisUrl) - return err - } err = c.wantsLockoutUpdate(ctx, c.Client) if err != nil { return err } - err = c.wantsLockoutUpdate(ctx, newRedisCoordinator.Client) - if err != nil { - return err - } c.prevRedisCoordinator = &c.RedisCoordinator c.RedisCoordinator = *newRedisCoordinator } From 6dd8934f94ab1a2a5a367838ba3a5c3c86ae9a07 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Fri, 30 Aug 2024 17:50:24 +0530 Subject: [PATCH 076/489] fix build --- arbnode/node.go | 2 +- arbnode/seq_coordinator.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/arbnode/node.go b/arbnode/node.go index 569f0e1b08..93b58e800f 100644 --- a/arbnode/node.go +++ b/arbnode/node.go @@ -446,7 +446,7 @@ func createNodeImpl( } if config.SeqCoordinator.Enable { - coordinator, err = NewSeqCoordinator(dataSigner, bpVerifier, txStreamer, exec, config.SeqCoordinator) + coordinator, err = NewSeqCoordinator(dataSigner, bpVerifier, txStreamer, exec, syncMonitor, config.SeqCoordinator) if err != nil { return nil, err } diff --git a/arbnode/seq_coordinator.go b/arbnode/seq_coordinator.go index 2c02541683..7d058031f5 100644 --- a/arbnode/seq_coordinator.go +++ b/arbnode/seq_coordinator.go @@ -147,6 +147,7 @@ func NewSeqCoordinator( bpvalidator *contracts.AddressVerifier, streamer *TransactionStreamer, sequencer execution.ExecutionSequencer, + sync *SyncMonitor, config SeqCoordinatorConfig, ) (*SeqCoordinator, error) { redisCoordinator, err := redisutil.NewRedisCoordinator(config.RedisUrl) From 79c3aa2a7f9077225ca193ec9d8f41ca3d3f4f93 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Fri, 30 Aug 2024 17:52:40 +0530 Subject: [PATCH 077/489] simplify --- arbnode/seq_coordinator.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/arbnode/seq_coordinator.go b/arbnode/seq_coordinator.go index 7d058031f5..6071925780 100644 --- a/arbnode/seq_coordinator.go +++ b/arbnode/seq_coordinator.go @@ -631,17 +631,14 @@ func (c *SeqCoordinator) update(ctx context.Context) time.Duration { log.Warn("cannot get remote message count", "err", err) return c.retryAfterRedisError() } - readUntil := remoteMsgCount + readUntil := min(localMsgCount+c.config.MsgPerPoll, remoteMsgCount) client := c.Client // If we have a previous redis coordinator, // we can read from it until the local message count catches up to the prev coordinator's message count if c.prevRedisMessageCount > localMsgCount { - readUntil = c.prevRedisMessageCount + readUntil = min(readUntil, c.prevRedisMessageCount) client = c.prevRedisCoordinator.Client } - if readUntil > localMsgCount+c.config.MsgPerPoll { - readUntil = localMsgCount + c.config.MsgPerPoll - } var messages []arbostypes.MessageWithMetadata msgToRead := localMsgCount var msgReadErr error From 0d08e481f039c778ff0f9a7c60f38167eadb29f2 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 30 Aug 2024 20:46:02 -0300 Subject: [PATCH 078/489] Also uses DataPoster when only creating validator wallet contract and exiting --- arbnode/node.go | 23 ++++++++++ cmd/nitro/nitro.go | 15 ++++++- staker/validatorwallet/contract.go | 72 +++++++++++++++--------------- system_tests/fast_confirm_test.go | 54 +++++++++++----------- system_tests/staker_test.go | 25 ++++++++--- 5 files changed, 119 insertions(+), 70 deletions(-) diff --git a/arbnode/node.go b/arbnode/node.go index 93b58e800f..e13f92f30f 100644 --- a/arbnode/node.go +++ b/arbnode/node.go @@ -339,6 +339,29 @@ func checkArbDbSchemaVersion(arbDb ethdb.Database) error { return nil } +func DataposterOnlyUsedToCreateValidatorWalletContract( + ctx context.Context, + l1Reader *headerreader.HeaderReader, + transactOpts *bind.TransactOpts, + cfg *dataposter.DataPosterConfig, + parentChainID *big.Int, +) (*dataposter.DataPoster, error) { + return dataposter.NewDataPoster(ctx, + &dataposter.DataPosterOpts{ + HeaderReader: l1Reader, + Auth: transactOpts, + Config: func() *dataposter.DataPosterConfig { + cfg.UseNoOpStorage = true + return cfg + }, + MetadataRetriever: func(ctx context.Context, blockNum *big.Int) ([]byte, error) { + return nil, nil + }, + ParentChainID: parentChainID, + }, + ) +} + func StakerDataposter( ctx context.Context, db ethdb.Database, l1Reader *headerreader.HeaderReader, transactOpts *bind.TransactOpts, cfgFetcher ConfigFetcher, syncMonitor *SyncMonitor, diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index 8c26b95a92..bad48f430c 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -371,8 +371,21 @@ func mainImpl() int { if err != nil { log.Crit("error getting rollup addresses config", "err", err) } + + dataPoster, err := arbnode.DataposterOnlyUsedToCreateValidatorWalletContract( + ctx, + l1Reader, + l1TransactionOptsBatchPoster, + &nodeConfig.Node.Staker.DataPoster, + new(big.Int).SetUint64(nodeConfig.ParentChain.ID), + ) + if err != nil { + log.Crit("error creating data poster to create validator wallet contract", "err", err) + } + getExtraGas := func() uint64 { return nodeConfig.Node.Staker.ExtraGas } + // #nosec G115 - addr, err := validatorwallet.GetValidatorWalletContract(ctx, deployInfo.ValidatorWalletCreator, int64(deployInfo.DeployedAt), l1TransactionOptsValidator, l1Reader, true, nil) + addr, err := validatorwallet.GetValidatorWalletContract(ctx, deployInfo.ValidatorWalletCreator, int64(deployInfo.DeployedAt), l1TransactionOptsValidator, l1Reader, true, dataPoster, getExtraGas) if err != nil { log.Crit("error creating validator wallet contract", "error", err, "address", l1TransactionOptsValidator.From.Hex()) } diff --git a/staker/validatorwallet/contract.go b/staker/validatorwallet/contract.go index 495e796cd2..3db35d8232 100644 --- a/staker/validatorwallet/contract.go +++ b/staker/validatorwallet/contract.go @@ -155,16 +155,19 @@ func (v *Contract) From() common.Address { } // nil value == 0 value -func (v *Contract) getAuth(ctx context.Context, value *big.Int) (*bind.TransactOpts, error) { - newAuth := *v.auth - newAuth.Context = ctx - newAuth.Value = value - nonce, err := v.L1Client().NonceAt(ctx, v.auth.From, nil) +func getAuthWithUpdatedNonce(ctx context.Context, l1Reader *headerreader.HeaderReader, auth bind.TransactOpts, value *big.Int) (*bind.TransactOpts, error) { + auth.Context = ctx + auth.Value = value + nonce, err := l1Reader.Client().NonceAt(ctx, auth.From, nil) if err != nil { return nil, err } - newAuth.Nonce = new(big.Int).SetUint64(nonce) - return &newAuth, nil + auth.Nonce = new(big.Int).SetUint64(nonce) + return &auth, nil +} + +func (v *Contract) getAuth(ctx context.Context, value *big.Int) (*bind.TransactOpts, error) { + return getAuthWithUpdatedNonce(ctx, v.l1Reader, *v.auth, value) } func (v *Contract) executeTransaction(ctx context.Context, tx *types.Transaction, gasRefunder common.Address) (*types.Transaction, error) { @@ -183,9 +186,12 @@ func (v *Contract) executeTransaction(ctx context.Context, tx *types.Transaction return v.dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), *v.Address(), data, gas, auth.Value) } -// Exposed for tests -func (v *Contract) CreateWalletContract( +func createWalletContract( ctx context.Context, + l1Reader *headerreader.HeaderReader, + auth *bind.TransactOpts, + dataPoster *dataposter.DataPoster, + getExtraGas func() uint64, validatorWalletFactoryAddr common.Address, ) (*types.Transaction, error) { var initialExecutorAllowedDests []common.Address @@ -194,24 +200,19 @@ func (v *Contract) CreateWalletContract( return nil, err } - auth, err := v.getAuth(ctx, nil) - if err != nil { - return nil, err - } - gas, err := gasForTxData( ctx, - v.l1Reader, + l1Reader, auth, &validatorWalletFactoryAddr, txData, - v.getExtraGas, + getExtraGas, ) if err != nil { return nil, fmt.Errorf("getting gas for tx data: %w", err) } - return v.dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), validatorWalletFactoryAddr, txData, gas, auth.Value) + return dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), validatorWalletFactoryAddr, txData, gas, auth.Value) } func (v *Contract) populateWallet(ctx context.Context, createIfMissing bool) error { @@ -225,15 +226,10 @@ func (v *Contract) populateWallet(ctx context.Context, createIfMissing bool) err return nil } if v.address.Load() == nil { - auth, err := v.getAuth(ctx, nil) - if err != nil { - return err - } - - // By passing v.CreateWalletContract as a parameter to GetValidatorWalletContract we force to create a validator wallet through the Staker's DataPoster object. + // By passing v.dataPoster as a parameter to GetValidatorWalletContract we force to create a validator wallet through the Staker's DataPoster object. // DataPoster keeps in its internal state information related to the transactions sent through it, which is used to infer the expected nonce in a transaction for example. // If a transaction is sent using the Staker's DataPoster key, but not through the Staker's DataPoster object, DataPoster's internal state will be outdated, which can compromise the expected nonce inference. - addr, err := GetValidatorWalletContract(ctx, v.walletFactoryAddr, v.rollupFromBlock, auth, v.l1Reader, createIfMissing, v.CreateWalletContract) + addr, err := GetValidatorWalletContract(ctx, v.walletFactoryAddr, v.rollupFromBlock, v.auth, v.l1Reader, createIfMissing, v.dataPoster, v.getExtraGas) if err != nil { return err } @@ -439,6 +435,11 @@ func (b *Contract) DataPoster() *dataposter.DataPoster { return b.dataPoster } +// Exported for testing +func (b *Contract) GetExtraGas() func() uint64 { + return b.getExtraGas +} + func GetValidatorWalletContract( ctx context.Context, validatorWalletFactoryAddr common.Address, @@ -446,7 +447,8 @@ func GetValidatorWalletContract( transactAuth *bind.TransactOpts, l1Reader *headerreader.HeaderReader, createIfMissing bool, - createWalletFunc func(context.Context, common.Address) (*types.Transaction, error), + dataPoster *dataposter.DataPoster, + getExtraGas func() uint64, ) (*common.Address, error) { client := l1Reader.Client() @@ -483,18 +485,14 @@ func GetValidatorWalletContract( return nil, nil } - var tx *types.Transaction - if createWalletFunc == nil { - var initialExecutorAllowedDests []common.Address - tx, err = walletCreator.CreateWallet(transactAuth, initialExecutorAllowedDests) - if err != nil { - return nil, err - } - } else { - tx, err = createWalletFunc(ctx, validatorWalletFactoryAddr) - if err != nil { - return nil, err - } + transactAuth, err = getAuthWithUpdatedNonce(ctx, l1Reader, *transactAuth, nil) + if err != nil { + return nil, err + } + + tx, err := createWalletContract(ctx, l1Reader, transactAuth, dataPoster, getExtraGas, validatorWalletFactoryAddr) + if err != nil { + return nil, err } receipt, err := l1Reader.WaitForTxApproval(ctx, tx) diff --git a/system_tests/fast_confirm_test.go b/system_tests/fast_confirm_test.go index ae624be1e9..4598827418 100644 --- a/system_tests/fast_confirm_test.go +++ b/system_tests/fast_confirm_test.go @@ -115,10 +115,10 @@ func TestFastConfirmation(t *testing.T) { Require(t, err) valConfig.Strategy = "MakeNodes" - valWalletAddrPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, valWallet.CreateWalletContract) + valWalletAddrPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, valWallet.DataPoster(), valWallet.GetExtraGas()) Require(t, err) valWalletAddr := *valWalletAddrPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, valWallet.CreateWalletContract) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, valWallet.DataPoster(), valWallet.GetExtraGas()) Require(t, err) if valWalletAddr == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddr.String(), "vs", valWalletAddrCheck.String()) @@ -278,15 +278,6 @@ func TestFastConfirmationWithSafe(t *testing.T) { builder.L1.TransferBalance(t, "Faucet", "ValidatorB", balance, builder.L1Info) l1authB := builder.L1Info.GetDefaultTransactOpts("ValidatorB", ctx) - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, nil) - Require(t, err) - valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, nil) - Require(t, err) - if valWalletAddrA == *valWalletAddrCheck { - Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) - } - rollup, err := rollupgen.NewRollupAdminLogic(l2nodeA.DeployInfo.Rollup, builder.L1.Client) Require(t, err) @@ -295,28 +286,13 @@ func TestFastConfirmationWithSafe(t *testing.T) { rollupABI, err := abi.JSON(strings.NewReader(rollupgen.RollupAdminLogicABI)) Require(t, err, "unable to parse rollup ABI") - safeAddress := deploySafe(t, builder.L1, builder.L1.Client, deployAuth, []common.Address{valWalletAddrA, srv.Address}) - setValidatorCalldata, err := rollupABI.Pack("setValidator", []common.Address{valWalletAddrA, l1authB.From, srv.Address, safeAddress}, []bool{true, true, true, true}) - Require(t, err, "unable to generate setValidator calldata") - tx, err := upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setValidatorCalldata) - Require(t, err, "unable to set validators") - _, err = builder.L1.EnsureTxSucceeded(tx) - Require(t, err) - setMinAssertPeriodCalldata, err := rollupABI.Pack("setMinimumAssertionPeriod", big.NewInt(1)) Require(t, err, "unable to generate setMinimumAssertionPeriod calldata") - tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setMinAssertPeriodCalldata) + tx, err := upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setMinAssertPeriodCalldata) Require(t, err, "unable to set minimum assertion period") _, err = builder.L1.EnsureTxSucceeded(tx) Require(t, err) - setAnyTrustFastConfirmerCalldata, err := rollupABI.Pack("setAnyTrustFastConfirmer", safeAddress) - Require(t, err, "unable to generate setAnyTrustFastConfirmer calldata") - tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setAnyTrustFastConfirmerCalldata) - Require(t, err, "unable to set anytrust fast confirmer") - _, err = builder.L1.EnsureTxSucceeded(tx) - Require(t, err) - valConfigA := staker.TestL1ValidatorConfig valConfigA.EnableFastConfirmation = true @@ -339,6 +315,30 @@ func TestFastConfirmationWithSafe(t *testing.T) { Require(t, err) valConfigA.Strategy = "MakeNodes" + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) + Require(t, err) + valWalletAddrA := *valWalletAddrAPtr + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) + Require(t, err) + if valWalletAddrA == *valWalletAddrCheck { + Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) + } + + safeAddress := deploySafe(t, builder.L1, builder.L1.Client, deployAuth, []common.Address{valWalletAddrA, srv.Address}) + setValidatorCalldata, err := rollupABI.Pack("setValidator", []common.Address{valWalletAddrA, l1authB.From, srv.Address, safeAddress}, []bool{true, true, true, true}) + Require(t, err, "unable to generate setValidator calldata") + tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setValidatorCalldata) + Require(t, err, "unable to set validators") + _, err = builder.L1.EnsureTxSucceeded(tx) + Require(t, err) + + setAnyTrustFastConfirmerCalldata, err := rollupABI.Pack("setAnyTrustFastConfirmer", safeAddress) + Require(t, err, "unable to generate setAnyTrustFastConfirmer calldata") + tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setAnyTrustFastConfirmerCalldata) + Require(t, err, "unable to set anytrust fast confirmer") + _, err = builder.L1.EnsureTxSucceeded(tx) + Require(t, err) + _, valStack := createTestValidationNode(t, ctx, &valnode.TestValidationConfig) blockValidatorConfig := staker.TestBlockValidatorConfig diff --git a/system_tests/staker_test.go b/system_tests/staker_test.go index 3fdf82ca74..e61dcedc2b 100644 --- a/system_tests/staker_test.go +++ b/system_tests/staker_test.go @@ -176,10 +176,10 @@ func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) valConfigA.Strategy = "MakeNodes" } - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.CreateWalletContract) + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) Require(t, err) valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.CreateWalletContract) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) Require(t, err) if valWalletAddrA == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) @@ -476,7 +476,7 @@ func TestStakersCooperative(t *testing.T) { stakerTestImpl(t, false, false) } -func TestGetValidatorWalletContractWithoutDataPoster(t *testing.T) { +func TestGetValidatorWalletContractWithDataposterOnlyUsedToCreateValidatorWalletContract(t *testing.T) { t.Parallel() ctx, cancelCtx := context.WithCancel(context.Background()) @@ -492,10 +492,25 @@ func TestGetValidatorWalletContractWithoutDataPoster(t *testing.T) { builder.L1.TransferBalance(t, "Faucet", "ValidatorA", balance, builder.L1Info) l1auth := builder.L1Info.GetDefaultTransactOpts("ValidatorA", ctx) - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, &l1auth, builder.L2.ConsensusNode.L1Reader, true, nil) + parentChainID, err := builder.L1.Client.ChainID(ctx) + Require(t, err) + + dataPoster, err := arbnode.DataposterOnlyUsedToCreateValidatorWalletContract( + ctx, + builder.L2.ConsensusNode.L1Reader, + &l1auth, + &builder.nodeConfig.Staker.DataPoster, + parentChainID, + ) + if err != nil { + log.Crit("error creating data poster to create validator wallet contract", "err", err) + } + getExtraGas := func() uint64 { return builder.nodeConfig.Staker.ExtraGas } + + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, &l1auth, builder.L2.ConsensusNode.L1Reader, true, dataPoster, getExtraGas) Require(t, err) valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, &l1auth, builder.L2.ConsensusNode.L1Reader, true, nil) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, &l1auth, builder.L2.ConsensusNode.L1Reader, true, dataPoster, getExtraGas) Require(t, err) if valWalletAddrA == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) From 6eae487866df45f46db437141e30254c0eefa4c6 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 30 Aug 2024 20:47:15 -0300 Subject: [PATCH 079/489] Creates validator wallet contract with transaction with value set to zero --- staker/validatorwallet/contract.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/staker/validatorwallet/contract.go b/staker/validatorwallet/contract.go index 3db35d8232..8a522b44a3 100644 --- a/staker/validatorwallet/contract.go +++ b/staker/validatorwallet/contract.go @@ -212,7 +212,7 @@ func createWalletContract( return nil, fmt.Errorf("getting gas for tx data: %w", err) } - return dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), validatorWalletFactoryAddr, txData, gas, auth.Value) + return dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), validatorWalletFactoryAddr, txData, gas, common.Big0) } func (v *Contract) populateWallet(ctx context.Context, createIfMissing bool) error { From caadac62a7be06a2296a0c1c0db0330e5745d800 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 30 Aug 2024 21:05:48 -0300 Subject: [PATCH 080/489] Improves error message when getting gas to create validator wallet --- staker/validatorwallet/contract.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/staker/validatorwallet/contract.go b/staker/validatorwallet/contract.go index 8a522b44a3..1910db1d63 100644 --- a/staker/validatorwallet/contract.go +++ b/staker/validatorwallet/contract.go @@ -209,7 +209,7 @@ func createWalletContract( getExtraGas, ) if err != nil { - return nil, fmt.Errorf("getting gas for tx data: %w", err) + return nil, fmt.Errorf("getting gas for tx data when creating validator wallet, validatorWalletFactory=%v: %w", validatorWalletFactoryAddr, err) } return dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), validatorWalletFactoryAddr, txData, gas, common.Big0) From 716bfea74964a6f2e51a2a7f119a1d9f9d314149 Mon Sep 17 00:00:00 2001 From: Pepper Lebeck-Jobe Date: Mon, 2 Sep 2024 12:23:03 +0200 Subject: [PATCH 081/489] Add Mac OSX environment variables The unfortunate thing is that it would still be good for developers to set these in their shell environment so that if they do end up running any "go" commands directly from the CLI, they won't be bombarded with all the linker warnings. But, at least this makes the default experience of using the make files a little nicer. --- Makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Makefile b/Makefile index e46bdbbe61..0a71d64f12 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,14 @@ ifneq ($(origin GOLANG_LDFLAGS),undefined) GOLANG_PARAMS = -ldflags="-extldflags '-ldl' $(GOLANG_LDFLAGS)" endif +UNAME_S := $(shell uname -s) + +# In Mac OSX, there are a lot of warnings emitted if these environment variables aren't set. +ifeq ($(UNAME_S), Darwin) + export MACOSX_DEPLOYMENT_TARGET := $(shell sw_vers -productVersion) + export CGO_LDFLAGS := -Wl,-no_warn_duplicate_libraries +endif + precompile_names = AddressTable Aggregator BLS Debug FunctionTable GasInfo Info osTest Owner RetryableTx Statistics Sys precompiles = $(patsubst %,./solgen/generated/%.go, $(precompile_names)) From 33337b8cd074c4a2ddf38dbcbe574e9712d1123f Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 2 Sep 2024 10:39:38 -0300 Subject: [PATCH 082/489] Renames getAuthWithUpdatedNonce to getAuthWithUpdatedNonceFromL1 --- staker/validatorwallet/contract.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/staker/validatorwallet/contract.go b/staker/validatorwallet/contract.go index 1910db1d63..83dedcad84 100644 --- a/staker/validatorwallet/contract.go +++ b/staker/validatorwallet/contract.go @@ -155,7 +155,7 @@ func (v *Contract) From() common.Address { } // nil value == 0 value -func getAuthWithUpdatedNonce(ctx context.Context, l1Reader *headerreader.HeaderReader, auth bind.TransactOpts, value *big.Int) (*bind.TransactOpts, error) { +func getAuthWithUpdatedNonceFromL1(ctx context.Context, l1Reader *headerreader.HeaderReader, auth bind.TransactOpts, value *big.Int) (*bind.TransactOpts, error) { auth.Context = ctx auth.Value = value nonce, err := l1Reader.Client().NonceAt(ctx, auth.From, nil) @@ -167,7 +167,7 @@ func getAuthWithUpdatedNonce(ctx context.Context, l1Reader *headerreader.HeaderR } func (v *Contract) getAuth(ctx context.Context, value *big.Int) (*bind.TransactOpts, error) { - return getAuthWithUpdatedNonce(ctx, v.l1Reader, *v.auth, value) + return getAuthWithUpdatedNonceFromL1(ctx, v.l1Reader, *v.auth, value) } func (v *Contract) executeTransaction(ctx context.Context, tx *types.Transaction, gasRefunder common.Address) (*types.Transaction, error) { @@ -485,7 +485,7 @@ func GetValidatorWalletContract( return nil, nil } - transactAuth, err = getAuthWithUpdatedNonce(ctx, l1Reader, *transactAuth, nil) + transactAuth, err = getAuthWithUpdatedNonceFromL1(ctx, l1Reader, *transactAuth, nil) if err != nil { return nil, err } From 9150808093cbfbc2fb98c9bb44e34b3b4ad927a7 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 2 Sep 2024 16:12:49 -0300 Subject: [PATCH 083/489] Don't modify config inside config fetcher --- arbnode/node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arbnode/node.go b/arbnode/node.go index e13f92f30f..a9da4ea24b 100644 --- a/arbnode/node.go +++ b/arbnode/node.go @@ -346,12 +346,12 @@ func DataposterOnlyUsedToCreateValidatorWalletContract( cfg *dataposter.DataPosterConfig, parentChainID *big.Int, ) (*dataposter.DataPoster, error) { + cfg.UseNoOpStorage = true return dataposter.NewDataPoster(ctx, &dataposter.DataPosterOpts{ HeaderReader: l1Reader, Auth: transactOpts, Config: func() *dataposter.DataPosterConfig { - cfg.UseNoOpStorage = true return cfg }, MetadataRetriever: func(ctx context.Context, blockNum *big.Int) ([]byte, error) { From 1bc34f20ff8f21a7c860dc059393309fd11d1d00 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Mon, 2 Sep 2024 14:23:02 -0500 Subject: [PATCH 084/489] Close opened files explicitly instead of relying on the GC --- arbnode/resourcemanager/resource_management.go | 2 ++ cmd/nitro/init.go | 1 + 2 files changed, 3 insertions(+) diff --git a/arbnode/resourcemanager/resource_management.go b/arbnode/resourcemanager/resource_management.go index aba823cc25..249b689443 100644 --- a/arbnode/resourcemanager/resource_management.go +++ b/arbnode/resourcemanager/resource_management.go @@ -256,6 +256,7 @@ func readIntFromFile(fileName string) (int, error) { if err != nil { return 0, err } + defer file.Close() var limit int if _, err = fmt.Fscanf(file, "%d", &limit); err != nil { @@ -269,6 +270,7 @@ func readFromMemStats(fileName string, re *regexp.Regexp) (int, error) { if err != nil { return 0, err } + defer file.Close() scanner := bufio.NewScanner(file) for scanner.Scan() { diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index 26cd698efc..51d895c069 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -423,6 +423,7 @@ func extractSnapshot(archive string, location string, importWasm bool) error { if err != nil { return fmt.Errorf("couln't open init '%v' archive: %w", archive, err) } + defer reader.Close() stat, err := reader.Stat() if err != nil { return err From 0bf0b9accf2ca8b6e6ef81d2ac6f9fa67bd6bdaa Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 2 Sep 2024 18:07:15 -0300 Subject: [PATCH 085/489] Uses auth from data poster in GetValidatorWalletContract --- cmd/nitro/nitro.go | 2 +- staker/validatorwallet/contract.go | 4 ++-- system_tests/fast_confirm_test.go | 8 ++++---- system_tests/staker_test.go | 8 ++++---- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index bad48f430c..0cf81c45e9 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -385,7 +385,7 @@ func mainImpl() int { getExtraGas := func() uint64 { return nodeConfig.Node.Staker.ExtraGas } // #nosec G115 - addr, err := validatorwallet.GetValidatorWalletContract(ctx, deployInfo.ValidatorWalletCreator, int64(deployInfo.DeployedAt), l1TransactionOptsValidator, l1Reader, true, dataPoster, getExtraGas) + addr, err := validatorwallet.GetValidatorWalletContract(ctx, deployInfo.ValidatorWalletCreator, int64(deployInfo.DeployedAt), l1Reader, true, dataPoster, getExtraGas) if err != nil { log.Crit("error creating validator wallet contract", "error", err, "address", l1TransactionOptsValidator.From.Hex()) } diff --git a/staker/validatorwallet/contract.go b/staker/validatorwallet/contract.go index 83dedcad84..6346029c3a 100644 --- a/staker/validatorwallet/contract.go +++ b/staker/validatorwallet/contract.go @@ -229,7 +229,7 @@ func (v *Contract) populateWallet(ctx context.Context, createIfMissing bool) err // By passing v.dataPoster as a parameter to GetValidatorWalletContract we force to create a validator wallet through the Staker's DataPoster object. // DataPoster keeps in its internal state information related to the transactions sent through it, which is used to infer the expected nonce in a transaction for example. // If a transaction is sent using the Staker's DataPoster key, but not through the Staker's DataPoster object, DataPoster's internal state will be outdated, which can compromise the expected nonce inference. - addr, err := GetValidatorWalletContract(ctx, v.walletFactoryAddr, v.rollupFromBlock, v.auth, v.l1Reader, createIfMissing, v.dataPoster, v.getExtraGas) + addr, err := GetValidatorWalletContract(ctx, v.walletFactoryAddr, v.rollupFromBlock, v.l1Reader, createIfMissing, v.dataPoster, v.getExtraGas) if err != nil { return err } @@ -444,13 +444,13 @@ func GetValidatorWalletContract( ctx context.Context, validatorWalletFactoryAddr common.Address, fromBlock int64, - transactAuth *bind.TransactOpts, l1Reader *headerreader.HeaderReader, createIfMissing bool, dataPoster *dataposter.DataPoster, getExtraGas func() uint64, ) (*common.Address, error) { client := l1Reader.Client() + transactAuth := dataPoster.Auth() // TODO: If we just save a mapping in the wallet creator we won't need log search walletCreator, err := rollupgen.NewValidatorWalletCreator(validatorWalletFactoryAddr, client) diff --git a/system_tests/fast_confirm_test.go b/system_tests/fast_confirm_test.go index 4598827418..dae2699b9f 100644 --- a/system_tests/fast_confirm_test.go +++ b/system_tests/fast_confirm_test.go @@ -115,10 +115,10 @@ func TestFastConfirmation(t *testing.T) { Require(t, err) valConfig.Strategy = "MakeNodes" - valWalletAddrPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, valWallet.DataPoster(), valWallet.GetExtraGas()) + valWalletAddrPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, l2node.L1Reader, true, valWallet.DataPoster(), valWallet.GetExtraGas()) Require(t, err) valWalletAddr := *valWalletAddrPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, valWallet.DataPoster(), valWallet.GetExtraGas()) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, l2node.L1Reader, true, valWallet.DataPoster(), valWallet.GetExtraGas()) Require(t, err) if valWalletAddr == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddr.String(), "vs", valWalletAddrCheck.String()) @@ -315,10 +315,10 @@ func TestFastConfirmationWithSafe(t *testing.T) { Require(t, err) valConfigA.Strategy = "MakeNodes" - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) Require(t, err) valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) Require(t, err) if valWalletAddrA == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) diff --git a/system_tests/staker_test.go b/system_tests/staker_test.go index e61dcedc2b..67ce260529 100644 --- a/system_tests/staker_test.go +++ b/system_tests/staker_test.go @@ -176,10 +176,10 @@ func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) valConfigA.Strategy = "MakeNodes" } - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) Require(t, err) valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) Require(t, err) if valWalletAddrA == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) @@ -507,10 +507,10 @@ func TestGetValidatorWalletContractWithDataposterOnlyUsedToCreateValidatorWallet } getExtraGas := func() uint64 { return builder.nodeConfig.Staker.ExtraGas } - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, &l1auth, builder.L2.ConsensusNode.L1Reader, true, dataPoster, getExtraGas) + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, builder.L2.ConsensusNode.L1Reader, true, dataPoster, getExtraGas) Require(t, err) valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, &l1auth, builder.L2.ConsensusNode.L1Reader, true, dataPoster, getExtraGas) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, builder.L2.ConsensusNode.L1Reader, true, dataPoster, getExtraGas) Require(t, err) if valWalletAddrA == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) From 857073870e455354ba13aa99c6bf06b81ba846ba Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 2 Sep 2024 18:08:05 -0300 Subject: [PATCH 086/489] Uses validator auth instead of batch poster auth when creating data poster to create validator wallet contract --- cmd/nitro/nitro.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index 0cf81c45e9..d4b2a87a30 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -375,7 +375,7 @@ func mainImpl() int { dataPoster, err := arbnode.DataposterOnlyUsedToCreateValidatorWalletContract( ctx, l1Reader, - l1TransactionOptsBatchPoster, + l1TransactionOptsValidator, &nodeConfig.Node.Staker.DataPoster, new(big.Int).SetUint64(nodeConfig.ParentChain.ID), ) From b553f8c3cb172b1b4e627cea59119e8e205600ef Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Tue, 3 Sep 2024 19:32:01 +0530 Subject: [PATCH 087/489] update submodule --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index d251082ca3..3aaaa0470d 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit d251082ca3311dc72b6fdb188988fa8d94e2b0bf +Subproject commit 3aaaa0470d7fbb0af61958ff543b123d4c7ee496 From 48a50164c378dc16131bcaab82b573793990922e Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Tue, 3 Sep 2024 19:45:21 +0530 Subject: [PATCH 088/489] update submodule --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 3aaaa0470d..cf49bca3cd 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 3aaaa0470d7fbb0af61958ff543b123d4c7ee496 +Subproject commit cf49bca3cd28fb3525eca9e7db2c33bd34acf125 From 236884f6ccf5e383ea654295d9b0a4f14d665cff Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Tue, 3 Sep 2024 20:13:24 +0530 Subject: [PATCH 089/489] fix stylus build --- execution/gethexec/stylus_tracer.go | 42 ++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/execution/gethexec/stylus_tracer.go b/execution/gethexec/stylus_tracer.go index 4c18bb2ebe..f054c5cb0b 100644 --- a/execution/gethexec/stylus_tracer.go +++ b/execution/gethexec/stylus_tracer.go @@ -13,6 +13,8 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/tracing" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/log" @@ -67,10 +69,27 @@ var nestsHostios = map[string]bool{ "static_call_contract": true, } -func newStylusTracer(ctx *tracers.Context, _ json.RawMessage) (tracers.Tracer, error) { - return &stylusTracer{ +func newStylusTracer(ctx *tracers.Context, _ json.RawMessage) (*tracers.Tracer, error) { + t := &stylusTracer{ open: containers.NewStack[HostioTraceInfo](), stack: containers.NewStack[*containers.Stack[HostioTraceInfo]](), + } + + return &tracers.Tracer{ + Hooks: &tracing.Hooks{ + OnExit: t.OnExit, + OnEnter: t.OnEnter, + OnOpcode: t.OnOpcode, + OnFault: t.OnFault, + OnTxStart: t.OnTxStart, + OnTxEnd: t.OnTxEnd, + CaptureArbitrumTransfer: t.CaptureArbitrumTransfer, + CaptureArbitrumStorageGet: t.CaptureArbitrumStorageGet, + CaptureArbitrumStorageSet: t.CaptureArbitrumStorageSet, + CaptureStylusHostio: t.CaptureStylusHostio, + }, + GetResult: t.GetResult, + Stop: t.Stop, }, nil } @@ -104,8 +123,7 @@ func (t *stylusTracer) CaptureStylusHostio(name string, args, outs []byte, start } t.open.Push(info) } - -func (t *stylusTracer) CaptureEnter(typ vm.OpCode, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { +func (t *stylusTracer) OnEnter(depth int, typ byte, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { if t.interrupt.Load() { return } @@ -113,7 +131,7 @@ func (t *stylusTracer) CaptureEnter(typ vm.OpCode, from common.Address, to commo // This function adds the prefix evm_ because it assumes the opcode came from the EVM. // If the opcode comes from WASM, the CaptureStylusHostio function will remove the evm prefix. var name string - switch typ { + switch vm.OpCode(typ) { case vm.CALL: name = "evm_call_contract" case vm.DELEGATECALL: @@ -139,7 +157,7 @@ func (t *stylusTracer) CaptureEnter(typ vm.OpCode, from common.Address, to commo t.open = inner } -func (t *stylusTracer) CaptureExit(output []byte, gasUsed uint64, _ error) { +func (t *stylusTracer) OnExit(depth int, output []byte, gasUsed uint64, _ error, reverted bool) { if t.interrupt.Load() { return } @@ -184,16 +202,14 @@ func (t *stylusTracer) Stop(err error) { // Unimplemented EVMLogger interface methods -func (t *stylusTracer) CaptureArbitrumTransfer(env *vm.EVM, from, to *common.Address, value *big.Int, before bool, purpose string) { +func (t *stylusTracer) CaptureArbitrumTransfer(from, to *common.Address, value *big.Int, before bool, purpose string) { } func (t *stylusTracer) CaptureArbitrumStorageGet(key common.Hash, depth int, before bool) {} func (t *stylusTracer) CaptureArbitrumStorageSet(key, value common.Hash, depth int, before bool) {} -func (t *stylusTracer) CaptureStart(env *vm.EVM, from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) { +func (t *stylusTracer) OnOpcode(pc uint64, opcode byte, gas, cost uint64, scope tracing.OpContext, rData []byte, depth int, err error) { } -func (t *stylusTracer) CaptureEnd(output []byte, gasUsed uint64, err error) {} -func (t *stylusTracer) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, rData []byte, depth int, err error) { +func (t *stylusTracer) OnFault(pc uint64, op byte, gas, cost uint64, _ tracing.OpContext, depth int, err error) { } -func (t *stylusTracer) CaptureFault(pc uint64, op vm.OpCode, gas, cost uint64, _ *vm.ScopeContext, depth int, err error) { +func (t *stylusTracer) OnTxStart(env *tracing.VMContext, tx *types.Transaction, from common.Address) { } -func (t *stylusTracer) CaptureTxStart(gasLimit uint64) {} -func (t *stylusTracer) CaptureTxEnd(restGas uint64) {} +func (t *stylusTracer) OnTxEnd(receipt *types.Receipt, err error) {} From 8e4e8f9dafcb40057ab59516e2dbc7604ef6e39d Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Tue, 3 Sep 2024 10:34:17 -0500 Subject: [PATCH 090/489] Use a macro to define native WASM imports --- arbitrator/stylus/src/host.rs | 88 +++++++++++---- arbitrator/stylus/src/native.rs | 186 +++++++++----------------------- 2 files changed, 118 insertions(+), 156 deletions(-) diff --git a/arbitrator/stylus/src/host.rs b/arbitrator/stylus/src/host.rs index 1afc1b4e51..ca48da35db 100644 --- a/arbitrator/stylus/src/host.rs +++ b/arbitrator/stylus/src/host.rs @@ -441,26 +441,76 @@ pub(crate) fn pay_for_memory_grow>( hostio!(env, pay_for_memory_grow(pages)) } -pub(crate) fn console_log_text>( - mut env: WasmEnvMut, - ptr: GuestPtr, - len: u32, -) -> MaybeEscape { - hostio!(env, console_log_text(ptr, len)) -} +pub(crate) mod console { + use super::*; -pub(crate) fn console_log, T: Into>( - mut env: WasmEnvMut, - value: T, -) -> MaybeEscape { - hostio!(env, console_log(value)) -} + pub(crate) fn log_txt>( + mut env: WasmEnvMut, + ptr: GuestPtr, + len: u32, + ) -> MaybeEscape { + hostio!(env, console_log_text(ptr, len)) + } -pub(crate) fn console_tee, T: Into + Copy>( - mut env: WasmEnvMut, - value: T, -) -> Result { - hostio!(env, console_tee(value)) + pub(crate) fn log_i32>( + mut env: WasmEnvMut, + value: u32, + ) -> MaybeEscape { + hostio!(env, console_log(value)) + } + + pub(crate) fn tee_i32>( + mut env: WasmEnvMut, + value: u32, + ) -> Result { + hostio!(env, console_tee(value)) + } + + pub(crate) fn log_i64>( + mut env: WasmEnvMut, + value: u64, + ) -> MaybeEscape { + hostio!(env, console_log(value)) + } + + pub(crate) fn tee_i64>( + mut env: WasmEnvMut, + value: u64, + ) -> Result { + hostio!(env, console_tee(value)) + } + + pub(crate) fn log_f32>( + mut env: WasmEnvMut, + value: f32, + ) -> MaybeEscape { + hostio!(env, console_log(value)) + } + + pub(crate) fn tee_f32>( + mut env: WasmEnvMut, + value: f32, + ) -> Result { + hostio!(env, console_tee(value)) + } + + pub(crate) fn log_f64>( + mut env: WasmEnvMut, + value: f64, + ) -> MaybeEscape { + hostio!(env, console_log(value)) + } + + pub(crate) fn tee_f64>( + mut env: WasmEnvMut, + value: f64, + ) -> Result { + hostio!(env, console_tee(value)) + } } -pub(crate) fn null_host>(_: WasmEnvMut) {} +pub(crate) mod debug { + use super::*; + + pub(crate) fn null_host>(_: WasmEnvMut) {} +} diff --git a/arbitrator/stylus/src/native.rs b/arbitrator/stylus/src/native.rs index cc1d191fe2..7a82314fbc 100644 --- a/arbitrator/stylus/src/native.rs +++ b/arbitrator/stylus/src/native.rs @@ -33,7 +33,7 @@ use std::{ ops::{Deref, DerefMut}, }; use wasmer::{ - imports, AsStoreMut, Function, FunctionEnv, Instance, Memory, Module, Pages, Store, Target, + AsStoreMut, Function, FunctionEnv, Imports, Instance, Memory, Module, Pages, Store, Target, TypedFunction, Value, WasmTypeList, }; use wasmer_vm::VMExtern; @@ -151,68 +151,58 @@ impl> NativeInstance { fn from_module(module: Module, mut store: Store, env: WasmEnv) -> Result { let debug_funcs = env.compile.debug.debug_funcs; let func_env = FunctionEnv::new(&mut store, env); + let mut imports = Imports::new(); macro_rules! func { - ($func:expr) => { - Function::new_typed_with_env(&mut store, &func_env, $func) + ($rust_mod:path, $func:ident) => {{ + use $rust_mod as rust_mod; + Function::new_typed_with_env(&mut store, &func_env, rust_mod::$func) + }}; + } + macro_rules! define_imports { + ($($wasm_mod:literal => $rust_mod:path { $( $import:ident ),* $(,)? },)* $(,)?) => { + $( + $( + define_imports!(@@imports $wasm_mod, func!($rust_mod, $import), $import, "arbitrator_forward__"); + )* + )* + }; + (@@imports $wasm_mod:literal, $func:expr, $import:ident, $($p:expr),*) => { + define_imports!(@imports $wasm_mod, $func, $import, $($p),*, ""); + }; + (@imports $wasm_mod:literal, $func:expr, $import:ident, $($p:expr),*) => { + $( + imports.define($wasm_mod, concat!($p, stringify!($import)), $func); + )* }; } - let mut imports = imports! { - "vm_hooks" => { - "read_args" => func!(host::read_args), - "write_result" => func!(host::write_result), - "exit_early" => func!(host::exit_early), - "storage_load_bytes32" => func!(host::storage_load_bytes32), - "storage_cache_bytes32" => func!(host::storage_cache_bytes32), - "storage_flush_cache" => func!(host::storage_flush_cache), - "transient_load_bytes32" => func!(host::transient_load_bytes32), - "transient_store_bytes32" => func!(host::transient_store_bytes32), - "call_contract" => func!(host::call_contract), - "delegate_call_contract" => func!(host::delegate_call_contract), - "static_call_contract" => func!(host::static_call_contract), - "create1" => func!(host::create1), - "create2" => func!(host::create2), - "read_return_data" => func!(host::read_return_data), - "return_data_size" => func!(host::return_data_size), - "emit_log" => func!(host::emit_log), - "account_balance" => func!(host::account_balance), - "account_code" => func!(host::account_code), - "account_codehash" => func!(host::account_codehash), - "account_code_size" => func!(host::account_code_size), - "evm_gas_left" => func!(host::evm_gas_left), - "evm_ink_left" => func!(host::evm_ink_left), - "block_basefee" => func!(host::block_basefee), - "chainid" => func!(host::chainid), - "block_coinbase" => func!(host::block_coinbase), - "block_gas_limit" => func!(host::block_gas_limit), - "block_number" => func!(host::block_number), - "block_timestamp" => func!(host::block_timestamp), - "contract_address" => func!(host::contract_address), - "math_div" => func!(host::math_div), - "math_mod" => func!(host::math_mod), - "math_pow" => func!(host::math_pow), - "math_add_mod" => func!(host::math_add_mod), - "math_mul_mod" => func!(host::math_mul_mod), - "msg_reentrant" => func!(host::msg_reentrant), - "msg_sender" => func!(host::msg_sender), - "msg_value" => func!(host::msg_value), - "tx_gas_price" => func!(host::tx_gas_price), - "tx_ink_price" => func!(host::tx_ink_price), - "tx_origin" => func!(host::tx_origin), - "pay_for_memory_grow" => func!(host::pay_for_memory_grow), - "native_keccak256" => func!(host::native_keccak256), + define_imports!( + "vm_hooks" => host { + read_args, write_result, exit_early, + storage_load_bytes32, storage_cache_bytes32, storage_flush_cache, transient_load_bytes32, transient_store_bytes32, + call_contract, delegate_call_contract, static_call_contract, create1, create2, read_return_data, return_data_size, + emit_log, + account_balance, account_code, account_codehash, account_code_size, + evm_gas_left, evm_ink_left, + block_basefee, chainid, block_coinbase, block_gas_limit, block_number, block_timestamp, + contract_address, + math_div, math_mod, math_pow, math_add_mod, math_mul_mod, + msg_reentrant, msg_sender, msg_value, + tx_gas_price, tx_ink_price, tx_origin, + pay_for_memory_grow, + native_keccak256, }, - }; + ); if debug_funcs { - imports.define("console", "log_txt", func!(host::console_log_text)); - imports.define("console", "log_i32", func!(host::console_log::)); - imports.define("console", "log_i64", func!(host::console_log::)); - imports.define("console", "log_f32", func!(host::console_log::)); - imports.define("console", "log_f64", func!(host::console_log::)); - imports.define("console", "tee_i32", func!(host::console_tee::)); - imports.define("console", "tee_i64", func!(host::console_tee::)); - imports.define("console", "tee_f32", func!(host::console_tee::)); - imports.define("console", "tee_f64", func!(host::console_tee::)); - imports.define("debug", "null_host", func!(host::null_host)); + define_imports!( + "console" => host::console { + log_txt, + log_i32, log_i64, log_f32, log_f64, + tee_i32, tee_i64, tee_f32, tee_f64, + }, + "debug" => host::debug { + null_host, + }, + ); } let instance = Instance::new(&mut store, &module, &imports)?; let exports = &instance.exports; @@ -351,86 +341,8 @@ impl> StartlessMachine for NativeInstance { } pub fn module(wasm: &[u8], compile: CompileConfig, target: Target) -> Result> { - let mut store = compile.store(target); + let store = compile.store(target); let module = Module::new(&store, wasm)?; - macro_rules! stub { - (u8 <- $($types:tt)+) => { - Function::new_typed(&mut store, $($types)+ -> u8 { panic!("incomplete import") }) - }; - (u32 <- $($types:tt)+) => { - Function::new_typed(&mut store, $($types)+ -> u32 { panic!("incomplete import") }) - }; - (u64 <- $($types:tt)+) => { - Function::new_typed(&mut store, $($types)+ -> u64 { panic!("incomplete import") }) - }; - (f32 <- $($types:tt)+) => { - Function::new_typed(&mut store, $($types)+ -> f32 { panic!("incomplete import") }) - }; - (f64 <- $($types:tt)+) => { - Function::new_typed(&mut store, $($types)+ -> f64 { panic!("incomplete import") }) - }; - ($($types:tt)+) => { - Function::new_typed(&mut store, $($types)+ -> () { panic!("incomplete import") }) - }; - } - let mut imports = imports! { - "vm_hooks" => { - "read_args" => stub!(|_: u32|), - "write_result" => stub!(|_: u32, _: u32|), - "exit_early" => stub!(|_: u32|), - "storage_load_bytes32" => stub!(|_: u32, _: u32|), - "storage_cache_bytes32" => stub!(|_: u32, _: u32|), - "storage_flush_cache" => stub!(|_: u32|), - "transient_load_bytes32" => stub!(|_: u32, _: u32|), - "transient_store_bytes32" => stub!(|_: u32, _: u32|), - "call_contract" => stub!(u8 <- |_: u32, _: u32, _: u32, _: u32, _: u64, _: u32|), - "delegate_call_contract" => stub!(u8 <- |_: u32, _: u32, _: u32, _: u64, _: u32|), - "static_call_contract" => stub!(u8 <- |_: u32, _: u32, _: u32, _: u64, _: u32|), - "create1" => stub!(|_: u32, _: u32, _: u32, _: u32, _: u32|), - "create2" => stub!(|_: u32, _: u32, _: u32, _: u32, _: u32, _: u32|), - "read_return_data" => stub!(u32 <- |_: u32, _: u32, _: u32|), - "return_data_size" => stub!(u32 <- ||), - "emit_log" => stub!(|_: u32, _: u32, _: u32|), - "account_balance" => stub!(|_: u32, _: u32|), - "account_code" => stub!(u32 <- |_: u32, _: u32, _: u32, _: u32|), - "account_codehash" => stub!(|_: u32, _: u32|), - "account_code_size" => stub!(u32 <- |_: u32|), - "evm_gas_left" => stub!(u64 <- ||), - "evm_ink_left" => stub!(u64 <- ||), - "block_basefee" => stub!(|_: u32|), - "chainid" => stub!(u64 <- ||), - "block_coinbase" => stub!(|_: u32|), - "block_gas_limit" => stub!(u64 <- ||), - "block_number" => stub!(u64 <- ||), - "block_timestamp" => stub!(u64 <- ||), - "contract_address" => stub!(|_: u32|), - "math_div" => stub!(|_: u32, _: u32|), - "math_mod" => stub!(|_: u32, _: u32|), - "math_pow" => stub!(|_: u32, _: u32|), - "math_add_mod" => stub!(|_: u32, _: u32, _: u32|), - "math_mul_mod" => stub!(|_: u32, _: u32, _: u32|), - "msg_reentrant" => stub!(u32 <- ||), - "msg_sender" => stub!(|_: u32|), - "msg_value" => stub!(|_: u32|), - "tx_gas_price" => stub!(|_: u32|), - "tx_ink_price" => stub!(u32 <- ||), - "tx_origin" => stub!(|_: u32|), - "pay_for_memory_grow" => stub!(|_: u16|), - "native_keccak256" => stub!(|_: u32, _: u32, _: u32|), - }, - }; - if compile.debug.debug_funcs { - imports.define("console", "log_txt", stub!(|_: u32, _: u32|)); - imports.define("console", "log_i32", stub!(|_: u32|)); - imports.define("console", "log_i64", stub!(|_: u64|)); - imports.define("console", "log_f32", stub!(|_: f32|)); - imports.define("console", "log_f64", stub!(|_: f64|)); - imports.define("console", "tee_i32", stub!(u32 <- |_: u32|)); - imports.define("console", "tee_i64", stub!(u64 <- |_: u64|)); - imports.define("console", "tee_f32", stub!(f32 <- |_: f32|)); - imports.define("console", "tee_f64", stub!(f64 <- |_: f64|)); - imports.define("debug", "null_host", stub!(||)); - } let module = module.serialize()?; Ok(module.to_vec()) From b913d6b53779489593fe4289d4a2335c86ce2b51 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Tue, 3 Sep 2024 10:51:48 -0500 Subject: [PATCH 091/489] Remove unused import --- arbitrator/stylus/src/host.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/arbitrator/stylus/src/host.rs b/arbitrator/stylus/src/host.rs index ca48da35db..fbe5657c5f 100644 --- a/arbitrator/stylus/src/host.rs +++ b/arbitrator/stylus/src/host.rs @@ -13,7 +13,6 @@ use arbutil::{ }; use caller_env::GuestPtr; use eyre::Result; -use prover::value::Value; use std::{ fmt::Display, mem::{self, MaybeUninit}, From d39f362da5c86964a68992b984a732ad618bf16c Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 4 Sep 2024 14:58:22 -0600 Subject: [PATCH 092/489] redis consumer fix --- validator/valnode/redis/consumer.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/validator/valnode/redis/consumer.go b/validator/valnode/redis/consumer.go index 7456466533..6432741eba 100644 --- a/validator/valnode/redis/consumer.go +++ b/validator/valnode/redis/consumer.go @@ -111,10 +111,12 @@ func (s *ValidationServer) Start(ctx_in context.Context) { req, err := c.Consume(ctx) if err != nil { log.Error("Consuming request", "error", err) + requestTokenQueue <- struct{}{} return 0 } if req == nil { - // There's nothing in the queue. + // There's nothing in the queue + requestTokenQueue <- struct{}{} return time.Second } select { From 8d61504e109a89b12112ddbd4be94eb2753b0802 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 4 Sep 2024 15:00:53 -0600 Subject: [PATCH 093/489] redis consumer: dont set result if error --- validator/valnode/redis/consumer.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/validator/valnode/redis/consumer.go b/validator/valnode/redis/consumer.go index 6432741eba..2b025600cc 100644 --- a/validator/valnode/redis/consumer.go +++ b/validator/valnode/redis/consumer.go @@ -154,9 +154,10 @@ func (s *ValidationServer) Start(ctx_in context.Context) { res, err := valRun.Await(ctx) if err != nil { log.Error("Error validating", "request value", work.req.Value, "error", err) - } - if err := s.consumers[work.moduleRoot].SetResult(ctx, work.req.ID, res); err != nil { - log.Error("Error setting result for request", "id", work.req.ID, "result", res, "error", err) + } else { + if err := s.consumers[work.moduleRoot].SetResult(ctx, work.req.ID, res); err != nil { + log.Error("Error setting result for request", "id", work.req.ID, "result", res, "error", err) + } } select { case <-ctx.Done(): From e1bd2cef08fe3c5abd4fc342b4f44d9e3753ceed Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Thu, 5 Sep 2024 16:42:18 +0530 Subject: [PATCH 094/489] Remove IPFS support --- cmd/conf/chain.go | 27 ++- cmd/ipfshelper/ipfshelper.bkup_go | 281 ------------------------------ cmd/ipfshelper/ipfshelper_stub.go | 31 ---- cmd/ipfshelper/ipfshelper_test.go | 123 ------------- cmd/nitro/init.go | 23 +-- cmd/nitro/nitro.go | 30 +--- cmd/util/chaininfoutil.go | 29 --- 7 files changed, 18 insertions(+), 526 deletions(-) delete mode 100644 cmd/ipfshelper/ipfshelper.bkup_go delete mode 100644 cmd/ipfshelper/ipfshelper_stub.go delete mode 100644 cmd/ipfshelper/ipfshelper_test.go delete mode 100644 cmd/util/chaininfoutil.go diff --git a/cmd/conf/chain.go b/cmd/conf/chain.go index b85f7727b1..28b06aad2b 100644 --- a/cmd/conf/chain.go +++ b/cmd/conf/chain.go @@ -52,23 +52,19 @@ func (c *ParentChainConfig) Validate() error { } type L2Config struct { - ID uint64 `koanf:"id"` - Name string `koanf:"name"` - InfoFiles []string `koanf:"info-files"` - InfoJson string `koanf:"info-json"` - DevWallet genericconf.WalletConfig `koanf:"dev-wallet"` - InfoIpfsUrl string `koanf:"info-ipfs-url"` - InfoIpfsDownloadPath string `koanf:"info-ipfs-download-path"` + ID uint64 `koanf:"id"` + Name string `koanf:"name"` + InfoFiles []string `koanf:"info-files"` + InfoJson string `koanf:"info-json"` + DevWallet genericconf.WalletConfig `koanf:"dev-wallet"` } var L2ConfigDefault = L2Config{ - ID: 0, - Name: "", - InfoFiles: []string{}, // Default file used is chaininfo/arbitrum_chain_info.json, stored in DefaultChainInfo in chain_info.go - InfoJson: "", - DevWallet: genericconf.WalletConfigDefault, - InfoIpfsUrl: "", - InfoIpfsDownloadPath: "/tmp/", + ID: 0, + Name: "", + InfoFiles: []string{}, // Default file used is chaininfo/arbitrum_chain_info.json, stored in DefaultChainInfo in chain_info.go + InfoJson: "", + DevWallet: genericconf.WalletConfigDefault, } func L2ConfigAddOptions(prefix string, f *flag.FlagSet) { @@ -79,9 +75,6 @@ func L2ConfigAddOptions(prefix string, f *flag.FlagSet) { // Dev wallet does not exist unless specified genericconf.WalletConfigAddOptions(prefix+".dev-wallet", f, "") - f.String(prefix+".info-ipfs-url", L2ConfigDefault.InfoIpfsUrl, "url to download chain info file") - f.String(prefix+".info-ipfs-download-path", L2ConfigDefault.InfoIpfsDownloadPath, "path to save temp downloaded file") - } func (c *L2Config) ResolveDirectoryNames(chain string) { diff --git a/cmd/ipfshelper/ipfshelper.bkup_go b/cmd/ipfshelper/ipfshelper.bkup_go deleted file mode 100644 index ccde492ca6..0000000000 --- a/cmd/ipfshelper/ipfshelper.bkup_go +++ /dev/null @@ -1,281 +0,0 @@ -//go:build ipfs -// +build ipfs - -package ipfshelper - -import ( - "context" - "fmt" - "io" - "math/rand" - "os" - "path/filepath" - "strings" - "sync" - - "github.com/ethereum/go-ethereum/log" - "github.com/ipfs/go-libipfs/files" - coreiface "github.com/ipfs/interface-go-ipfs-core" - "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/ipfs/interface-go-ipfs-core/path" - "github.com/ipfs/kubo/config" - "github.com/ipfs/kubo/core" - "github.com/ipfs/kubo/core/coreapi" - "github.com/ipfs/kubo/core/node/libp2p" - "github.com/ipfs/kubo/plugin/loader" - "github.com/ipfs/kubo/repo" - "github.com/ipfs/kubo/repo/fsrepo" - "github.com/libp2p/go-libp2p/core/host" - "github.com/libp2p/go-libp2p/core/peer" - ma "github.com/multiformats/go-multiaddr" -) - -const DefaultIpfsProfiles = "" - -type IpfsHelper struct { - api coreiface.CoreAPI - node *core.IpfsNode - cfg *config.Config - repoPath string - repo repo.Repo -} - -func (h *IpfsHelper) createRepo(downloadPath string, profiles string) error { - fileInfo, err := os.Stat(downloadPath) - if err != nil { - return fmt.Errorf("failed to stat ipfs repo directory: %w", err) - } - if !fileInfo.IsDir() { - return fmt.Errorf("%s is not a directory", downloadPath) - } - h.repoPath = filepath.Join(downloadPath, "ipfs-repo") - // Create a config with default options and a 2048 bit key - h.cfg, err = config.Init(io.Discard, 2048) - if err != nil { - return err - } - if len(profiles) > 0 { - for _, profile := range strings.Split(profiles, ",") { - transformer, ok := config.Profiles[profile] - if !ok { - return fmt.Errorf("invalid ipfs configuration profile: %s", profile) - } - - if err := transformer.Transform(h.cfg); err != nil { - return err - } - } - } - // Create the repo with the config - // fsrepo.Init initializes new repo only if it's not initialized yet - err = fsrepo.Init(h.repoPath, h.cfg) - if err != nil { - return fmt.Errorf("failed to init ipfs repo: %w", err) - } - h.repo, err = fsrepo.Open(h.repoPath) - if err != nil { - return fmt.Errorf("failed to open ipfs repo: %w", err) - } - return nil -} - -func (h *IpfsHelper) createNode(ctx context.Context, clientOnly bool) error { - var routing libp2p.RoutingOption - if clientOnly { - routing = libp2p.DHTClientOption - } else { - routing = libp2p.DHTOption - } - nodeOptions := &core.BuildCfg{ - Online: true, - Routing: routing, - Repo: h.repo, - } - var err error - h.node, err = core.NewNode(ctx, nodeOptions) - if err != nil { - return err - } - h.api, err = coreapi.NewCoreAPI(h.node) - return err -} - -func (h *IpfsHelper) connectToPeers(ctx context.Context, peers []string) error { - peerInfos := make(map[peer.ID]*peer.AddrInfo, len(peers)) - for _, addressString := range peers { - address, err := ma.NewMultiaddr(addressString) - if err != nil { - return err - } - addressInfo, err := peer.AddrInfoFromP2pAddr(address) - if err != nil { - return err - } - peerInfo, ok := peerInfos[addressInfo.ID] - if !ok { - peerInfo = &peer.AddrInfo{ID: addressInfo.ID} - peerInfos[peerInfo.ID] = peerInfo - } - peerInfo.Addrs = append(peerInfo.Addrs, addressInfo.Addrs...) - } - var wg sync.WaitGroup - wg.Add(len(peerInfos)) - for _, peerInfo := range peerInfos { - go func(peerInfo *peer.AddrInfo) { - defer wg.Done() - err := h.api.Swarm().Connect(ctx, *peerInfo) - if err != nil { - log.Warn("failed to connect to peer", "peerId", peerInfo.ID, "err", err) - return - } - }(peerInfo) - } - wg.Wait() - return nil -} - -func (h *IpfsHelper) GetPeerHostAddresses() ([]string, error) { - addresses, err := peer.AddrInfoToP2pAddrs(host.InfoFromHost(h.node.PeerHost)) - if err != nil { - return []string{}, err - } - addressesStrings := make([]string, len(addresses)) - for i, a := range addresses { - addressesStrings[i] = a.String() - } - return addressesStrings, nil -} - -func normalizeCidString(cidString string) string { - if strings.HasPrefix(cidString, "ipfs://") { - return "/ipfs/" + cidString[7:] - } - if strings.HasPrefix(cidString, "ipns://") { - return "/ipns/" + cidString[7:] - } - return cidString -} - -func (h *IpfsHelper) DownloadFile(ctx context.Context, cidString string, destinationDir string) (string, error) { - cidString = normalizeCidString(cidString) - cidPath := path.New(cidString) - resolvedPath, err := h.api.ResolvePath(ctx, cidPath) - if err != nil { - return "", fmt.Errorf("failed to resolve path: %w", err) - } - // first pin the root node, then all its children nodes in random order to improve sharing with peers started at the same time - if err := h.api.Pin().Add(ctx, resolvedPath, options.Pin.Recursive(false)); err != nil { - return "", fmt.Errorf("failed to pin root path: %w", err) - } - links, err := h.api.Object().Links(ctx, resolvedPath) - if err != nil { - return "", fmt.Errorf("failed to get root links: %w", err) - } - log.Info("Pinning ipfs subtrees...") - printProgress := func(done int, all int) { - if all == 0 { - all = 1 // avoid division by 0 - done = 1 - } - fmt.Printf("\033[2K\rPinned %d / %d subtrees (%.2f%%)", done, all, float32(done)/float32(all)*100) - } - permutation := rand.Perm(len(links)) - printProgress(0, len(links)) - for i, j := range permutation { - link := links[j] - if err := h.api.Pin().Add(ctx, path.IpfsPath(link.Cid), options.Pin.Recursive(true)); err != nil { - return "", fmt.Errorf("failed to pin child path: %w", err) - } - printProgress(i+1, len(links)) - } - fmt.Printf("\n") - rootNodeDirectory, err := h.api.Unixfs().Get(ctx, cidPath) - if err != nil { - return "", fmt.Errorf("could not get file with CID: %w", err) - } - log.Info("Writing file...") - outputFilePath := filepath.Join(destinationDir, resolvedPath.Cid().String()) - _ = os.Remove(outputFilePath) - err = files.WriteTo(rootNodeDirectory, outputFilePath) - if err != nil { - return "", fmt.Errorf("could not write out the fetched CID: %w", err) - } - log.Info("Download done.") - return outputFilePath, nil -} - -func (h *IpfsHelper) AddFile(ctx context.Context, filePath string, includeHidden bool) (path.Resolved, error) { - fileInfo, err := os.Stat(filePath) - if err != nil { - return nil, err - } - fileNode, err := files.NewSerialFile(filePath, includeHidden, fileInfo) - if err != nil { - return nil, err - } - return h.api.Unixfs().Add(ctx, fileNode) -} - -func CreateIpfsHelper(ctx context.Context, downloadPath string, clientOnly bool, peerList []string, profiles string) (*IpfsHelper, error) { - return createIpfsHelperImpl(ctx, downloadPath, clientOnly, peerList, profiles) -} - -func (h *IpfsHelper) Close() error { - return h.node.Close() -} - -func setupPlugins() error { - plugins, err := loader.NewPluginLoader("") - if err != nil { - return fmt.Errorf("error loading plugins: %w", err) - } - // Load preloaded and external plugins - if err := plugins.Initialize(); err != nil { - return fmt.Errorf("error initializing plugins: %w", err) - } - if err := plugins.Inject(); err != nil { - return fmt.Errorf("error initializing plugins: %w", err) - } - return nil -} - -var loadPluginsOnce sync.Once - -func createIpfsHelperImpl(ctx context.Context, downloadPath string, clientOnly bool, peerList []string, profiles string) (*IpfsHelper, error) { - var onceErr error - loadPluginsOnce.Do(func() { - onceErr = setupPlugins() - }) - if onceErr != nil { - return nil, onceErr - } - client := IpfsHelper{} - err := client.createRepo(downloadPath, profiles) - if err != nil { - return nil, err - } - err = client.createNode(ctx, clientOnly) - if err != nil { - return nil, err - } - err = client.connectToPeers(ctx, peerList) - if err != nil { - return nil, err - } - return &client, nil -} - -func CanBeIpfsPath(pathString string) bool { - path := path.New(pathString) - return path.IsValid() == nil || - strings.HasPrefix(pathString, "/ipfs/") || - strings.HasPrefix(pathString, "/ipld/") || - strings.HasPrefix(pathString, "/ipns/") || - strings.HasPrefix(pathString, "ipfs://") || - strings.HasPrefix(pathString, "ipns://") -} - -// TODO break abstraction for now til we figure out what fns are needed -func (h *IpfsHelper) GetAPI() coreiface.CoreAPI { - return h.api -} diff --git a/cmd/ipfshelper/ipfshelper_stub.go b/cmd/ipfshelper/ipfshelper_stub.go deleted file mode 100644 index fa6a451927..0000000000 --- a/cmd/ipfshelper/ipfshelper_stub.go +++ /dev/null @@ -1,31 +0,0 @@ -//go:build !ipfs -// +build !ipfs - -package ipfshelper - -import ( - "context" - "errors" -) - -type IpfsHelper struct{} - -var ErrIpfsNotSupported = errors.New("ipfs not supported") - -var DefaultIpfsProfiles = "default ipfs profiles stub" - -func CanBeIpfsPath(pathString string) bool { - return false -} - -func CreateIpfsHelper(ctx context.Context, downloadPath string, clientOnly bool, peerList []string, profiles string) (*IpfsHelper, error) { - return nil, ErrIpfsNotSupported -} - -func (h *IpfsHelper) DownloadFile(ctx context.Context, cidString string, destinationDir string) (string, error) { - return "", ErrIpfsNotSupported -} - -func (h *IpfsHelper) Close() error { - return ErrIpfsNotSupported -} diff --git a/cmd/ipfshelper/ipfshelper_test.go b/cmd/ipfshelper/ipfshelper_test.go deleted file mode 100644 index 80f10c21f6..0000000000 --- a/cmd/ipfshelper/ipfshelper_test.go +++ /dev/null @@ -1,123 +0,0 @@ -//go:build ipfs -// +build ipfs - -package ipfshelper - -import ( - "bytes" - "context" - "math/rand" - "os" - "path/filepath" - "testing" - "time" - - "github.com/offchainlabs/nitro/util/testhelpers" -) - -func getTempFileWithData(t *testing.T, data []byte) string { - path := filepath.Join(t.TempDir(), "config.json") - err := os.WriteFile(path, []byte(data), 0600) - testhelpers.RequireImpl(t, err) - return path -} - -func fileDataEqual(t *testing.T, path string, expected []byte) bool { - data, err := os.ReadFile(path) - testhelpers.RequireImpl(t, err) - return bytes.Equal(data, expected) -} - -func TestIpfsHelper(t *testing.T) { - ctx := context.Background() - ipfsA, err := createIpfsHelperImpl(ctx, t.TempDir(), false, []string{}, "test") - testhelpers.RequireImpl(t, err) - // add a test file to node A - testData := make([]byte, 1024*1024) - _, err = rand.Read(testData) - testhelpers.RequireImpl(t, err) - testFile := getTempFileWithData(t, testData) - ipfsTestFilePath, err := ipfsA.AddFile(ctx, testFile, false) - testhelpers.RequireImpl(t, err) - testFileCid := ipfsTestFilePath.Cid().String() - addrsA, err := ipfsA.GetPeerHostAddresses() - testhelpers.RequireImpl(t, err) - // create node B connected to node A - ipfsB, err := createIpfsHelperImpl(ctx, t.TempDir(), false, addrsA, "test") - testhelpers.RequireImpl(t, err) - // download the test file with node B - downloadedFile, err := ipfsB.DownloadFile(ctx, testFileCid, t.TempDir()) - testhelpers.RequireImpl(t, err) - if !fileDataEqual(t, downloadedFile, testData) { - testhelpers.FailImpl(t, "Downloaded file does not contain expected data") - } - // clean up node A and test downloading the file from yet another node C - err = ipfsA.Close() - os.RemoveAll(ipfsA.repoPath) - testhelpers.RequireImpl(t, err) - addrsB, err := ipfsB.GetPeerHostAddresses() - testhelpers.RequireImpl(t, err) - ipfsC, err := createIpfsHelperImpl(ctx, t.TempDir(), false, addrsB, "test") - testhelpers.RequireImpl(t, err) - downloadedFile, err = ipfsC.DownloadFile(ctx, testFileCid, t.TempDir()) - testhelpers.RequireImpl(t, err) - if !fileDataEqual(t, downloadedFile, testData) { - testhelpers.FailImpl(t, "Downloaded file does not contain expected data") - } - // make sure closing B and C nodes (A already closed) will make it impossible to download the test file from new node D - ipfsD, err := createIpfsHelperImpl(ctx, t.TempDir(), false, addrsB, "test") - testhelpers.RequireImpl(t, err) - err = ipfsB.Close() - testhelpers.RequireImpl(t, err) - err = ipfsC.Close() - testhelpers.RequireImpl(t, err) - testTimeout := 300 * time.Millisecond - timeoutCtx, cancel := context.WithTimeout(ctx, testTimeout) - defer cancel() - _, err = ipfsD.DownloadFile(timeoutCtx, testFileCid, t.TempDir()) - if err == nil { - testhelpers.FailImpl(t, "Download attempt did not fail as expected") - } - err = ipfsD.Close() - testhelpers.RequireImpl(t, err) -} - -func TestNormalizeCidString(t *testing.T) { - for _, test := range []struct { - input string - expected string - }{ - {"ipfs://QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", "/ipfs/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ"}, - {"ipns://k51qzi5uqu5dlvj2baxnqndepeb86cbk3ng7n3i46uzyxzyqj2xjonzllnv0v8", "/ipns/k51qzi5uqu5dlvj2baxnqndepeb86cbk3ng7n3i46uzyxzyqj2xjonzllnv0v8"}, - {"ipns://docs.ipfs.tech/introduction/", "/ipns/docs.ipfs.tech/introduction/"}, - {"/ipfs/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", "/ipfs/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ"}, - {"/ipns/k51qzi5uqu5dlvj2baxnqndepeb86cbk3ng7n3i46uzyxzyqj2xjonzllnv0v8", "/ipns/k51qzi5uqu5dlvj2baxnqndepeb86cbk3ng7n3i46uzyxzyqj2xjonzllnv0v8"}, - {"QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", "QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ"}, - } { - if res := normalizeCidString(test.input); res != test.expected { - testhelpers.FailImpl(t, "Failed to normalize cid string, input: ", test.input, " got: ", res, " expected: ", test.expected) - } - } -} - -func TestCanBeIpfsPath(t *testing.T) { - correctPaths := []string{ - "QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", - "/ipfs/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", - "/ipns/k51qzi5uqu5dlvj2baxnqndepeb86cbk3ng7n3i46uzyxzyqj2xjonzllnv0v8", - "/ipns/docs.ipfs.tech/introduction/", - "ipfs://QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", - "ipns://k51qzi5uqu5dlvj2baxnqndepeb86cbk3ng7n3i46uzyxzyqj2xjonzllnv0v8", - } - for _, path := range correctPaths { - if !CanBeIpfsPath(path) { - testhelpers.FailImpl(t, "false negative result for path:", path) - } - } - incorrectPaths := []string{"www.ipfs.tech", "https://www.ipfs.tech", "QmIncorrect"} - for _, path := range incorrectPaths { - if CanBeIpfsPath(path) { - testhelpers.FailImpl(t, "false positive result for path:", path) - } - } -} diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index a8463a7d21..9048ffd609 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -40,7 +40,6 @@ import ( "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/cmd/chaininfo" "github.com/offchainlabs/nitro/cmd/conf" - "github.com/offchainlabs/nitro/cmd/ipfshelper" "github.com/offchainlabs/nitro/cmd/pruning" "github.com/offchainlabs/nitro/cmd/staterecovery" "github.com/offchainlabs/nitro/execution/gethexec" @@ -58,25 +57,6 @@ func downloadInit(ctx context.Context, initConfig *conf.InitConfig) (string, err if strings.HasPrefix(initConfig.Url, "file:") { return initConfig.Url[5:], nil } - if ipfshelper.CanBeIpfsPath(initConfig.Url) { - ipfsNode, err := ipfshelper.CreateIpfsHelper(ctx, initConfig.DownloadPath, false, []string{}, ipfshelper.DefaultIpfsProfiles) - if err != nil { - return "", err - } - log.Info("Downloading initial database via IPFS", "url", initConfig.Url) - initFile, downloadErr := ipfsNode.DownloadFile(ctx, initConfig.Url, initConfig.DownloadPath) - closeErr := ipfsNode.Close() - if downloadErr != nil { - if closeErr != nil { - log.Error("Failed to close IPFS node after download error", "err", closeErr) - } - return "", fmt.Errorf("Failed to download file from IPFS: %w", downloadErr) - } - if closeErr != nil { - return "", fmt.Errorf("Failed to close IPFS node: %w", err) - } - return initFile, nil - } log.Info("Downloading initial database", "url", initConfig.Url) if !initConfig.ValidateChecksum { file, err := downloadFile(ctx, initConfig, initConfig.Url, nil) @@ -732,8 +712,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo if err != nil { return chainDb, nil, err } - combinedL2ChainInfoFiles := aggregateL2ChainInfoFiles(ctx, config.Chain.InfoFiles, config.Chain.InfoIpfsUrl, config.Chain.InfoIpfsDownloadPath) - chainConfig, err = chaininfo.GetChainConfig(new(big.Int).SetUint64(config.Chain.ID), config.Chain.Name, genesisBlockNr, combinedL2ChainInfoFiles, config.Chain.InfoJson) + chainConfig, err = chaininfo.GetChainConfig(new(big.Int).SetUint64(config.Chain.ID), config.Chain.Name, genesisBlockNr, config.Chain.InfoFiles, config.Chain.InfoJson) if err != nil { return chainDb, nil, err } diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index 146a0049e7..2c192a1d8e 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -285,8 +285,6 @@ func mainImpl() int { } } - combinedL2ChainInfoFile := aggregateL2ChainInfoFiles(ctx, nodeConfig.Chain.InfoFiles, nodeConfig.Chain.InfoIpfsUrl, nodeConfig.Chain.InfoIpfsDownloadPath) - if nodeConfig.Node.Staker.Enable { if !nodeConfig.Node.ParentChainReader.Enable { flag.Usage() @@ -335,7 +333,7 @@ func mainImpl() int { log.Info("connected to l1 chain", "l1url", nodeConfig.ParentChain.Connection.URL, "l1chainid", nodeConfig.ParentChain.ID) - rollupAddrs, err = chaininfo.GetRollupAddressesConfig(nodeConfig.Chain.ID, nodeConfig.Chain.Name, combinedL2ChainInfoFile, nodeConfig.Chain.InfoJson) + rollupAddrs, err = chaininfo.GetRollupAddressesConfig(nodeConfig.Chain.ID, nodeConfig.Chain.Name, nodeConfig.Chain.InfoFiles, nodeConfig.Chain.InfoJson) if err != nil { log.Crit("error getting rollup addresses", "err", err) } @@ -367,7 +365,7 @@ func mainImpl() int { log.Crit("--node.validator.only-create-wallet-contract conflicts with --node.dangerous.no-l1-listener") } // Just create validator smart wallet if needed then exit - deployInfo, err := chaininfo.GetRollupAddressesConfig(nodeConfig.Chain.ID, nodeConfig.Chain.Name, combinedL2ChainInfoFile, nodeConfig.Chain.InfoJson) + deployInfo, err := chaininfo.GetRollupAddressesConfig(nodeConfig.Chain.ID, nodeConfig.Chain.Name, nodeConfig.Chain.InfoFiles, nodeConfig.Chain.InfoJson) if err != nil { log.Crit("error getting rollup addresses config", "err", err) } @@ -541,7 +539,7 @@ func mainImpl() int { return 0 } - chainInfo, err := chaininfo.ProcessChainInfo(nodeConfig.Chain.ID, nodeConfig.Chain.Name, combinedL2ChainInfoFile, nodeConfig.Chain.InfoJson) + chainInfo, err := chaininfo.ProcessChainInfo(nodeConfig.Chain.ID, nodeConfig.Chain.Name, nodeConfig.Chain.InfoFiles, nodeConfig.Chain.InfoJson) if err != nil { log.Error("error processing l2 chain info", "err", err) return 1 @@ -888,11 +886,9 @@ func ParseNode(ctx context.Context, args []string) (*NodeConfig, *genericconf.Wa l2ChainId := k.Int64("chain.id") l2ChainName := k.String("chain.name") - l2ChainInfoIpfsUrl := k.String("chain.info-ipfs-url") - l2ChainInfoIpfsDownloadPath := k.String("chain.info-ipfs-download-path") l2ChainInfoFiles := k.Strings("chain.info-files") l2ChainInfoJson := k.String("chain.info-json") - err = applyChainParameters(ctx, k, uint64(l2ChainId), l2ChainName, l2ChainInfoFiles, l2ChainInfoJson, l2ChainInfoIpfsUrl, l2ChainInfoIpfsDownloadPath) + err = applyChainParameters(k, uint64(l2ChainId), l2ChainName, l2ChainInfoFiles, l2ChainInfoJson) if err != nil { return nil, nil, err } @@ -955,20 +951,8 @@ func ParseNode(ctx context.Context, args []string) (*NodeConfig, *genericconf.Wa return &nodeConfig, &l2DevWallet, nil } -func aggregateL2ChainInfoFiles(ctx context.Context, l2ChainInfoFiles []string, l2ChainInfoIpfsUrl string, l2ChainInfoIpfsDownloadPath string) []string { - if l2ChainInfoIpfsUrl != "" { - l2ChainInfoIpfsFile, err := util.GetL2ChainInfoIpfsFile(ctx, l2ChainInfoIpfsUrl, l2ChainInfoIpfsDownloadPath) - if err != nil { - log.Error("error getting l2 chain info file from ipfs", "err", err) - } - l2ChainInfoFiles = append(l2ChainInfoFiles, l2ChainInfoIpfsFile) - } - return l2ChainInfoFiles -} - -func applyChainParameters(ctx context.Context, k *koanf.Koanf, chainId uint64, chainName string, l2ChainInfoFiles []string, l2ChainInfoJson string, l2ChainInfoIpfsUrl string, l2ChainInfoIpfsDownloadPath string) error { - combinedL2ChainInfoFiles := aggregateL2ChainInfoFiles(ctx, l2ChainInfoFiles, l2ChainInfoIpfsUrl, l2ChainInfoIpfsDownloadPath) - chainInfo, err := chaininfo.ProcessChainInfo(chainId, chainName, combinedL2ChainInfoFiles, l2ChainInfoJson) +func applyChainParameters(k *koanf.Koanf, chainId uint64, chainName string, l2ChainInfoFiles []string, l2ChainInfoJson string) error { + chainInfo, err := chaininfo.ProcessChainInfo(chainId, chainName, l2ChainInfoFiles, l2ChainInfoJson) if err != nil { return err } @@ -977,7 +961,7 @@ func applyChainParameters(ctx context.Context, k *koanf.Koanf, chainId uint64, c parentChainIsArbitrum = *chainInfo.ParentChainIsArbitrum } else { log.Warn("Chain info field parent-chain-is-arbitrum is missing, in the future this will be required", "chainId", chainInfo.ChainConfig.ChainID, "parentChainId", chainInfo.ParentChainId) - _, err := chaininfo.ProcessChainInfo(chainInfo.ParentChainId, "", combinedL2ChainInfoFiles, "") + _, err := chaininfo.ProcessChainInfo(chainInfo.ParentChainId, "", l2ChainInfoFiles, "") if err == nil { parentChainIsArbitrum = true } diff --git a/cmd/util/chaininfoutil.go b/cmd/util/chaininfoutil.go deleted file mode 100644 index 906aa234ed..0000000000 --- a/cmd/util/chaininfoutil.go +++ /dev/null @@ -1,29 +0,0 @@ -package util - -import ( - "context" - "fmt" - - "github.com/ethereum/go-ethereum/log" - "github.com/offchainlabs/nitro/cmd/ipfshelper" -) - -func GetL2ChainInfoIpfsFile(ctx context.Context, l2ChainInfoIpfsUrl string, l2ChainInfoIpfsDownloadPath string) (string, error) { - ipfsNode, err := ipfshelper.CreateIpfsHelper(ctx, l2ChainInfoIpfsDownloadPath, false, []string{}, ipfshelper.DefaultIpfsProfiles) - if err != nil { - return "", err - } - log.Info("Downloading l2 info file via IPFS", "url", l2ChainInfoIpfsDownloadPath) - l2ChainInfoFile, downloadErr := ipfsNode.DownloadFile(ctx, l2ChainInfoIpfsUrl, l2ChainInfoIpfsDownloadPath) - closeErr := ipfsNode.Close() - if downloadErr != nil { - if closeErr != nil { - log.Error("Failed to close IPFS node after download error", "err", closeErr) - } - return "", fmt.Errorf("failed to download file from IPFS: %w", downloadErr) - } - if closeErr != nil { - return "", fmt.Errorf("failed to close IPFS node: %w", err) - } - return l2ChainInfoFile, nil -} From 8f946b3d46c4db46b03389c3502ac4ddcae323e5 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Thu, 5 Sep 2024 07:48:00 -0600 Subject: [PATCH 095/489] block_validator: introduce recording-iter-limit --- staker/block_validator.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/staker/block_validator.go b/staker/block_validator.go index e6773a0854..7a7efca846 100644 --- a/staker/block_validator.go +++ b/staker/block_validator.go @@ -106,6 +106,7 @@ type BlockValidatorConfig struct { ValidationServerConfigs []rpcclient.ClientConfig `koanf:"validation-server-configs"` ValidationPoll time.Duration `koanf:"validation-poll" reload:"hot"` PrerecordedBlocks uint64 `koanf:"prerecorded-blocks" reload:"hot"` + RecordingIterLimit uint64 `koanf:"recording-iter-limit"` ForwardBlocks uint64 `koanf:"forward-blocks" reload:"hot"` CurrentModuleRoot string `koanf:"current-module-root"` // TODO(magic) requires reinitialization on hot reload PendingUpgradeModuleRoot string `koanf:"pending-upgrade-module-root"` // TODO(magic) requires StatelessBlockValidator recreation on hot reload @@ -174,6 +175,7 @@ func BlockValidatorConfigAddOptions(prefix string, f *pflag.FlagSet) { f.Uint64(prefix+".forward-blocks", DefaultBlockValidatorConfig.ForwardBlocks, "prepare entries for up to that many blocks ahead of validation (small footprint)") f.Uint64(prefix+".prerecorded-blocks", DefaultBlockValidatorConfig.PrerecordedBlocks, "record that many blocks ahead of validation (larger footprint)") f.String(prefix+".current-module-root", DefaultBlockValidatorConfig.CurrentModuleRoot, "current wasm module root ('current' read from chain, 'latest' from machines/latest dir, or provide hash)") + f.Uint64(prefix+".recording-iter-limit", DefaultBlockValidatorConfig.RecordingIterLimit, "limit on block recordings sent per iteration") f.String(prefix+".pending-upgrade-module-root", DefaultBlockValidatorConfig.PendingUpgradeModuleRoot, "pending upgrade wasm module root to additionally validate (hash, 'latest' or empty)") f.Bool(prefix+".failure-is-fatal", DefaultBlockValidatorConfig.FailureIsFatal, "failing a validation is treated as a fatal error") BlockValidatorDangerousConfigAddOptions(prefix+".dangerous", f) @@ -197,6 +199,7 @@ var DefaultBlockValidatorConfig = BlockValidatorConfig{ FailureIsFatal: true, Dangerous: DefaultBlockValidatorDangerousConfig, MemoryFreeLimit: "default", + RecordingIterLimit: 20, } var TestBlockValidatorConfig = BlockValidatorConfig{ @@ -207,6 +210,7 @@ var TestBlockValidatorConfig = BlockValidatorConfig{ ValidationPoll: 100 * time.Millisecond, ForwardBlocks: 128, PrerecordedBlocks: uint64(2 * runtime.NumCPU()), + RecordingIterLimit: 20, CurrentModuleRoot: "latest", PendingUpgradeModuleRoot: "latest", FailureIsFatal: true, @@ -652,6 +656,10 @@ func (v *BlockValidator) sendNextRecordRequests(ctx context.Context) (bool, erro if recordUntil < pos { return false, nil } + recordUntilLimit := pos + arbutil.MessageIndex(v.config().RecordingIterLimit) + if recordUntil > recordUntilLimit { + recordUntil = recordUntilLimit + } log.Trace("preparing to record", "pos", pos, "until", recordUntil) // prepare could take a long time so we do it without a lock err := v.recorder.PrepareForRecord(ctx, pos, recordUntil) From 0d5293d24de3c995609c063bdc8743622c6d0762 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Fri, 6 Sep 2024 13:43:11 +0530 Subject: [PATCH 096/489] Fix ArbWASM precompile minInitGas() method from reverting --- contracts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts b/contracts index f7894d3a6d..a5c2d54561 160000 --- a/contracts +++ b/contracts @@ -1 +1 @@ -Subproject commit f7894d3a6d4035ba60f51a7f1334f0f2d4f02dce +Subproject commit a5c2d54561c2547e05d6118d6f072882c5fc7a9d From 1468088c84ef1ff1b0fd4c852a3fd3517c852f2c Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Fri, 6 Sep 2024 17:48:57 +0530 Subject: [PATCH 097/489] Remove L1 interface and use *ethclient.Client instead --- arbnode/delayed.go | 5 ++-- arbnode/inbox_reader.go | 5 ++-- arbnode/inbox_tracker.go | 5 ++-- arbnode/node.go | 5 ++-- arbnode/sequencer_inbox.go | 9 ++++--- arbutil/correspondingl1blocknumber.go | 6 ++++- arbutil/transaction_data.go | 5 ++-- arbutil/wait_for_l1.go | 9 ++++--- cmd/nitro/init.go | 4 +-- cmd/pruning/pruning.go | 5 ++-- das/aggregator.go | 4 +-- das/chain_fetch_das.go | 4 +-- das/factory.go | 4 +-- das/rpc_aggregator.go | 4 +-- das/syncing_fallback_storage.go | 3 ++- execution/gethexec/node.go | 3 ++- staker/l1_validator.go | 5 ++-- staker/rollup_watcher.go | 10 ++++++-- staker/staker.go | 3 ++- staker/txbuilder/builder.go | 10 ++++---- staker/validatorwallet/contract.go | 4 +-- staker/validatorwallet/eoa.go | 8 +++--- staker/validatorwallet/noop.go | 8 +++--- system_tests/common_test.go | 32 +++++++++++------------- system_tests/das_test.go | 3 +-- system_tests/full_challenge_impl_test.go | 5 ++-- system_tests/wrap_transaction_test.go | 13 +++++----- util/headerreader/blob_client.go | 6 ++--- util/headerreader/header_reader.go | 7 +++--- 29 files changed, 107 insertions(+), 87 deletions(-) diff --git a/arbnode/delayed.go b/arbnode/delayed.go index c166aa2b90..aba5832146 100644 --- a/arbnode/delayed.go +++ b/arbnode/delayed.go @@ -19,6 +19,7 @@ import ( "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethclient" "github.com/offchainlabs/nitro/arbos/arbostypes" "github.com/offchainlabs/nitro/arbutil" @@ -58,11 +59,11 @@ type DelayedBridge struct { con *bridgegen.IBridge address common.Address fromBlock uint64 - client arbutil.L1Interface + client *ethclient.Client messageProviders map[common.Address]*bridgegen.IDelayedMessageProvider } -func NewDelayedBridge(client arbutil.L1Interface, addr common.Address, fromBlock uint64) (*DelayedBridge, error) { +func NewDelayedBridge(client *ethclient.Client, addr common.Address, fromBlock uint64) (*DelayedBridge, error) { con, err := bridgegen.NewIBridge(addr, client) if err != nil { return nil, err diff --git a/arbnode/inbox_reader.go b/arbnode/inbox_reader.go index fd050b5f67..f8e48bffd6 100644 --- a/arbnode/inbox_reader.go +++ b/arbnode/inbox_reader.go @@ -14,6 +14,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" flag "github.com/spf13/pflag" @@ -93,7 +94,7 @@ type InboxReader struct { delayedBridge *DelayedBridge sequencerInbox *SequencerInbox caughtUpChan chan struct{} - client arbutil.L1Interface + client *ethclient.Client l1Reader *headerreader.HeaderReader // Atomic @@ -101,7 +102,7 @@ type InboxReader struct { lastReadBatchCount atomic.Uint64 } -func NewInboxReader(tracker *InboxTracker, client arbutil.L1Interface, l1Reader *headerreader.HeaderReader, firstMessageBlock *big.Int, delayedBridge *DelayedBridge, sequencerInbox *SequencerInbox, config InboxReaderConfigFetcher) (*InboxReader, error) { +func NewInboxReader(tracker *InboxTracker, client *ethclient.Client, l1Reader *headerreader.HeaderReader, firstMessageBlock *big.Int, delayedBridge *DelayedBridge, sequencerInbox *SequencerInbox, config InboxReaderConfigFetcher) (*InboxReader, error) { err := config().Validate() if err != nil { return nil, err diff --git a/arbnode/inbox_tracker.go b/arbnode/inbox_tracker.go index fe4149c80e..7686fe413f 100644 --- a/arbnode/inbox_tracker.go +++ b/arbnode/inbox_tracker.go @@ -13,6 +13,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" @@ -599,7 +600,7 @@ type multiplexerBackend struct { positionWithinMessage uint64 ctx context.Context - client arbutil.L1Interface + client *ethclient.Client inbox *InboxTracker } @@ -639,7 +640,7 @@ func (b *multiplexerBackend) ReadDelayedInbox(seqNum uint64) (*arbostypes.L1Inco var delayedMessagesMismatch = errors.New("sequencer batch delayed messages missing or different") -func (t *InboxTracker) AddSequencerBatches(ctx context.Context, client arbutil.L1Interface, batches []*SequencerInboxBatch) error { +func (t *InboxTracker) AddSequencerBatches(ctx context.Context, client *ethclient.Client, batches []*SequencerInboxBatch) error { var nextAcc common.Hash var prevbatchmeta BatchMetadata sequenceNumberToKeep := uint64(0) diff --git a/arbnode/node.go b/arbnode/node.go index a9da4ea24b..c5b3bbe071 100644 --- a/arbnode/node.go +++ b/arbnode/node.go @@ -18,6 +18,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" @@ -407,7 +408,7 @@ func createNodeImpl( arbDb ethdb.Database, configFetcher ConfigFetcher, l2Config *params.ChainConfig, - l1client arbutil.L1Interface, + l1client *ethclient.Client, deployInfo *chaininfo.RollupAddresses, txOptsValidator *bind.TransactOpts, txOptsBatchPoster *bind.TransactOpts, @@ -781,7 +782,7 @@ func CreateNode( arbDb ethdb.Database, configFetcher ConfigFetcher, l2Config *params.ChainConfig, - l1client arbutil.L1Interface, + l1client *ethclient.Client, deployInfo *chaininfo.RollupAddresses, txOptsValidator *bind.TransactOpts, txOptsBatchPoster *bind.TransactOpts, diff --git a/arbnode/sequencer_inbox.go b/arbnode/sequencer_inbox.go index 73e52ded53..81146ed46e 100644 --- a/arbnode/sequencer_inbox.go +++ b/arbnode/sequencer_inbox.go @@ -15,6 +15,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" "github.com/offchainlabs/nitro/arbstate/daprovider" "github.com/offchainlabs/nitro/arbutil" @@ -52,10 +53,10 @@ type SequencerInbox struct { con *bridgegen.SequencerInbox address common.Address fromBlock int64 - client arbutil.L1Interface + client *ethclient.Client } -func NewSequencerInbox(client arbutil.L1Interface, addr common.Address, fromBlock int64) (*SequencerInbox, error) { +func NewSequencerInbox(client *ethclient.Client, addr common.Address, fromBlock int64) (*SequencerInbox, error) { con, err := bridgegen.NewSequencerInbox(addr, client) if err != nil { return nil, err @@ -111,7 +112,7 @@ type SequencerInboxBatch struct { serialized []byte // nil if serialization isn't cached yet } -func (m *SequencerInboxBatch) getSequencerData(ctx context.Context, client arbutil.L1Interface) ([]byte, error) { +func (m *SequencerInboxBatch) getSequencerData(ctx context.Context, client *ethclient.Client) ([]byte, error) { switch m.dataLocation { case batchDataTxInput: data, err := arbutil.GetLogEmitterTxData(ctx, client, m.rawLog) @@ -169,7 +170,7 @@ func (m *SequencerInboxBatch) getSequencerData(ctx context.Context, client arbut } } -func (m *SequencerInboxBatch) Serialize(ctx context.Context, client arbutil.L1Interface) ([]byte, error) { +func (m *SequencerInboxBatch) Serialize(ctx context.Context, client *ethclient.Client) ([]byte, error) { if m.serialized != nil { return m.serialized, nil } diff --git a/arbutil/correspondingl1blocknumber.go b/arbutil/correspondingl1blocknumber.go index d654e471e2..c8770e2034 100644 --- a/arbutil/correspondingl1blocknumber.go +++ b/arbutil/correspondingl1blocknumber.go @@ -19,7 +19,11 @@ func ParentHeaderToL1BlockNumber(header *types.Header) uint64 { return header.Number.Uint64() } -func CorrespondingL1BlockNumber(ctx context.Context, client L1Interface, parentBlockNumber uint64) (uint64, error) { +type ParentHeaderFetcher interface { + HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) +} + +func CorrespondingL1BlockNumber(ctx context.Context, client ParentHeaderFetcher, parentBlockNumber uint64) (uint64, error) { // #nosec G115 header, err := client.HeaderByNumber(ctx, big.NewInt(int64(parentBlockNumber))) if err != nil { diff --git a/arbutil/transaction_data.go b/arbutil/transaction_data.go index 8270a628bd..c5728967c7 100644 --- a/arbutil/transaction_data.go +++ b/arbutil/transaction_data.go @@ -8,9 +8,10 @@ import ( "fmt" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" ) -func GetLogTransaction(ctx context.Context, client L1Interface, log types.Log) (*types.Transaction, error) { +func GetLogTransaction(ctx context.Context, client *ethclient.Client, log types.Log) (*types.Transaction, error) { tx, err := client.TransactionInBlock(ctx, log.BlockHash, log.TxIndex) if err != nil { return nil, err @@ -22,7 +23,7 @@ func GetLogTransaction(ctx context.Context, client L1Interface, log types.Log) ( } // GetLogEmitterTxData requires that the tx's data is at least 4 bytes long -func GetLogEmitterTxData(ctx context.Context, client L1Interface, log types.Log) ([]byte, error) { +func GetLogEmitterTxData(ctx context.Context, client *ethclient.Client, log types.Log) ([]byte, error) { tx, err := GetLogTransaction(ctx, client, log) if err != nil { return nil, err diff --git a/arbutil/wait_for_l1.go b/arbutil/wait_for_l1.go index 4b4819156d..49eea6af78 100644 --- a/arbutil/wait_for_l1.go +++ b/arbutil/wait_for_l1.go @@ -14,6 +14,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/rpc" ) @@ -30,7 +31,7 @@ type L1Interface interface { Client() rpc.ClientInterface } -func SendTxAsCall(ctx context.Context, client L1Interface, tx *types.Transaction, from common.Address, blockNum *big.Int, unlimitedGas bool) ([]byte, error) { +func SendTxAsCall(ctx context.Context, client *ethclient.Client, tx *types.Transaction, from common.Address, blockNum *big.Int, unlimitedGas bool) ([]byte, error) { var gas uint64 if unlimitedGas { gas = 0 @@ -50,7 +51,7 @@ func SendTxAsCall(ctx context.Context, client L1Interface, tx *types.Transaction return client.CallContract(ctx, callMsg, blockNum) } -func GetPendingCallBlockNumber(ctx context.Context, client L1Interface) (*big.Int, error) { +func GetPendingCallBlockNumber(ctx context.Context, client *ethclient.Client) (*big.Int, error) { msg := ethereum.CallMsg{ // Pretend to be a contract deployment to execute EVM code without calling a contract. To: nil, @@ -70,7 +71,7 @@ func GetPendingCallBlockNumber(ctx context.Context, client L1Interface) (*big.In return new(big.Int).SetBytes(callRes), nil } -func DetailTxError(ctx context.Context, client L1Interface, tx *types.Transaction, txRes *types.Receipt) error { +func DetailTxError(ctx context.Context, client *ethclient.Client, tx *types.Transaction, txRes *types.Receipt) error { // Re-execute the transaction as a call to get a better error if ctx.Err() != nil { return ctx.Err() @@ -96,7 +97,7 @@ func DetailTxError(ctx context.Context, client L1Interface, tx *types.Transactio return fmt.Errorf("SendTxAsCall got: %w for tx hash %v", err, tx.Hash()) } -func DetailTxErrorUsingCallMsg(ctx context.Context, client L1Interface, txHash common.Hash, txRes *types.Receipt, callMsg ethereum.CallMsg) error { +func DetailTxErrorUsingCallMsg(ctx context.Context, client *ethclient.Client, txHash common.Hash, txRes *types.Receipt, callMsg ethereum.CallMsg) error { // Re-execute the transaction as a call to get a better error if ctx.Err() != nil { return ctx.Err() diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index a8463a7d21..956b878152 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" @@ -37,7 +38,6 @@ import ( "github.com/offchainlabs/nitro/arbnode" "github.com/offchainlabs/nitro/arbos/arbosState" "github.com/offchainlabs/nitro/arbos/arbostypes" - "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/cmd/chaininfo" "github.com/offchainlabs/nitro/cmd/conf" "github.com/offchainlabs/nitro/cmd/ipfshelper" @@ -560,7 +560,7 @@ func rebuildLocalWasm(ctx context.Context, config *gethexec.Config, l2BlockChain return chainDb, l2BlockChain, nil } -func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeConfig, chainId *big.Int, cacheConfig *core.CacheConfig, targetConfig *gethexec.StylusTargetConfig, persistentConfig *conf.PersistentConfig, l1Client arbutil.L1Interface, rollupAddrs chaininfo.RollupAddresses) (ethdb.Database, *core.BlockChain, error) { +func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeConfig, chainId *big.Int, cacheConfig *core.CacheConfig, targetConfig *gethexec.StylusTargetConfig, persistentConfig *conf.PersistentConfig, l1Client *ethclient.Client, rollupAddrs chaininfo.RollupAddresses) (ethdb.Database, *core.BlockChain, error) { if !config.Init.Force { if readOnlyDb, err := stack.OpenDatabaseWithFreezerWithExtraOptions("l2chaindata", 0, 0, config.Persistent.Ancient, "l2chaindata/", true, persistentConfig.Pebble.ExtraOptions("l2chaindata")); err == nil { if chainConfig := gethexec.TryReadStoredChainConfig(readOnlyDb); chainConfig != nil { diff --git a/cmd/pruning/pruning.go b/cmd/pruning/pruning.go index 096bb4b1ae..c5923501f0 100644 --- a/cmd/pruning/pruning.go +++ b/cmd/pruning/pruning.go @@ -15,6 +15,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/pruner" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" @@ -80,7 +81,7 @@ func (r *importantRoots) addHeader(header *types.Header, overwrite bool) error { var hashListRegex = regexp.MustCompile("^(0x)?[0-9a-fA-F]{64}(,(0x)?[0-9a-fA-F]{64})*$") // Finds important roots to retain while proving -func findImportantRoots(ctx context.Context, chainDb ethdb.Database, stack *node.Node, initConfig *conf.InitConfig, cacheConfig *core.CacheConfig, persistentConfig *conf.PersistentConfig, l1Client arbutil.L1Interface, rollupAddrs chaininfo.RollupAddresses, validatorRequired bool) ([]common.Hash, error) { +func findImportantRoots(ctx context.Context, chainDb ethdb.Database, stack *node.Node, initConfig *conf.InitConfig, cacheConfig *core.CacheConfig, persistentConfig *conf.PersistentConfig, l1Client *ethclient.Client, rollupAddrs chaininfo.RollupAddresses, validatorRequired bool) ([]common.Hash, error) { chainConfig := gethexec.TryReadStoredChainConfig(chainDb) if chainConfig == nil { return nil, errors.New("database doesn't have a chain config (was this node initialized?)") @@ -232,7 +233,7 @@ func findImportantRoots(ctx context.Context, chainDb ethdb.Database, stack *node return roots.roots, nil } -func PruneChainDb(ctx context.Context, chainDb ethdb.Database, stack *node.Node, initConfig *conf.InitConfig, cacheConfig *core.CacheConfig, persistentConfig *conf.PersistentConfig, l1Client arbutil.L1Interface, rollupAddrs chaininfo.RollupAddresses, validatorRequired bool) error { +func PruneChainDb(ctx context.Context, chainDb ethdb.Database, stack *node.Node, initConfig *conf.InitConfig, cacheConfig *core.CacheConfig, persistentConfig *conf.PersistentConfig, l1Client *ethclient.Client, rollupAddrs chaininfo.RollupAddresses, validatorRequired bool) error { if cacheConfig.StateScheme == rawdb.PathScheme { return nil } diff --git a/das/aggregator.go b/das/aggregator.go index e8972447ad..87d67cd0c4 100644 --- a/das/aggregator.go +++ b/das/aggregator.go @@ -15,11 +15,11 @@ import ( flag "github.com/spf13/pflag" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/offchainlabs/nitro/arbstate/daprovider" - "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/blsSignatures" "github.com/offchainlabs/nitro/das/dastree" "github.com/offchainlabs/nitro/solgen/go/bridgegen" @@ -114,7 +114,7 @@ func NewAggregator(ctx context.Context, config DataAvailabilityConfig, services func NewAggregatorWithL1Info( config DataAvailabilityConfig, services []ServiceDetails, - l1client arbutil.L1Interface, + l1client *ethclient.Client, seqInboxAddress common.Address, ) (*Aggregator, error) { seqInboxCaller, err := bridgegen.NewSequencerInboxCaller(seqInboxAddress, l1client) diff --git a/das/chain_fetch_das.go b/das/chain_fetch_das.go index 465b54f400..4de6c981cf 100644 --- a/das/chain_fetch_das.go +++ b/das/chain_fetch_das.go @@ -12,8 +12,8 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" - "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/das/dastree" "github.com/offchainlabs/nitro/solgen/go/bridgegen" ) @@ -42,7 +42,7 @@ type KeysetFetcher struct { keysetCache syncedKeysetCache } -func NewKeysetFetcher(l1client arbutil.L1Interface, seqInboxAddr common.Address) (*KeysetFetcher, error) { +func NewKeysetFetcher(l1client *ethclient.Client, seqInboxAddr common.Address) (*KeysetFetcher, error) { seqInbox, err := bridgegen.NewSequencerInbox(seqInboxAddr, l1client) if err != nil { return nil, err diff --git a/das/factory.go b/das/factory.go index 5742a39479..c085f78332 100644 --- a/das/factory.go +++ b/das/factory.go @@ -10,8 +10,8 @@ import ( "math" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethclient" - "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/solgen/go/bridgegen" "github.com/offchainlabs/nitro/util/headerreader" "github.com/offchainlabs/nitro/util/signature" @@ -113,7 +113,7 @@ func CreateBatchPosterDAS( ctx context.Context, config *DataAvailabilityConfig, dataSigner signature.DataSignerFunc, - l1Reader arbutil.L1Interface, + l1Reader *ethclient.Client, sequencerInboxAddr common.Address, ) (DataAvailabilityServiceWriter, DataAvailabilityServiceReader, *KeysetFetcher, *LifecycleManager, error) { if !config.Enable { diff --git a/das/rpc_aggregator.go b/das/rpc_aggregator.go index 24a470be5b..0a1b87e6ca 100644 --- a/das/rpc_aggregator.go +++ b/das/rpc_aggregator.go @@ -21,7 +21,7 @@ import ( "github.com/offchainlabs/nitro/util/signature" "github.com/ethereum/go-ethereum/common" - "github.com/offchainlabs/nitro/arbutil" + "github.com/ethereum/go-ethereum/ethclient" ) type BackendConfig struct { @@ -83,7 +83,7 @@ func NewRPCAggregator(ctx context.Context, config DataAvailabilityConfig, signer return NewAggregator(ctx, config, services) } -func NewRPCAggregatorWithL1Info(config DataAvailabilityConfig, l1client arbutil.L1Interface, seqInboxAddress common.Address, signer signature.DataSignerFunc) (*Aggregator, error) { +func NewRPCAggregatorWithL1Info(config DataAvailabilityConfig, l1client *ethclient.Client, seqInboxAddress common.Address, signer signature.DataSignerFunc) (*Aggregator, error) { services, err := ParseServices(config.RPCAggregator, signer) if err != nil { return nil, err diff --git a/das/syncing_fallback_storage.go b/das/syncing_fallback_storage.go index 8af46b7fc5..3fea8f3119 100644 --- a/das/syncing_fallback_storage.go +++ b/das/syncing_fallback_storage.go @@ -17,6 +17,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" "github.com/offchainlabs/nitro/arbstate/daprovider" "github.com/offchainlabs/nitro/arbutil" @@ -242,7 +243,7 @@ func FindDASDataFromLog( inboxContract *bridgegen.SequencerInbox, deliveredEvent *bridgegen.SequencerInboxSequencerBatchDelivered, inboxAddr common.Address, - l1Client arbutil.L1Interface, + l1Client *ethclient.Client, batchDeliveredLog types.Log) ([]byte, error) { data := []byte{} if deliveredEvent.DataLocation == uint8(batchDataSeparateEvent) { diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index 21c2b4bece..f1ca64082d 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -15,6 +15,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/filters" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" @@ -173,7 +174,7 @@ func CreateExecutionNode( stack *node.Node, chainDB ethdb.Database, l2BlockChain *core.BlockChain, - l1client arbutil.L1Interface, + l1client *ethclient.Client, configFetcher ConfigFetcher, ) (*ExecutionNode, error) { config := configFetcher() diff --git a/staker/l1_validator.go b/staker/l1_validator.go index 6ea9fd8ded..5b0c211324 100644 --- a/staker/l1_validator.go +++ b/staker/l1_validator.go @@ -19,6 +19,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/solgen/go/rollupgen" @@ -45,7 +46,7 @@ type L1Validator struct { rollup *RollupWatcher rollupAddress common.Address validatorUtils *rollupgen.ValidatorUtils - client arbutil.L1Interface + client *ethclient.Client builder *txbuilder.Builder wallet ValidatorWalletInterface callOpts bind.CallOpts @@ -57,7 +58,7 @@ type L1Validator struct { } func NewL1Validator( - client arbutil.L1Interface, + client *ethclient.Client, wallet ValidatorWalletInterface, validatorUtilsAddress common.Address, callOpts bind.CallOpts, diff --git a/staker/rollup_watcher.go b/staker/rollup_watcher.go index 5ef28a49dc..9c51e659f5 100644 --- a/staker/rollup_watcher.go +++ b/staker/rollup_watcher.go @@ -48,12 +48,18 @@ type RollupWatcher struct { *rollupgen.RollupUserLogic address common.Address fromBlock *big.Int - client arbutil.L1Interface + client WatcherL1Interface baseCallOpts bind.CallOpts unSupportedL3Method atomic.Bool } -func NewRollupWatcher(address common.Address, client arbutil.L1Interface, callOpts bind.CallOpts) (*RollupWatcher, error) { +type WatcherL1Interface interface { + bind.ContractBackend + HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) + FilterLogs(ctx context.Context, q ethereum.FilterQuery) ([]types.Log, error) +} + +func NewRollupWatcher(address common.Address, client WatcherL1Interface, callOpts bind.CallOpts) (*RollupWatcher, error) { con, err := rollupgen.NewRollupUserLogic(address, client) if err != nil { return nil, err diff --git a/staker/staker.go b/staker/staker.go index 6e93d27311..1d49063a89 100644 --- a/staker/staker.go +++ b/staker/staker.go @@ -15,6 +15,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rpc" @@ -281,7 +282,7 @@ type ValidatorWalletInterface interface { TxSenderAddress() *common.Address RollupAddress() common.Address ChallengeManagerAddress() common.Address - L1Client() arbutil.L1Interface + L1Client() *ethclient.Client TestTransactions(context.Context, []*types.Transaction) error ExecuteTransactions(context.Context, *txbuilder.Builder, common.Address) (*types.Transaction, error) TimeoutChallenges(context.Context, []uint64) (*types.Transaction, error) diff --git a/staker/txbuilder/builder.go b/staker/txbuilder/builder.go index 9a5e9df2b5..fa40c6a816 100644 --- a/staker/txbuilder/builder.go +++ b/staker/txbuilder/builder.go @@ -12,13 +12,13 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "github.com/offchainlabs/nitro/arbutil" + "github.com/ethereum/go-ethereum/ethclient" ) type ValidatorWalletInterface interface { // Address must be able to be called concurrently with other functions Address() *common.Address - L1Client() arbutil.L1Interface + L1Client() *ethclient.Client TestTransactions(context.Context, []*types.Transaction) error ExecuteTransactions(context.Context, *Builder, common.Address) (*types.Transaction, error) AuthIfEoa() *bind.TransactOpts @@ -30,7 +30,7 @@ type ValidatorWalletInterface interface { // This inherits from an eth client so it can be used as an L1Interface, // where it transparently intercepts calls to SendTransaction and queues them for the next batch. type Builder struct { - arbutil.L1Interface + *ethclient.Client transactions []*types.Transaction builderAuth *bind.TransactOpts isAuthFake bool @@ -55,7 +55,7 @@ func NewBuilder(wallet ValidatorWalletInterface) (*Builder, error) { return &Builder{ builderAuth: builderAuth, wallet: wallet, - L1Interface: wallet.L1Client(), + Client: wallet.L1Client(), isAuthFake: isAuthFake, }, nil } @@ -70,7 +70,7 @@ func (b *Builder) ClearTransactions() { func (b *Builder) EstimateGas(ctx context.Context, call ethereum.CallMsg) (gas uint64, err error) { if len(b.transactions) == 0 && !b.isAuthFake { - return b.L1Interface.EstimateGas(ctx, call) + return b.Client.EstimateGas(ctx, call) } return 0, nil } diff --git a/staker/validatorwallet/contract.go b/staker/validatorwallet/contract.go index 6346029c3a..3202d58569 100644 --- a/staker/validatorwallet/contract.go +++ b/staker/validatorwallet/contract.go @@ -16,10 +16,10 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/offchainlabs/nitro/arbnode/dataposter" - "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/solgen/go/rollupgen" "github.com/offchainlabs/nitro/staker/txbuilder" "github.com/offchainlabs/nitro/util/arbmath" @@ -384,7 +384,7 @@ func (v *Contract) TimeoutChallenges(ctx context.Context, challenges []uint64) ( return v.dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), *v.Address(), data, gas, auth.Value) } -func (v *Contract) L1Client() arbutil.L1Interface { +func (v *Contract) L1Client() *ethclient.Client { return v.l1Reader.Client() } diff --git a/staker/validatorwallet/eoa.go b/staker/validatorwallet/eoa.go index 3ae305b36c..7c7f472579 100644 --- a/staker/validatorwallet/eoa.go +++ b/staker/validatorwallet/eoa.go @@ -10,8 +10,8 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" "github.com/offchainlabs/nitro/arbnode/dataposter" - "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/solgen/go/challengegen" "github.com/offchainlabs/nitro/solgen/go/rollupgen" "github.com/offchainlabs/nitro/staker/txbuilder" @@ -19,7 +19,7 @@ import ( type EOA struct { auth *bind.TransactOpts - client arbutil.L1Interface + client *ethclient.Client rollupAddress common.Address challengeManager *challengegen.ChallengeManager challengeManagerAddress common.Address @@ -27,7 +27,7 @@ type EOA struct { getExtraGas func() uint64 } -func NewEOA(dataPoster *dataposter.DataPoster, rollupAddress common.Address, l1Client arbutil.L1Interface, getExtraGas func() uint64) (*EOA, error) { +func NewEOA(dataPoster *dataposter.DataPoster, rollupAddress common.Address, l1Client *ethclient.Client, getExtraGas func() uint64) (*EOA, error) { return &EOA{ auth: dataPoster.Auth(), client: l1Client, @@ -63,7 +63,7 @@ func (w *EOA) TxSenderAddress() *common.Address { return &w.auth.From } -func (w *EOA) L1Client() arbutil.L1Interface { +func (w *EOA) L1Client() *ethclient.Client { return w.client } diff --git a/staker/validatorwallet/noop.go b/staker/validatorwallet/noop.go index b050ebe861..fec39ac2b1 100644 --- a/staker/validatorwallet/noop.go +++ b/staker/validatorwallet/noop.go @@ -10,18 +10,18 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" "github.com/offchainlabs/nitro/arbnode/dataposter" - "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/staker/txbuilder" ) // NoOp validator wallet is used for watchtower mode. type NoOp struct { - l1Client arbutil.L1Interface + l1Client *ethclient.Client rollupAddress common.Address } -func NewNoOp(l1Client arbutil.L1Interface, rollupAddress common.Address) *NoOp { +func NewNoOp(l1Client *ethclient.Client, rollupAddress common.Address) *NoOp { return &NoOp{ l1Client: l1Client, rollupAddress: rollupAddress, @@ -46,7 +46,7 @@ func (*NoOp) TimeoutChallenges(ctx context.Context, challenges []uint64) (*types return nil, errors.New("no op validator wallet cannot timeout challenges") } -func (n *NoOp) L1Client() arbutil.L1Interface { return n.l1Client } +func (n *NoOp) L1Client() *ethclient.Client { return n.l1Client } func (n *NoOp) RollupAddress() common.Address { return n.rollupAddress } diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 457dae0910..2b2f1c6407 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -69,7 +69,6 @@ import ( "github.com/ethereum/go-ethereum/rpc" "github.com/offchainlabs/nitro/arbnode" - "github.com/offchainlabs/nitro/arbutil" _ "github.com/offchainlabs/nitro/execution/nodeInterface" "github.com/offchainlabs/nitro/solgen/go/bridgegen" "github.com/offchainlabs/nitro/solgen/go/mocksgen" @@ -83,7 +82,6 @@ import ( ) type info = *BlockchainTestInfo -type client = arbutil.L1Interface type SecondNodeParams struct { nodeConfig *arbnode.Config @@ -540,7 +538,7 @@ func (b *NodeBuilder) BridgeBalance(t *testing.T, account string, amount *big.In return BridgeBalance(t, account, amount, b.L1Info, b.L2Info, b.L1.Client, b.L2.Client, b.ctx) } -func SendWaitTestTransactions(t *testing.T, ctx context.Context, client client, txs []*types.Transaction) { +func SendWaitTestTransactions(t *testing.T, ctx context.Context, client *ethclient.Client, txs []*types.Transaction) { t.Helper() for _, tx := range txs { Require(t, client.SendTransaction(ctx, tx)) @@ -552,14 +550,14 @@ func SendWaitTestTransactions(t *testing.T, ctx context.Context, client client, } func TransferBalance( - t *testing.T, from, to string, amount *big.Int, l2info info, client client, ctx context.Context, + t *testing.T, from, to string, amount *big.Int, l2info info, client *ethclient.Client, ctx context.Context, ) (*types.Transaction, *types.Receipt) { t.Helper() return TransferBalanceTo(t, from, l2info.GetAddress(to), amount, l2info, client, ctx) } func TransferBalanceTo( - t *testing.T, from string, to common.Address, amount *big.Int, l2info info, client client, ctx context.Context, + t *testing.T, from string, to common.Address, amount *big.Int, l2info info, client *ethclient.Client, ctx context.Context, ) (*types.Transaction, *types.Receipt) { t.Helper() tx := l2info.PrepareTxTo(from, &to, l2info.TransferGas, amount, nil) @@ -572,7 +570,7 @@ func TransferBalanceTo( // if l2client is not nil - will wait until balance appears in l2 func BridgeBalance( - t *testing.T, account string, amount *big.Int, l1info info, l2info info, l1client client, l2client client, ctx context.Context, + t *testing.T, account string, amount *big.Int, l1info info, l2info info, l1client *ethclient.Client, l2client *ethclient.Client, ctx context.Context, ) (*types.Transaction, *types.Receipt) { t.Helper() @@ -632,8 +630,8 @@ func SendSignedTxesInBatchViaL1( t *testing.T, ctx context.Context, l1info *BlockchainTestInfo, - l1client arbutil.L1Interface, - l2client arbutil.L1Interface, + l1client *ethclient.Client, + l2client *ethclient.Client, delayedTxes types.Transactions, ) types.Receipts { delayedInboxContract, err := bridgegen.NewInbox(l1info.GetAddress("Inbox"), l1client) @@ -683,8 +681,8 @@ func SendSignedTxViaL1( t *testing.T, ctx context.Context, l1info *BlockchainTestInfo, - l1client arbutil.L1Interface, - l2client arbutil.L1Interface, + l1client *ethclient.Client, + l2client *ethclient.Client, delayedTx *types.Transaction, ) *types.Receipt { delayedInboxContract, err := bridgegen.NewInbox(l1info.GetAddress("Inbox"), l1client) @@ -714,8 +712,8 @@ func SendUnsignedTxViaL1( t *testing.T, ctx context.Context, l1info *BlockchainTestInfo, - l1client arbutil.L1Interface, - l2client arbutil.L1Interface, + l1client *ethclient.Client, + l2client *ethclient.Client, templateTx *types.Transaction, ) *types.Receipt { delayedInboxContract, err := bridgegen.NewInbox(l1info.GetAddress("Inbox"), l1client) @@ -761,13 +759,13 @@ func SendUnsignedTxViaL1( return receipt } -func GetBaseFee(t *testing.T, client client, ctx context.Context) *big.Int { +func GetBaseFee(t *testing.T, client *ethclient.Client, ctx context.Context) *big.Int { header, err := client.HeaderByNumber(ctx, nil) Require(t, err) return header.BaseFee } -func GetBaseFeeAt(t *testing.T, client client, ctx context.Context, blockNum *big.Int) *big.Int { +func GetBaseFeeAt(t *testing.T, client *ethclient.Client, ctx context.Context, blockNum *big.Int) *big.Int { header, err := client.HeaderByNumber(ctx, blockNum) Require(t, err) return header.BaseFee @@ -989,7 +987,7 @@ func createTestL1BlockChain(t *testing.T, l1info info) (info, *ethclient.Client, return l1info, l1Client, l1backend, stack } -func getInitMessage(ctx context.Context, t *testing.T, l1client client, addresses *chaininfo.RollupAddresses) *arbostypes.ParsedInitMessage { +func getInitMessage(ctx context.Context, t *testing.T, l1client *ethclient.Client, addresses *chaininfo.RollupAddresses) *arbostypes.ParsedInitMessage { bridge, err := arbnode.NewDelayedBridge(l1client, addresses.Bridge, addresses.DeployedAt) Require(t, err) deployedAtBig := arbmath.UintToBig(addresses.DeployedAt) @@ -1005,7 +1003,7 @@ func getInitMessage(ctx context.Context, t *testing.T, l1client client, addresse } func DeployOnTestL1( - t *testing.T, ctx context.Context, l1info info, l1client client, chainConfig *params.ChainConfig, wasmModuleRoot common.Hash, prodConfirmPeriodBlocks bool, + t *testing.T, ctx context.Context, l1info info, l1client *ethclient.Client, chainConfig *params.ChainConfig, wasmModuleRoot common.Hash, prodConfirmPeriodBlocks bool, ) (*chaininfo.RollupAddresses, *arbostypes.ParsedInitMessage) { l1info.GenerateAccount("RollupOwner") l1info.GenerateAccount("Sequencer") @@ -1223,7 +1221,7 @@ func authorizeDASKeyset( ctx context.Context, dasSignerKey *blsSignatures.PublicKey, l1info info, - l1client arbutil.L1Interface, + l1client *ethclient.Client, ) { if dasSignerKey == nil { return diff --git a/system_tests/das_test.go b/system_tests/das_test.go index 9f4d153b6f..69872b3e62 100644 --- a/system_tests/das_test.go +++ b/system_tests/das_test.go @@ -22,7 +22,6 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/offchainlabs/nitro/arbnode" - "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/blsSignatures" "github.com/offchainlabs/nitro/cmd/genericconf" "github.com/offchainlabs/nitro/das" @@ -37,7 +36,7 @@ func startLocalDASServer( t *testing.T, ctx context.Context, dataDir string, - l1client arbutil.L1Interface, + l1client *ethclient.Client, seqInboxAddress common.Address, ) (*http.Server, *blsSignatures.PublicKey, das.BackendConfig, *das.RestfulDasServer, string) { keyDir := t.TempDir() diff --git a/system_tests/full_challenge_impl_test.go b/system_tests/full_challenge_impl_test.go index ddc229074c..bf30c928d8 100644 --- a/system_tests/full_challenge_impl_test.go +++ b/system_tests/full_challenge_impl_test.go @@ -27,7 +27,6 @@ import ( "github.com/offchainlabs/nitro/arbnode" "github.com/offchainlabs/nitro/arbos" "github.com/offchainlabs/nitro/arbstate" - "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/solgen/go/challengegen" "github.com/offchainlabs/nitro/solgen/go/mocksgen" "github.com/offchainlabs/nitro/solgen/go/ospgen" @@ -178,7 +177,7 @@ func makeBatch(t *testing.T, l2Node *arbnode.Node, l2Info *BlockchainTestInfo, b Require(t, err, "failed to get batch metadata after adding batch:") } -func confirmLatestBlock(ctx context.Context, t *testing.T, l1Info *BlockchainTestInfo, backend arbutil.L1Interface) { +func confirmLatestBlock(ctx context.Context, t *testing.T, l1Info *BlockchainTestInfo, backend *ethclient.Client) { t.Helper() // With SimulatedBeacon running in on-demand block production mode, the // finalized block is considered to be be the nearest multiple of 32 less @@ -190,7 +189,7 @@ func confirmLatestBlock(ctx context.Context, t *testing.T, l1Info *BlockchainTes } } -func setupSequencerInboxStub(ctx context.Context, t *testing.T, l1Info *BlockchainTestInfo, l1Client arbutil.L1Interface, chainConfig *params.ChainConfig) (common.Address, *mocksgen.SequencerInboxStub, common.Address) { +func setupSequencerInboxStub(ctx context.Context, t *testing.T, l1Info *BlockchainTestInfo, l1Client *ethclient.Client, chainConfig *params.ChainConfig) (common.Address, *mocksgen.SequencerInboxStub, common.Address) { txOpts := l1Info.GetDefaultTransactOpts("deployer", ctx) bridgeAddr, tx, bridge, err := mocksgen.DeployBridgeUnproxied(&txOpts, l1Client) Require(t, err) diff --git a/system_tests/wrap_transaction_test.go b/system_tests/wrap_transaction_test.go index bd561ad5e5..36052fb2db 100644 --- a/system_tests/wrap_transaction_test.go +++ b/system_tests/wrap_transaction_test.go @@ -15,6 +15,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" "github.com/offchainlabs/nitro/arbutil" @@ -22,7 +23,7 @@ import ( "github.com/offchainlabs/nitro/util/headerreader" ) -func GetPendingBlockNumber(ctx context.Context, client arbutil.L1Interface) (*big.Int, error) { +func GetPendingBlockNumber(ctx context.Context, client *ethclient.Client) (*big.Int, error) { // Attempt to get the block number from ArbSys, if it exists arbSys, err := precompilesgen.NewArbSys(common.BigToAddress(big.NewInt(100)), client) if err != nil { @@ -37,7 +38,7 @@ func GetPendingBlockNumber(ctx context.Context, client arbutil.L1Interface) (*bi } // Will wait until txhash is in the blockchain and return its receipt -func WaitForTx(ctxinput context.Context, client arbutil.L1Interface, txhash common.Hash, timeout time.Duration) (*types.Receipt, error) { +func WaitForTx(ctxinput context.Context, client *ethclient.Client, txhash common.Hash, timeout time.Duration) (*types.Receipt, error) { ctx, cancel := context.WithTimeout(ctxinput, timeout) defer cancel() @@ -75,11 +76,11 @@ func WaitForTx(ctxinput context.Context, client arbutil.L1Interface, txhash comm } } -func EnsureTxSucceeded(ctx context.Context, client arbutil.L1Interface, tx *types.Transaction) (*types.Receipt, error) { +func EnsureTxSucceeded(ctx context.Context, client *ethclient.Client, tx *types.Transaction) (*types.Receipt, error) { return EnsureTxSucceededWithTimeout(ctx, client, tx, time.Second*5) } -func EnsureTxSucceededWithTimeout(ctx context.Context, client arbutil.L1Interface, tx *types.Transaction, timeout time.Duration) (*types.Receipt, error) { +func EnsureTxSucceededWithTimeout(ctx context.Context, client *ethclient.Client, tx *types.Transaction, timeout time.Duration) (*types.Receipt, error) { receipt, err := WaitForTx(ctx, client, tx.Hash(), timeout) if err != nil { return nil, fmt.Errorf("waitFoxTx (tx=%s) got: %w", tx.Hash().Hex(), err) @@ -103,12 +104,12 @@ func EnsureTxSucceededWithTimeout(ctx context.Context, client arbutil.L1Interfac return receipt, arbutil.DetailTxError(ctx, client, tx, receipt) } -func EnsureTxFailed(t *testing.T, ctx context.Context, client arbutil.L1Interface, tx *types.Transaction) *types.Receipt { +func EnsureTxFailed(t *testing.T, ctx context.Context, client *ethclient.Client, tx *types.Transaction) *types.Receipt { t.Helper() return EnsureTxFailedWithTimeout(t, ctx, client, tx, time.Second*5) } -func EnsureTxFailedWithTimeout(t *testing.T, ctx context.Context, client arbutil.L1Interface, tx *types.Transaction, timeout time.Duration) *types.Receipt { +func EnsureTxFailedWithTimeout(t *testing.T, ctx context.Context, client *ethclient.Client, tx *types.Transaction, timeout time.Duration) *types.Receipt { t.Helper() receipt, err := WaitForTx(ctx, client, tx.Hash(), timeout) Require(t, err) diff --git a/util/headerreader/blob_client.go b/util/headerreader/blob_client.go index 2b47a940c3..945fc27691 100644 --- a/util/headerreader/blob_client.go +++ b/util/headerreader/blob_client.go @@ -18,8 +18,8 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto/kzg4844" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" - "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/util/blobs" "github.com/offchainlabs/nitro/util/jsonapi" "github.com/offchainlabs/nitro/util/pretty" @@ -28,7 +28,7 @@ import ( ) type BlobClient struct { - ec arbutil.L1Interface + ec *ethclient.Client beaconUrl *url.URL secondaryBeaconUrl *url.URL httpClient *http.Client @@ -63,7 +63,7 @@ func BlobClientAddOptions(prefix string, f *pflag.FlagSet) { f.String(prefix+".authorization", DefaultBlobClientConfig.Authorization, "Value to send with the HTTP Authorization: header for Beacon REST requests, must include both scheme and scheme parameters") } -func NewBlobClient(config BlobClientConfig, ec arbutil.L1Interface) (*BlobClient, error) { +func NewBlobClient(config BlobClientConfig, ec *ethclient.Client) (*BlobClient, error) { beaconUrl, err := url.Parse(config.BeaconUrl) if err != nil { return nil, fmt.Errorf("failed to parse beacon chain URL: %w", err) diff --git a/util/headerreader/header_reader.go b/util/headerreader/header_reader.go index c8041dc871..98f778dee8 100644 --- a/util/headerreader/header_reader.go +++ b/util/headerreader/header_reader.go @@ -16,6 +16,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" "github.com/offchainlabs/nitro/arbutil" @@ -33,7 +34,7 @@ type ArbSysInterface interface { type HeaderReader struct { stopwaiter.StopWaiter config ConfigFetcher - client arbutil.L1Interface + client *ethclient.Client isParentChainArbitrum bool arbSys ArbSysInterface @@ -120,7 +121,7 @@ var TestConfig = Config{ }, } -func New(ctx context.Context, client arbutil.L1Interface, config ConfigFetcher, arbSysPrecompile ArbSysInterface) (*HeaderReader, error) { +func New(ctx context.Context, client *ethclient.Client, config ConfigFetcher, arbSysPrecompile ArbSysInterface) (*HeaderReader, error) { isParentChainArbitrum := false var arbSys ArbSysInterface if arbSysPrecompile != nil { @@ -522,7 +523,7 @@ func (s *HeaderReader) LatestFinalizedBlockNr(ctx context.Context) (uint64, erro return header.Number.Uint64(), nil } -func (s *HeaderReader) Client() arbutil.L1Interface { +func (s *HeaderReader) Client() *ethclient.Client { return s.client } From fab821e384e51d744da8d1212fef7f1571c03072 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 14 Aug 2024 14:19:24 -0300 Subject: [PATCH 098/489] First step in lru to lru_mem migration in stylus cache --- arbitrator/Cargo.lock | 10 ++++++++++ arbitrator/Cargo.toml | 2 -- arbitrator/stylus/Cargo.toml | 3 ++- arbitrator/stylus/src/cache.rs | 35 +++++++++++++++++++++++++--------- 4 files changed, 38 insertions(+), 12 deletions(-) diff --git a/arbitrator/Cargo.lock b/arbitrator/Cargo.lock index 79a9117a31..a46e000ce2 100644 --- a/arbitrator/Cargo.lock +++ b/arbitrator/Cargo.lock @@ -1342,6 +1342,15 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "lru-mem" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf5c8c26d903a41c80d4cc171940a57a4d1bc51139ebd6aad87e2f9ae3774780" +dependencies = [ + "hashbrown 0.14.5", +] + [[package]] name = "mach" version = "0.3.2" @@ -2277,6 +2286,7 @@ dependencies = [ "lazy_static", "libc", "lru", + "lru-mem", "num-bigint", "parking_lot", "prover", diff --git a/arbitrator/Cargo.toml b/arbitrator/Cargo.toml index 94ca08b0b5..eaafb6e439 100644 --- a/arbitrator/Cargo.toml +++ b/arbitrator/Cargo.toml @@ -24,9 +24,7 @@ repository = "https://github.com/OffchainLabs/nitro.git" rust-version = "1.67" [workspace.dependencies] -cfg-if = "1.0.0" lazy_static = "1.4.0" -lru = "0.12.3" num_enum = { version = "0.7.2", default-features = false } ruint2 = "1.9.0" wasmparser = "0.121" diff --git a/arbitrator/stylus/Cargo.toml b/arbitrator/stylus/Cargo.toml index 4717bd631a..45c06fc881 100644 --- a/arbitrator/stylus/Cargo.toml +++ b/arbitrator/stylus/Cargo.toml @@ -21,11 +21,12 @@ thiserror = "1.0.33" bincode = "1.3.3" lazy_static.workspace = true libc = "0.2.108" -lru.workspace = true eyre = "0.6.5" rand = "0.8.5" fnv = "1.0.7" hex = "0.4.3" +lru-mem = "0.3.0" +lru = "0.12.4" [dev-dependencies] num-bigint = "0.4.4" diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index fa38d45419..338dcb9dd0 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -4,16 +4,16 @@ use arbutil::Bytes32; use eyre::Result; use lazy_static::lazy_static; -use lru::LruCache; +use lru_mem::{HeapSize, LruCache}; use parking_lot::Mutex; use prover::programs::config::CompileConfig; -use std::{collections::HashMap, num::NonZeroUsize}; +use std::collections::HashMap; use wasmer::{Engine, Module, Store}; use crate::target_cache::target_native; lazy_static! { - static ref INIT_CACHE: Mutex = Mutex::new(InitCache::new(256)); + static ref INIT_CACHE: Mutex = Mutex::new(InitCache::new(256 * 100_000)); } macro_rules! cache { @@ -44,6 +44,12 @@ impl CacheKey { } } +impl HeapSize for CacheKey { + fn heap_size(&self) -> usize { + 0 + } +} + #[derive(Clone)] struct CacheItem { module: Module, @@ -60,6 +66,13 @@ impl CacheItem { } } +impl HeapSize for CacheItem { + // TODO: implement heap_size + fn heap_size(&self) -> usize { + 100_000 + } +} + impl InitCache { // current implementation only has one tag that stores to the long_term // future implementations might have more, but 0 is a reserved tag @@ -69,14 +82,15 @@ impl InitCache { fn new(size: usize) -> Self { Self { long_term: HashMap::new(), - lru: LruCache::new(NonZeroUsize::new(size).unwrap()), + lru: LruCache::new(size), } } + // TODO: Check if needs to shrink capacity pub fn set_lru_size(size: u32) { cache!() .lru - .resize(NonZeroUsize::new(size.try_into().unwrap()).unwrap()) + .set_max_size(size.try_into().unwrap()) } /// Retrieves a cached value, updating items as necessary. @@ -116,7 +130,7 @@ impl InitCache { if long_term_tag == Self::ARBOS_TAG { cache.long_term.insert(key, item.clone()); } else { - cache.lru.promote(&key) + cache.lru.touch(&key) } return Ok(item.data()); } @@ -129,7 +143,8 @@ impl InitCache { let data = item.data(); let mut cache = cache!(); if long_term_tag != Self::ARBOS_TAG { - cache.lru.put(key, item); + // TODO: handle result + let _ = cache.lru.insert(key, item); } else { cache.long_term.insert(key, item); } @@ -144,7 +159,8 @@ impl InitCache { let key = CacheKey::new(module_hash, version, debug); let mut cache = cache!(); if let Some(item) = cache.long_term.remove(&key) { - cache.lru.put(key, item); + // TODO: handle result + let _ = cache.lru.insert(key, item); } } @@ -155,7 +171,8 @@ impl InitCache { let mut cache = cache!(); let cache = &mut *cache; for (key, item) in cache.long_term.drain() { - cache.lru.put(key, item); // not all will fit, just a heuristic + // TODO: handle result + let _ = cache.lru.insert(key, item); // not all will fit, just a heuristic } } } From e4c9406c1293a3c9e119285e9ae4e8a69fd5b7a1 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 14 Aug 2024 15:22:18 -0300 Subject: [PATCH 099/489] stylus-lru-cache-size instead of stylus-lru-cache flag --- arbnode/inbox_test.go | 2 +- execution/gethexec/blockchain.go | 6 +++--- execution/gethexec/node.go | 2 +- system_tests/common_test.go | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/arbnode/inbox_test.go b/arbnode/inbox_test.go index d579b7c278..e0c97bb871 100644 --- a/arbnode/inbox_test.go +++ b/arbnode/inbox_test.go @@ -74,7 +74,7 @@ func NewTransactionStreamerForTest(t *testing.T, ownerAddress common.Address) (* } stylusTargetConfig := &gethexec.DefaultStylusTargetConfig Require(t, stylusTargetConfig.Validate()) // pre-processes config (i.a. parses wasmTargets) - if err := execEngine.Initialize(gethexec.DefaultCachingConfig.StylusLRUCache, stylusTargetConfig); err != nil { + if err := execEngine.Initialize(gethexec.DefaultCachingConfig.StylusLRUCacheSize, &gethexec.DefaultStylusTargetConfig); err != nil { Fail(t, err) } execSeq := &execClientWrapper{execEngine, t} diff --git a/execution/gethexec/blockchain.go b/execution/gethexec/blockchain.go index 996b87a9e6..54934dbdf3 100644 --- a/execution/gethexec/blockchain.go +++ b/execution/gethexec/blockchain.go @@ -37,7 +37,7 @@ type CachingConfig struct { SnapshotRestoreGasLimit uint64 `koanf:"snapshot-restore-gas-limit"` MaxNumberOfBlocksToSkipStateSaving uint32 `koanf:"max-number-of-blocks-to-skip-state-saving"` MaxAmountOfGasToSkipStateSaving uint64 `koanf:"max-amount-of-gas-to-skip-state-saving"` - StylusLRUCache uint32 `koanf:"stylus-lru-cache"` + StylusLRUCacheSize uint32 `koanf:"stylus-lru-cache-size"` StateScheme string `koanf:"state-scheme"` StateHistory uint64 `koanf:"state-history"` } @@ -54,7 +54,7 @@ func CachingConfigAddOptions(prefix string, f *flag.FlagSet) { f.Uint64(prefix+".snapshot-restore-gas-limit", DefaultCachingConfig.SnapshotRestoreGasLimit, "maximum gas rolled back to recover snapshot") f.Uint32(prefix+".max-number-of-blocks-to-skip-state-saving", DefaultCachingConfig.MaxNumberOfBlocksToSkipStateSaving, "maximum number of blocks to skip state saving to persistent storage (archive node only) -- warning: this option seems to cause issues") f.Uint64(prefix+".max-amount-of-gas-to-skip-state-saving", DefaultCachingConfig.MaxAmountOfGasToSkipStateSaving, "maximum amount of gas in blocks to skip saving state to Persistent storage (archive node only) -- warning: this option seems to cause issues") - f.Uint32(prefix+".stylus-lru-cache", DefaultCachingConfig.StylusLRUCache, "initialized stylus programs to keep in LRU cache") + f.Uint32(prefix+".stylus-lru-cache-size", DefaultCachingConfig.StylusLRUCacheSize, "capacity, in bytes, of the LRU cache that keeps initialized stylus programs") f.String(prefix+".state-scheme", DefaultCachingConfig.StateScheme, "scheme to use for state trie storage (hash, path)") f.Uint64(prefix+".state-history", DefaultCachingConfig.StateHistory, "number of recent blocks to retain state history for (path state-scheme only)") } @@ -75,7 +75,7 @@ var DefaultCachingConfig = CachingConfig{ SnapshotRestoreGasLimit: 300_000_000_000, MaxNumberOfBlocksToSkipStateSaving: 0, MaxAmountOfGasToSkipStateSaving: 0, - StylusLRUCache: 256, + StylusLRUCacheSize: 256 * 10 * 1024, StateScheme: rawdb.HashScheme, StateHistory: getStateHistory(DefaultSequencerConfig.MaxBlockSpeed), } diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index 21c2b4bece..93bb254ed8 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -308,7 +308,7 @@ func (n *ExecutionNode) MarkFeedStart(to arbutil.MessageIndex) { func (n *ExecutionNode) Initialize(ctx context.Context) error { config := n.ConfigFetcher() - err := n.ExecEngine.Initialize(config.Caching.StylusLRUCache, &config.StylusTarget) + err := n.ExecEngine.Initialize(config.Caching.StylusLRUCacheSize, &config.StylusTarget) if err != nil { return fmt.Errorf("error initializing execution engine: %w", err) } diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 457dae0910..363f1050fb 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -166,7 +166,7 @@ var TestCachingConfig = gethexec.CachingConfig{ SnapshotRestoreGasLimit: 300_000_000_000, MaxNumberOfBlocksToSkipStateSaving: 0, MaxAmountOfGasToSkipStateSaving: 0, - StylusLRUCache: 0, + StylusLRUCacheSize: 0, StateScheme: env.GetTestStateScheme(), } From a3b2dd7b6c98163b57ac5c17677785ed623ee488 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 15 Aug 2024 12:32:05 -0300 Subject: [PATCH 100/489] Uses asm size estimate to define amount of heap memory used by a stylus LRU cache entry --- arbitrator/stylus/src/cache.rs | 13 +++++++------ arbitrator/stylus/src/lib.rs | 5 ++++- arbitrator/stylus/src/native.rs | 3 ++- arbos/programs/native.go | 12 +++++++----- arbos/programs/programs.go | 12 +++++++++--- arbos/programs/wasm.go | 3 ++- 6 files changed, 31 insertions(+), 17 deletions(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index 338dcb9dd0..300f0de35a 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -13,7 +13,7 @@ use wasmer::{Engine, Module, Store}; use crate::target_cache::target_native; lazy_static! { - static ref INIT_CACHE: Mutex = Mutex::new(InitCache::new(256 * 100_000)); + static ref INIT_CACHE: Mutex = Mutex::new(InitCache::new(256 * 10 * 1024)); } macro_rules! cache { @@ -54,11 +54,12 @@ impl HeapSize for CacheKey { struct CacheItem { module: Module, engine: Engine, + asm_size_estimate: u32, } impl CacheItem { - fn new(module: Module, engine: Engine) -> Self { - Self { module, engine } + fn new(module: Module, engine: Engine, asm_size_estimate: u32) -> Self { + Self { module, engine, asm_size_estimate } } fn data(&self) -> (Module, Store) { @@ -67,9 +68,8 @@ impl CacheItem { } impl HeapSize for CacheItem { - // TODO: implement heap_size fn heap_size(&self) -> usize { - 100_000 + return self.asm_size_estimate.try_into().unwrap(); } } @@ -115,6 +115,7 @@ impl InitCache { pub fn insert( module_hash: Bytes32, module: &[u8], + asm_size_estimate: u32, version: u16, long_term_tag: u32, debug: bool, @@ -139,7 +140,7 @@ impl InitCache { let engine = CompileConfig::version(version, debug).engine(target_native()); let module = unsafe { Module::deserialize_unchecked(&engine, module)? }; - let item = CacheItem::new(module, engine); + let item = CacheItem::new(module, engine, asm_size_estimate); let data = item.data(); let mut cache = cache!(); if long_term_tag != Self::ARBOS_TAG { diff --git a/arbitrator/stylus/src/lib.rs b/arbitrator/stylus/src/lib.rs index a252b60a01..979736054c 100644 --- a/arbitrator/stylus/src/lib.rs +++ b/arbitrator/stylus/src/lib.rs @@ -256,6 +256,7 @@ pub unsafe extern "C" fn stylus_target_set( #[no_mangle] pub unsafe extern "C" fn stylus_call( module: GoSliceData, + asm_size_estimate: u32, calldata: GoSliceData, config: StylusConfig, req_handler: NativeRequestHandler, @@ -276,6 +277,7 @@ pub unsafe extern "C" fn stylus_call( let instance = unsafe { NativeInstance::deserialize_cached( module, + asm_size_estimate, config.version, evm_api, evm_data, @@ -317,11 +319,12 @@ pub extern "C" fn stylus_cache_lru_resize(size: u32) { pub unsafe extern "C" fn stylus_cache_module( module: GoSliceData, module_hash: Bytes32, + asm_size_estimate: u32, version: u16, arbos_tag: u32, debug: bool, ) { - if let Err(error) = InitCache::insert(module_hash, module.slice(), version, arbos_tag, debug) { + if let Err(error) = InitCache::insert(module_hash, module.slice(), asm_size_estimate, version, arbos_tag, debug) { panic!("tried to cache invalid asm!: {error}"); } } diff --git a/arbitrator/stylus/src/native.rs b/arbitrator/stylus/src/native.rs index 7a82314fbc..d150d71ccc 100644 --- a/arbitrator/stylus/src/native.rs +++ b/arbitrator/stylus/src/native.rs @@ -112,6 +112,7 @@ impl> NativeInstance { /// `module` must represent a valid module. pub unsafe fn deserialize_cached( module: &[u8], + asm_size_estimate: u32, version: u16, evm: E, evm_data: EvmData, @@ -129,7 +130,7 @@ impl> NativeInstance { long_term_tag = 0; } let (module, store) = - InitCache::insert(module_hash, module, version, long_term_tag, debug)?; + InitCache::insert(module_hash, module, asm_size_estimate, version, long_term_tag, debug)?; Self::from_module(module, store, env) } diff --git a/arbos/programs/native.go b/arbos/programs/native.go index 377e25a31e..5908741abf 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -211,6 +211,7 @@ func callProgram( address common.Address, moduleHash common.Hash, localAsm []byte, + asmSizeEstimate uint32, scope *vm.ScopeContext, interpreter *vm.EVMInterpreter, tracingInfo *util.TracingInfo, @@ -238,6 +239,7 @@ func callProgram( output := &rustBytes{} status := userStatus(C.stylus_call( goSlice(localAsm), + u32(asmSizeEstimate), goSlice(calldata), stylusParams.encode(), evmApi.cNative, @@ -280,26 +282,26 @@ func cacheProgram(db vm.StateDB, module common.Hash, program Program, addressFor panic("unable to recreate wasm") } tag := db.Database().WasmCacheTag() - state.CacheWasmRust(asm, module, program.version, tag, debug) + state.CacheWasmRust(asm, module, program.asmSize(), program.version, tag, debug) db.RecordCacheWasm(state.CacheWasm{ModuleHash: module, Version: program.version, Tag: tag, Debug: debug}) } } // Evicts a program in Rust. We write a record so that we can undo on revert, unless we don't need to (e.g. expired) // For gas estimation and eth_call, we ignore permanent updates and rely on Rust's LRU. -func evictProgram(db vm.StateDB, module common.Hash, version uint16, debug bool, runMode core.MessageRunMode, forever bool) { +func evictProgram(db vm.StateDB, module common.Hash, asmSizeEstimate uint32, version uint16, debug bool, runMode core.MessageRunMode, forever bool) { if runMode == core.MessageCommitMode { tag := db.Database().WasmCacheTag() state.EvictWasmRust(module, version, tag, debug) if !forever { - db.RecordEvictWasm(state.EvictWasm{ModuleHash: module, Version: version, Tag: tag, Debug: debug}) + db.RecordEvictWasm(state.EvictWasm{ModuleHash: module, Version: version, Tag: tag, Debug: debug, AsmSizeEstimate: asmSizeEstimate}) } } } func init() { - state.CacheWasmRust = func(asm []byte, moduleHash common.Hash, version uint16, tag uint32, debug bool) { - C.stylus_cache_module(goSlice(asm), hashToBytes32(moduleHash), u16(version), u32(tag), cbool(debug)) + state.CacheWasmRust = func(asm []byte, moduleHash common.Hash, asmSizeEstimate uint32, version uint16, tag uint32, debug bool) { + C.stylus_cache_module(goSlice(asm), hashToBytes32(moduleHash), u32(asmSizeEstimate), u16(version), u32(tag), cbool(debug)) } state.EvictWasmRust = func(moduleHash common.Hash, version uint16, tag uint32, debug bool) { C.stylus_evict_module(hashToBytes32(moduleHash), u16(version), u32(tag), cbool(debug)) diff --git a/arbos/programs/programs.go b/arbos/programs/programs.go index 12102bac84..4e50c1fd05 100644 --- a/arbos/programs/programs.go +++ b/arbos/programs/programs.go @@ -127,7 +127,13 @@ func (p Programs) ActivateProgram(evm *vm.EVM, address common.Address, runMode c if err != nil { return 0, codeHash, common.Hash{}, nil, true, err } - evictProgram(statedb, oldModuleHash, currentVersion, debugMode, runMode, expired) + + program, err := p.getActiveProgram(codeHash, time, params) + if err != nil { + return 0, codeHash, common.Hash{}, nil, true, err + } + + evictProgram(statedb, oldModuleHash, program.asmSize(), currentVersion, debugMode, runMode, expired) } if err := p.moduleHashes.Set(codeHash, info.moduleHash); err != nil { return 0, codeHash, common.Hash{}, nil, true, err @@ -247,7 +253,7 @@ func (p Programs) CallProgram( if runmode == core.MessageCommitMode { arbos_tag = statedb.Database().WasmCacheTag() } - ret, err := callProgram(address, moduleHash, localAsm, scope, interpreter, tracingInfo, calldata, evmData, goParams, model, arbos_tag) + ret, err := callProgram(address, moduleHash, localAsm, program.asmSize(), scope, interpreter, tracingInfo, calldata, evmData, goParams, model, arbos_tag) if len(ret) > 0 && arbosVersion >= gethParams.ArbosVersion_StylusFixes { // Ensure that return data costs as least as much as it would in the EVM. evmCost := evmMemoryCost(uint64(len(ret))) @@ -433,7 +439,7 @@ func (p Programs) SetProgramCached( } cacheProgram(db, moduleHash, program, address, code, codeHash, params, debug, time, runMode) } else { - evictProgram(db, moduleHash, program.version, debug, runMode, expired) + evictProgram(db, moduleHash, program.asmSize(), program.version, debug, runMode, expired) } program.cached = cache return p.setProgram(codeHash, program) diff --git a/arbos/programs/wasm.go b/arbos/programs/wasm.go index f7191dca8f..df64a6fd39 100644 --- a/arbos/programs/wasm.go +++ b/arbos/programs/wasm.go @@ -97,7 +97,7 @@ func activateProgram( // stub any non-consensus, Rust-side caching updates func cacheProgram(db vm.StateDB, module common.Hash, program Program, addressForLogging common.Address, code []byte, codeHash common.Hash, params *StylusParams, debug bool, time uint64, runMode core.MessageRunMode) { } -func evictProgram(db vm.StateDB, module common.Hash, version uint16, debug bool, mode core.MessageRunMode, forever bool) { +func evictProgram(db vm.StateDB, module common.Hash, asmSizeEstimate uint32, version uint16, debug bool, mode core.MessageRunMode, forever bool) { } //go:wasmimport programs new_program @@ -136,6 +136,7 @@ func callProgram( address common.Address, moduleHash common.Hash, _localAsm []byte, + _asmSizeEstimate uint32, scope *vm.ScopeContext, interpreter *vm.EVMInterpreter, tracingInfo *util.TracingInfo, From 7e0de89ca655a8c930c302ecab7049f244dc02cf Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 15 Aug 2024 16:47:41 -0300 Subject: [PATCH 101/489] Basic stylus cache metrics --- arbitrator/stylus/src/cache.rs | 26 ++++++++++++++++++++++++++ arbitrator/stylus/src/lib.rs | 8 +++++++- arbos/programs/native.go | 17 +++++++++++++++++ arbos/programs/programs.go | 1 + 4 files changed, 51 insertions(+), 1 deletion(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index 300f0de35a..fc4cce853a 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -73,6 +73,18 @@ impl HeapSize for CacheItem { } } +#[repr(C)] +pub struct CacheContainerMetrics { + pub size_bytes: u32, + pub size_entries: u32, +} + +#[repr(C)] +pub struct CacheMetrics { + pub lru: CacheContainerMetrics, + pub long_term: CacheContainerMetrics, +} + impl InitCache { // current implementation only has one tag that stores to the long_term // future implementations might have more, but 0 is a reserved tag @@ -176,4 +188,18 @@ impl InitCache { let _ = cache.lru.insert(key, item); // not all will fit, just a heuristic } } + + pub fn get_metrics() -> CacheMetrics { + let cache = cache!(); + return CacheMetrics { + lru: CacheContainerMetrics { + size_bytes: cache.lru.current_size().try_into().unwrap(), + size_entries: cache.lru.len().try_into().unwrap(), + }, + long_term: CacheContainerMetrics { + size_bytes: 0, // not tracked at this moment + size_entries: cache.long_term.len().try_into().unwrap(), + }, + } + } } diff --git a/arbitrator/stylus/src/lib.rs b/arbitrator/stylus/src/lib.rs index 979736054c..a660096045 100644 --- a/arbitrator/stylus/src/lib.rs +++ b/arbitrator/stylus/src/lib.rs @@ -11,7 +11,7 @@ use arbutil::{ format::DebugBytes, Bytes32, }; -use cache::InitCache; +use cache::{InitCache, CacheMetrics}; use evm_api::NativeRequestHandler; use eyre::ErrReport; use native::NativeInstance; @@ -357,3 +357,9 @@ pub unsafe extern "C" fn stylus_drop_vec(vec: RustBytes) { mem::drop(vec.into_vec()) } } + +/// Gets cache metrics. +#[no_mangle] +pub extern "C" fn stylus_get_cache_metrics() -> CacheMetrics { + InitCache::get_metrics() +} diff --git a/arbos/programs/native.go b/arbos/programs/native.go index 5908741abf..b59c96e0bf 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/offchainlabs/nitro/arbos/burn" "github.com/offchainlabs/nitro/arbos/util" "github.com/offchainlabs/nitro/arbutil" @@ -45,6 +46,13 @@ type bytes32 = C.Bytes32 type rustBytes = C.RustBytes type rustSlice = C.RustSlice +var ( + stylusCacheLRUSizeBytesGauge = metrics.NewRegisteredGauge("arb/arbos/stylus/cache/lru/size/bytes", nil) + stylusCacheLRUSizeEntriesGauge = metrics.NewRegisteredGauge("arb/arbos/stylus/cache/lru/size/entries", nil) + + stylusCacheLongTermSizeEntriesGauge = metrics.NewRegisteredGauge("arb/arbos/stylus/cache/longterm/size/entries", nil) +) + func activateProgram( db vm.StateDB, program common.Address, @@ -261,6 +269,15 @@ func callProgram( return data, err } +func getMetrics() { + metrics := C.stylus_get_cache_metrics() + + stylusCacheLRUSizeBytesGauge.Update(int64(metrics.lru.size_bytes)) + stylusCacheLRUSizeEntriesGauge.Update(int64(metrics.lru.size_entries)) + + stylusCacheLongTermSizeEntriesGauge.Update(int64(metrics.lru.size_entries)) +} + //export handleReqImpl func handleReqImpl(apiId usize, req_type u32, data *rustSlice, costPtr *u64, out_response *C.GoSliceData, out_raw_data *C.GoSliceData) { api := getApi(apiId) diff --git a/arbos/programs/programs.go b/arbos/programs/programs.go index 4e50c1fd05..bf2e6cc8e1 100644 --- a/arbos/programs/programs.go +++ b/arbos/programs/programs.go @@ -264,6 +264,7 @@ func (p Programs) CallProgram( maxGasToReturn := startingGas - evmCost contract.Gas = am.MinInt(contract.Gas, maxGasToReturn) } + getMetrics() return ret, err } From 9bc34deef45a27b5262ebbaa8aba9d62cec32af7 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 16 Aug 2024 10:51:17 -0300 Subject: [PATCH 102/489] clru instead of lru-mem --- arbitrator/Cargo.lock | 18 +++++++---------- arbitrator/stylus/Cargo.toml | 3 +-- arbitrator/stylus/src/cache.rs | 36 +++++++++++++++------------------- arbos/programs/native.go | 1 + arbos/programs/wasm.go | 2 ++ 5 files changed, 27 insertions(+), 33 deletions(-) diff --git a/arbitrator/Cargo.lock b/arbitrator/Cargo.lock index a46e000ce2..6048733acb 100644 --- a/arbitrator/Cargo.lock +++ b/arbitrator/Cargo.lock @@ -496,6 +496,12 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +[[package]] +name = "clru" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbd0f76e066e64fdc5631e3bb46381254deab9ef1158292f27c8c57e3bf3fe59" + [[package]] name = "colorchoice" version = "1.0.2" @@ -1342,15 +1348,6 @@ dependencies = [ "hashbrown 0.14.5", ] -[[package]] -name = "lru-mem" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf5c8c26d903a41c80d4cc171940a57a4d1bc51139ebd6aad87e2f9ae3774780" -dependencies = [ - "hashbrown 0.14.5", -] - [[package]] name = "mach" version = "0.3.2" @@ -2279,14 +2276,13 @@ dependencies = [ "bincode", "brotli", "caller-env", + "clru", "derivative", "eyre", "fnv", "hex", "lazy_static", "libc", - "lru", - "lru-mem", "num-bigint", "parking_lot", "prover", diff --git a/arbitrator/stylus/Cargo.toml b/arbitrator/stylus/Cargo.toml index 45c06fc881..ea1d878ea0 100644 --- a/arbitrator/stylus/Cargo.toml +++ b/arbitrator/stylus/Cargo.toml @@ -25,8 +25,7 @@ eyre = "0.6.5" rand = "0.8.5" fnv = "1.0.7" hex = "0.4.3" -lru-mem = "0.3.0" -lru = "0.12.4" +clru = "0.6.2" [dev-dependencies] num-bigint = "0.4.4" diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index fc4cce853a..7b625de216 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -4,11 +4,12 @@ use arbutil::Bytes32; use eyre::Result; use lazy_static::lazy_static; -use lru_mem::{HeapSize, LruCache}; +use clru::{CLruCache, CLruCacheConfig, WeightScale}; use parking_lot::Mutex; use prover::programs::config::CompileConfig; -use std::collections::HashMap; +use std::{collections::HashMap, num::NonZeroUsize}; use wasmer::{Engine, Module, Store}; +use std::hash::RandomState; use crate::target_cache::target_native; @@ -24,7 +25,7 @@ macro_rules! cache { pub struct InitCache { long_term: HashMap, - lru: LruCache, + lru: CLruCache, } #[derive(Clone, Copy, Hash, PartialEq, Eq)] @@ -44,12 +45,6 @@ impl CacheKey { } } -impl HeapSize for CacheKey { - fn heap_size(&self) -> usize { - 0 - } -} - #[derive(Clone)] struct CacheItem { module: Module, @@ -67,9 +62,10 @@ impl CacheItem { } } -impl HeapSize for CacheItem { - fn heap_size(&self) -> usize { - return self.asm_size_estimate.try_into().unwrap(); +struct CustomWeightScale; +impl WeightScale for CustomWeightScale { + fn weight(&self, _key: &CacheKey, val: &CacheItem) -> usize { + val.asm_size_estimate.try_into().unwrap() } } @@ -94,15 +90,14 @@ impl InitCache { fn new(size: usize) -> Self { Self { long_term: HashMap::new(), - lru: LruCache::new(size), + lru: CLruCache::with_config(CLruCacheConfig::new(NonZeroUsize::new(size).unwrap()).with_scale(CustomWeightScale)), } } - // TODO: Check if needs to shrink capacity pub fn set_lru_size(size: u32) { cache!() .lru - .set_max_size(size.try_into().unwrap()) + .resize(NonZeroUsize::new(size.try_into().unwrap()).unwrap()) } /// Retrieves a cached value, updating items as necessary. @@ -143,7 +138,8 @@ impl InitCache { if long_term_tag == Self::ARBOS_TAG { cache.long_term.insert(key, item.clone()); } else { - cache.lru.touch(&key) + // only calls get to move the key to the head of the LRU list + cache.lru.get(&key); } return Ok(item.data()); } @@ -157,7 +153,7 @@ impl InitCache { let mut cache = cache!(); if long_term_tag != Self::ARBOS_TAG { // TODO: handle result - let _ = cache.lru.insert(key, item); + let _ = cache.lru.put_with_weight(key, item); } else { cache.long_term.insert(key, item); } @@ -173,7 +169,7 @@ impl InitCache { let mut cache = cache!(); if let Some(item) = cache.long_term.remove(&key) { // TODO: handle result - let _ = cache.lru.insert(key, item); + let _ = cache.lru.put_with_weight(key, item); } } @@ -185,7 +181,7 @@ impl InitCache { let cache = &mut *cache; for (key, item) in cache.long_term.drain() { // TODO: handle result - let _ = cache.lru.insert(key, item); // not all will fit, just a heuristic + let _ = cache.lru.put_with_weight(key, item); // not all will fit, just a heuristic } } @@ -193,7 +189,7 @@ impl InitCache { let cache = cache!(); return CacheMetrics { lru: CacheContainerMetrics { - size_bytes: cache.lru.current_size().try_into().unwrap(), + size_bytes: cache.lru.weight().try_into().unwrap(), size_entries: cache.lru.len().try_into().unwrap(), }, long_term: CacheContainerMetrics { diff --git a/arbos/programs/native.go b/arbos/programs/native.go index b59c96e0bf..22e5f411e3 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -271,6 +271,7 @@ func callProgram( func getMetrics() { metrics := C.stylus_get_cache_metrics() + log.Error("CacheMetrics", "metrics", metrics) stylusCacheLRUSizeBytesGauge.Update(int64(metrics.lru.size_bytes)) stylusCacheLRUSizeEntriesGauge.Update(int64(metrics.lru.size_entries)) diff --git a/arbos/programs/wasm.go b/arbos/programs/wasm.go index df64a6fd39..ecf685824e 100644 --- a/arbos/programs/wasm.go +++ b/arbos/programs/wasm.go @@ -152,6 +152,8 @@ func callProgram( return retData, err } +func getMetrics() {} + func CallProgramLoop( moduleHash common.Hash, calldata []byte, From f4c723cc9c7c25aa33929587781151f0e6f7dc91 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 16 Aug 2024 19:39:09 -0300 Subject: [PATCH 103/489] Simplifies stylus lru cache metrics retrieval --- arbitrator/stylus/src/cache.rs | 26 +++++++------------------- arbitrator/stylus/src/lib.rs | 8 ++++---- arbos/programs/native.go | 15 +++++---------- 3 files changed, 16 insertions(+), 33 deletions(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index 7b625de216..035286fc4a 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -70,15 +70,9 @@ impl WeightScale for CustomWeightScale { } #[repr(C)] -pub struct CacheContainerMetrics { - pub size_bytes: u32, - pub size_entries: u32, -} - -#[repr(C)] -pub struct CacheMetrics { - pub lru: CacheContainerMetrics, - pub long_term: CacheContainerMetrics, +pub struct LruCacheMetrics { + pub size: u64, + pub count: u64, } impl InitCache { @@ -185,17 +179,11 @@ impl InitCache { } } - pub fn get_metrics() -> CacheMetrics { + pub fn get_lru_metrics() -> LruCacheMetrics { let cache = cache!(); - return CacheMetrics { - lru: CacheContainerMetrics { - size_bytes: cache.lru.weight().try_into().unwrap(), - size_entries: cache.lru.len().try_into().unwrap(), - }, - long_term: CacheContainerMetrics { - size_bytes: 0, // not tracked at this moment - size_entries: cache.long_term.len().try_into().unwrap(), - }, + return LruCacheMetrics{ + size: cache.lru.weight().try_into().unwrap(), + count: cache.lru.len().try_into().unwrap(), } } } diff --git a/arbitrator/stylus/src/lib.rs b/arbitrator/stylus/src/lib.rs index a660096045..e983dd5320 100644 --- a/arbitrator/stylus/src/lib.rs +++ b/arbitrator/stylus/src/lib.rs @@ -11,7 +11,7 @@ use arbutil::{ format::DebugBytes, Bytes32, }; -use cache::{InitCache, CacheMetrics}; +use cache::{InitCache, LruCacheMetrics}; use evm_api::NativeRequestHandler; use eyre::ErrReport; use native::NativeInstance; @@ -358,8 +358,8 @@ pub unsafe extern "C" fn stylus_drop_vec(vec: RustBytes) { } } -/// Gets cache metrics. +/// Gets lru cache metrics. #[no_mangle] -pub extern "C" fn stylus_get_cache_metrics() -> CacheMetrics { - InitCache::get_metrics() +pub extern "C" fn stylus_get_lru_cache_metrics() -> LruCacheMetrics { + InitCache::get_lru_metrics() } diff --git a/arbos/programs/native.go b/arbos/programs/native.go index 22e5f411e3..4b778a9e14 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -47,10 +47,8 @@ type rustBytes = C.RustBytes type rustSlice = C.RustSlice var ( - stylusCacheLRUSizeBytesGauge = metrics.NewRegisteredGauge("arb/arbos/stylus/cache/lru/size/bytes", nil) - stylusCacheLRUSizeEntriesGauge = metrics.NewRegisteredGauge("arb/arbos/stylus/cache/lru/size/entries", nil) - - stylusCacheLongTermSizeEntriesGauge = metrics.NewRegisteredGauge("arb/arbos/stylus/cache/longterm/size/entries", nil) + stylusLRUCacheSizeKbGauge = metrics.NewRegisteredGauge("arb/arbos/stylus/cache/lru/size_kilobytes", nil) + stylusLRUCacheSizeCountGauge = metrics.NewRegisteredGauge("arb/arbos/stylus/cache/lru/count", nil) ) func activateProgram( @@ -270,13 +268,10 @@ func callProgram( } func getMetrics() { - metrics := C.stylus_get_cache_metrics() - log.Error("CacheMetrics", "metrics", metrics) - - stylusCacheLRUSizeBytesGauge.Update(int64(metrics.lru.size_bytes)) - stylusCacheLRUSizeEntriesGauge.Update(int64(metrics.lru.size_entries)) + metrics := C.stylus_get_lru_cache_metrics() - stylusCacheLongTermSizeEntriesGauge.Update(int64(metrics.lru.size_entries)) + stylusLRUCacheSizeKbGauge.Update(int64(metrics.size)) + stylusLRUCacheSizeCountGauge.Update(int64(metrics.count)) } //export handleReqImpl From 098fca51ee6eb8567b6f8e8fa96533d936d630e6 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Sun, 18 Aug 2024 21:06:32 -0300 Subject: [PATCH 104/489] AsmSizeEstimate to AsmSizeEstimateKb, CLI rust cache size flag in Mb --- arbitrator/stylus/src/cache.rs | 20 ++++++++++---------- arbitrator/stylus/src/lib.rs | 4 ++-- arbnode/inbox_test.go | 2 +- arbos/programs/native.go | 18 +++++++++--------- arbos/programs/programs.go | 2 +- arbos/programs/wasm.go | 2 +- execution/gethexec/blockchain.go | 6 +++--- execution/gethexec/executionengine.go | 6 +++--- execution/gethexec/node.go | 2 +- system_tests/common_test.go | 2 +- 10 files changed, 32 insertions(+), 32 deletions(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index 035286fc4a..3d760e40c9 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -49,12 +49,12 @@ impl CacheKey { struct CacheItem { module: Module, engine: Engine, - asm_size_estimate: u32, + asm_size_estimate_kb: u32, } impl CacheItem { - fn new(module: Module, engine: Engine, asm_size_estimate: u32) -> Self { - Self { module, engine, asm_size_estimate } + fn new(module: Module, engine: Engine, asm_size_estimate_kb: u32) -> Self { + Self { module, engine, asm_size_estimate_kb } } fn data(&self) -> (Module, Store) { @@ -65,13 +65,13 @@ impl CacheItem { struct CustomWeightScale; impl WeightScale for CustomWeightScale { fn weight(&self, _key: &CacheKey, val: &CacheItem) -> usize { - val.asm_size_estimate.try_into().unwrap() + val.asm_size_estimate_kb.try_into().unwrap() } } #[repr(C)] pub struct LruCacheMetrics { - pub size: u64, + pub size_kb: u64, pub count: u64, } @@ -88,10 +88,10 @@ impl InitCache { } } - pub fn set_lru_size(size: u32) { + pub fn set_lru_size(size_kb: u32) { cache!() .lru - .resize(NonZeroUsize::new(size.try_into().unwrap()).unwrap()) + .resize(NonZeroUsize::new(size_kb.try_into().unwrap()).unwrap()) } /// Retrieves a cached value, updating items as necessary. @@ -116,7 +116,7 @@ impl InitCache { pub fn insert( module_hash: Bytes32, module: &[u8], - asm_size_estimate: u32, + asm_size_estimate_kb: u32, version: u16, long_term_tag: u32, debug: bool, @@ -142,7 +142,7 @@ impl InitCache { let engine = CompileConfig::version(version, debug).engine(target_native()); let module = unsafe { Module::deserialize_unchecked(&engine, module)? }; - let item = CacheItem::new(module, engine, asm_size_estimate); + let item = CacheItem::new(module, engine, asm_size_estimate_kb); let data = item.data(); let mut cache = cache!(); if long_term_tag != Self::ARBOS_TAG { @@ -182,7 +182,7 @@ impl InitCache { pub fn get_lru_metrics() -> LruCacheMetrics { let cache = cache!(); return LruCacheMetrics{ - size: cache.lru.weight().try_into().unwrap(), + size_kb: cache.lru.weight().try_into().unwrap(), count: cache.lru.len().try_into().unwrap(), } } diff --git a/arbitrator/stylus/src/lib.rs b/arbitrator/stylus/src/lib.rs index e983dd5320..6b62e4103a 100644 --- a/arbitrator/stylus/src/lib.rs +++ b/arbitrator/stylus/src/lib.rs @@ -304,8 +304,8 @@ pub unsafe extern "C" fn stylus_call( /// resize lru #[no_mangle] -pub extern "C" fn stylus_cache_lru_resize(size: u32) { - InitCache::set_lru_size(size); +pub extern "C" fn stylus_cache_lru_resize(size_kb: u32) { + InitCache::set_lru_size(size_kb); } /// Caches an activated user program. diff --git a/arbnode/inbox_test.go b/arbnode/inbox_test.go index e0c97bb871..42661c44a6 100644 --- a/arbnode/inbox_test.go +++ b/arbnode/inbox_test.go @@ -74,7 +74,7 @@ func NewTransactionStreamerForTest(t *testing.T, ownerAddress common.Address) (* } stylusTargetConfig := &gethexec.DefaultStylusTargetConfig Require(t, stylusTargetConfig.Validate()) // pre-processes config (i.a. parses wasmTargets) - if err := execEngine.Initialize(gethexec.DefaultCachingConfig.StylusLRUCacheSize, &gethexec.DefaultStylusTargetConfig); err != nil { + if err := execEngine.Initialize(gethexec.DefaultCachingConfig.StylusLRUCacheSizeMb, &gethexec.DefaultStylusTargetConfig); err != nil { Fail(t, err) } execSeq := &execClientWrapper{execEngine, t} diff --git a/arbos/programs/native.go b/arbos/programs/native.go index 4b778a9e14..6cbf01c0cd 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -217,7 +217,7 @@ func callProgram( address common.Address, moduleHash common.Hash, localAsm []byte, - asmSizeEstimate uint32, + asmSizeEstimateKb uint32, scope *vm.ScopeContext, interpreter *vm.EVMInterpreter, tracingInfo *util.TracingInfo, @@ -245,7 +245,7 @@ func callProgram( output := &rustBytes{} status := userStatus(C.stylus_call( goSlice(localAsm), - u32(asmSizeEstimate), + u32(asmSizeEstimateKb), goSlice(calldata), stylusParams.encode(), evmApi.cNative, @@ -270,7 +270,7 @@ func callProgram( func getMetrics() { metrics := C.stylus_get_lru_cache_metrics() - stylusLRUCacheSizeKbGauge.Update(int64(metrics.size)) + stylusLRUCacheSizeKbGauge.Update(int64(metrics.size_kb)) stylusLRUCacheSizeCountGauge.Update(int64(metrics.count)) } @@ -302,27 +302,27 @@ func cacheProgram(db vm.StateDB, module common.Hash, program Program, addressFor // Evicts a program in Rust. We write a record so that we can undo on revert, unless we don't need to (e.g. expired) // For gas estimation and eth_call, we ignore permanent updates and rely on Rust's LRU. -func evictProgram(db vm.StateDB, module common.Hash, asmSizeEstimate uint32, version uint16, debug bool, runMode core.MessageRunMode, forever bool) { +func evictProgram(db vm.StateDB, module common.Hash, asmSizeEstimateKb uint32, version uint16, debug bool, runMode core.MessageRunMode, forever bool) { if runMode == core.MessageCommitMode { tag := db.Database().WasmCacheTag() state.EvictWasmRust(module, version, tag, debug) if !forever { - db.RecordEvictWasm(state.EvictWasm{ModuleHash: module, Version: version, Tag: tag, Debug: debug, AsmSizeEstimate: asmSizeEstimate}) + db.RecordEvictWasm(state.EvictWasm{ModuleHash: module, Version: version, Tag: tag, Debug: debug, AsmSizeEstimateKb: asmSizeEstimateKb}) } } } func init() { - state.CacheWasmRust = func(asm []byte, moduleHash common.Hash, asmSizeEstimate uint32, version uint16, tag uint32, debug bool) { - C.stylus_cache_module(goSlice(asm), hashToBytes32(moduleHash), u32(asmSizeEstimate), u16(version), u32(tag), cbool(debug)) + state.CacheWasmRust = func(asm []byte, moduleHash common.Hash, asmSizeEstimateKb uint32, version uint16, tag uint32, debug bool) { + C.stylus_cache_module(goSlice(asm), hashToBytes32(moduleHash), u32(asmSizeEstimateKb), u16(version), u32(tag), cbool(debug)) } state.EvictWasmRust = func(moduleHash common.Hash, version uint16, tag uint32, debug bool) { C.stylus_evict_module(hashToBytes32(moduleHash), u16(version), u32(tag), cbool(debug)) } } -func ResizeWasmLruCache(size uint32) { - C.stylus_cache_lru_resize(u32(size)) +func ResizeWasmLruCache(sizeKb uint32) { + C.stylus_cache_lru_resize(u32(sizeKb)) } const DefaultTargetDescriptionArm = "arm64-linux-unknown+neon" diff --git a/arbos/programs/programs.go b/arbos/programs/programs.go index bf2e6cc8e1..9ca43fa224 100644 --- a/arbos/programs/programs.go +++ b/arbos/programs/programs.go @@ -253,7 +253,7 @@ func (p Programs) CallProgram( if runmode == core.MessageCommitMode { arbos_tag = statedb.Database().WasmCacheTag() } - ret, err := callProgram(address, moduleHash, localAsm, program.asmSize(), scope, interpreter, tracingInfo, calldata, evmData, goParams, model, arbos_tag) + ret, err := callProgram(address, moduleHash, localAsm, program.asmEstimateKb.ToUint32(), scope, interpreter, tracingInfo, calldata, evmData, goParams, model, arbos_tag) if len(ret) > 0 && arbosVersion >= gethParams.ArbosVersion_StylusFixes { // Ensure that return data costs as least as much as it would in the EVM. evmCost := evmMemoryCost(uint64(len(ret))) diff --git a/arbos/programs/wasm.go b/arbos/programs/wasm.go index ecf685824e..4b9a3ad473 100644 --- a/arbos/programs/wasm.go +++ b/arbos/programs/wasm.go @@ -136,7 +136,7 @@ func callProgram( address common.Address, moduleHash common.Hash, _localAsm []byte, - _asmSizeEstimate uint32, + _asmSizeEstimateKb uint32, scope *vm.ScopeContext, interpreter *vm.EVMInterpreter, tracingInfo *util.TracingInfo, diff --git a/execution/gethexec/blockchain.go b/execution/gethexec/blockchain.go index 54934dbdf3..da6a22bdb4 100644 --- a/execution/gethexec/blockchain.go +++ b/execution/gethexec/blockchain.go @@ -37,7 +37,7 @@ type CachingConfig struct { SnapshotRestoreGasLimit uint64 `koanf:"snapshot-restore-gas-limit"` MaxNumberOfBlocksToSkipStateSaving uint32 `koanf:"max-number-of-blocks-to-skip-state-saving"` MaxAmountOfGasToSkipStateSaving uint64 `koanf:"max-amount-of-gas-to-skip-state-saving"` - StylusLRUCacheSize uint32 `koanf:"stylus-lru-cache-size"` + StylusLRUCacheSizeMb uint32 `koanf:"stylus-lru-cache-size"` StateScheme string `koanf:"state-scheme"` StateHistory uint64 `koanf:"state-history"` } @@ -54,7 +54,7 @@ func CachingConfigAddOptions(prefix string, f *flag.FlagSet) { f.Uint64(prefix+".snapshot-restore-gas-limit", DefaultCachingConfig.SnapshotRestoreGasLimit, "maximum gas rolled back to recover snapshot") f.Uint32(prefix+".max-number-of-blocks-to-skip-state-saving", DefaultCachingConfig.MaxNumberOfBlocksToSkipStateSaving, "maximum number of blocks to skip state saving to persistent storage (archive node only) -- warning: this option seems to cause issues") f.Uint64(prefix+".max-amount-of-gas-to-skip-state-saving", DefaultCachingConfig.MaxAmountOfGasToSkipStateSaving, "maximum amount of gas in blocks to skip saving state to Persistent storage (archive node only) -- warning: this option seems to cause issues") - f.Uint32(prefix+".stylus-lru-cache-size", DefaultCachingConfig.StylusLRUCacheSize, "capacity, in bytes, of the LRU cache that keeps initialized stylus programs") + f.Uint32(prefix+".stylus-lru-cache-size", DefaultCachingConfig.StylusLRUCacheSizeMb, "capacity, in megabytes, of the LRU cache that keeps initialized stylus programs") f.String(prefix+".state-scheme", DefaultCachingConfig.StateScheme, "scheme to use for state trie storage (hash, path)") f.Uint64(prefix+".state-history", DefaultCachingConfig.StateHistory, "number of recent blocks to retain state history for (path state-scheme only)") } @@ -75,7 +75,7 @@ var DefaultCachingConfig = CachingConfig{ SnapshotRestoreGasLimit: 300_000_000_000, MaxNumberOfBlocksToSkipStateSaving: 0, MaxAmountOfGasToSkipStateSaving: 0, - StylusLRUCacheSize: 256 * 10 * 1024, + StylusLRUCacheSizeMb: 10, StateScheme: rawdb.HashScheme, StateHistory: getStateHistory(DefaultSequencerConfig.MaxBlockSpeed), } diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index 8594d5867d..42651ed800 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -182,9 +182,9 @@ func populateStylusTargetCache(targetConfig *StylusTargetConfig) error { return nil } -func (s *ExecutionEngine) Initialize(rustCacheSize uint32, targetConfig *StylusTargetConfig) error { - if rustCacheSize != 0 { - programs.ResizeWasmLruCache(rustCacheSize) +func (s *ExecutionEngine) Initialize(rustCacheSizeMb uint32, targetConfig *StylusTargetConfig) error { + if rustCacheSizeMb != 0 { + programs.ResizeWasmLruCache(arbmath.SaturatingUMul(rustCacheSizeMb, 1024)) } if err := populateStylusTargetCache(targetConfig); err != nil { return fmt.Errorf("error populating stylus target cache: %w", err) diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index 93bb254ed8..b602dbd323 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -308,7 +308,7 @@ func (n *ExecutionNode) MarkFeedStart(to arbutil.MessageIndex) { func (n *ExecutionNode) Initialize(ctx context.Context) error { config := n.ConfigFetcher() - err := n.ExecEngine.Initialize(config.Caching.StylusLRUCacheSize, &config.StylusTarget) + err := n.ExecEngine.Initialize(config.Caching.StylusLRUCacheSizeMb, &config.StylusTarget) if err != nil { return fmt.Errorf("error initializing execution engine: %w", err) } diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 363f1050fb..c1e9ff384b 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -166,7 +166,7 @@ var TestCachingConfig = gethexec.CachingConfig{ SnapshotRestoreGasLimit: 300_000_000_000, MaxNumberOfBlocksToSkipStateSaving: 0, MaxAmountOfGasToSkipStateSaving: 0, - StylusLRUCacheSize: 0, + StylusLRUCacheSizeMb: 0, StateScheme: env.GetTestStateScheme(), } From 6261168ea808ed3da2e226ce9baadadb4af822f2 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Sun, 18 Aug 2024 22:28:29 -0300 Subject: [PATCH 105/489] Fixes weight of stylus cache entry --- arbitrator/stylus/src/cache.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index 3d760e40c9..e7b9c8a3dc 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -65,7 +65,13 @@ impl CacheItem { struct CustomWeightScale; impl WeightScale for CustomWeightScale { fn weight(&self, _key: &CacheKey, val: &CacheItem) -> usize { - val.asm_size_estimate_kb.try_into().unwrap() + let mut w = val.asm_size_estimate_kb.try_into().unwrap(); + if w > 0 { + // clru defines the weight of an entry as the number returned by this function plus one. + // Therefore, we need to subtract one from the weight to get the actual weight of the entry. + w -= 1; + } + return w; } } From a715fb4c0eaf049e142fc2029ab6f9450f9c933c Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Sun, 18 Aug 2024 22:48:29 -0300 Subject: [PATCH 106/489] Handles cache.lru.put_with_weight return --- arbitrator/stylus/src/cache.rs | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index e7b9c8a3dc..e13c78edc1 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -152,8 +152,10 @@ impl InitCache { let data = item.data(); let mut cache = cache!(); if long_term_tag != Self::ARBOS_TAG { - // TODO: handle result - let _ = cache.lru.put_with_weight(key, item); + match cache.lru.put_with_weight(key, item) { + Err(_) => println!("Failed to insert into LRU cache, item too large"), + Ok(_) => (), + }; } else { cache.long_term.insert(key, item); } @@ -168,8 +170,10 @@ impl InitCache { let key = CacheKey::new(module_hash, version, debug); let mut cache = cache!(); if let Some(item) = cache.long_term.remove(&key) { - // TODO: handle result - let _ = cache.lru.put_with_weight(key, item); + match cache.lru.put_with_weight(key, item) { + Err(_) => println!("Failed to insert into LRU cache, item too large"), + Ok(_) => (), + }; } } @@ -180,8 +184,11 @@ impl InitCache { let mut cache = cache!(); let cache = &mut *cache; for (key, item) in cache.long_term.drain() { - // TODO: handle result - let _ = cache.lru.put_with_weight(key, item); // not all will fit, just a heuristic + // not all will fit, just a heuristic + match cache.lru.put_with_weight(key, item) { + Err(_) => println!("Failed to insert into LRU cache, item too large"), + Ok(_) => (), + }; } } From 1c21c3a5609dadb7e3ad89979beb4770e3e60493 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 20 Aug 2024 10:11:45 -0300 Subject: [PATCH 107/489] TestWasLruCache --- arbitrator/stylus/src/cache.rs | 10 +++- arbitrator/stylus/src/lib.rs | 7 +++ arbos/programs/native.go | 23 +++++++- arbos/programs/programs.go | 2 +- arbos/programs/wasm.go | 2 +- execution/gethexec/executionengine.go | 17 +++++- system_tests/program_test.go | 82 +++++++++++++++++++++++++++ 7 files changed, 137 insertions(+), 6 deletions(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index e13c78edc1..2ef674bf6e 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -194,9 +194,15 @@ impl InitCache { pub fn get_lru_metrics() -> LruCacheMetrics { let cache = cache!(); + let count = cache.lru.len(); return LruCacheMetrics{ - size_kb: cache.lru.weight().try_into().unwrap(), - count: cache.lru.len().try_into().unwrap(), + size_kb: (cache.lru.weight() + count).try_into().unwrap(), + count: count.try_into().unwrap(), } } + + // only used for testing + pub fn clear_lru_cache() { + cache!().lru.clear(); + } } diff --git a/arbitrator/stylus/src/lib.rs b/arbitrator/stylus/src/lib.rs index 6b62e4103a..576d041c28 100644 --- a/arbitrator/stylus/src/lib.rs +++ b/arbitrator/stylus/src/lib.rs @@ -363,3 +363,10 @@ pub unsafe extern "C" fn stylus_drop_vec(vec: RustBytes) { pub extern "C" fn stylus_get_lru_cache_metrics() -> LruCacheMetrics { InitCache::get_lru_metrics() } + +/// Clears lru cache. +/// Only used for testing purposes. +#[no_mangle] +pub extern "C" fn stylus_clear_lru_cache() { + InitCache::clear_lru_cache() +} diff --git a/arbos/programs/native.go b/arbos/programs/native.go index 6cbf01c0cd..c10cc50f30 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -267,7 +267,8 @@ func callProgram( return data, err } -func getMetrics() { +// exposes for testing +func GetMetrics() { metrics := C.stylus_get_lru_cache_metrics() stylusLRUCacheSizeKbGauge.Update(int64(metrics.size_kb)) @@ -325,6 +326,26 @@ func ResizeWasmLruCache(sizeKb uint32) { C.stylus_cache_lru_resize(u32(sizeKb)) } +// exported for testing +type WasmLruCacheMetrics struct { + SizeKb uint64 + Count uint64 +} + +// exported for testing +func GetWasmLruCacheMetrics() *WasmLruCacheMetrics { + metrics := C.stylus_get_lru_cache_metrics() + return &WasmLruCacheMetrics{ + SizeKb: uint64(metrics.size_kb), + Count: uint64(metrics.count), + } +} + +// exported for testing +func ClearWasmLruCache() { + C.stylus_clear_lru_cache() +} + const DefaultTargetDescriptionArm = "arm64-linux-unknown+neon" const DefaultTargetDescriptionX86 = "x86_64-linux-unknown+sse4.2+lzcnt+bmi" diff --git a/arbos/programs/programs.go b/arbos/programs/programs.go index 9ca43fa224..3909b0fceb 100644 --- a/arbos/programs/programs.go +++ b/arbos/programs/programs.go @@ -264,7 +264,7 @@ func (p Programs) CallProgram( maxGasToReturn := startingGas - evmCost contract.Gas = am.MinInt(contract.Gas, maxGasToReturn) } - getMetrics() + GetMetrics() return ret, err } diff --git a/arbos/programs/wasm.go b/arbos/programs/wasm.go index 4b9a3ad473..3889d47f34 100644 --- a/arbos/programs/wasm.go +++ b/arbos/programs/wasm.go @@ -152,7 +152,7 @@ func callProgram( return retData, err } -func getMetrics() {} +func GetMetrics() {} func CallProgramLoop( moduleHash common.Hash, diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index 42651ed800..dce3639d02 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -184,7 +184,7 @@ func populateStylusTargetCache(targetConfig *StylusTargetConfig) error { func (s *ExecutionEngine) Initialize(rustCacheSizeMb uint32, targetConfig *StylusTargetConfig) error { if rustCacheSizeMb != 0 { - programs.ResizeWasmLruCache(arbmath.SaturatingUMul(rustCacheSizeMb, 1024)) + s.ResizeWasmLruCache(arbmath.SaturatingUMul(rustCacheSizeMb, 1024)) } if err := populateStylusTargetCache(targetConfig); err != nil { return fmt.Errorf("error populating stylus target cache: %w", err) @@ -192,6 +192,21 @@ func (s *ExecutionEngine) Initialize(rustCacheSizeMb uint32, targetConfig *Stylu return nil } +// exported for testing +func (s *ExecutionEngine) ResizeWasmLruCache(sizeKb uint32) { + programs.ResizeWasmLruCache(sizeKb) +} + +// exported for testing +func (s *ExecutionEngine) GetWasmLruCacheMetrics() *programs.WasmLruCacheMetrics { + return programs.GetWasmLruCacheMetrics() +} + +// exported for testing +func (s *ExecutionEngine) ClearWasmLruCache() { + programs.ClearWasmLruCache() +} + func (s *ExecutionEngine) SetRecorder(recorder *BlockRecorder) { if s.Started() { panic("trying to set recorder after start") diff --git a/system_tests/program_test.go b/system_tests/program_test.go index 83c066fdb5..034e1c5fd3 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" + "github.com/google/go-cmp/cmp" "github.com/offchainlabs/nitro/arbcompress" "github.com/offchainlabs/nitro/arbos/programs" "github.com/offchainlabs/nitro/arbos/util" @@ -2007,3 +2008,84 @@ func checkWasmStoreContent(t *testing.T, wasmDb ethdb.KeyValueStore, targets []s } } } + +func TestWasmLruCache(t *testing.T) { + builder, auth, cleanup := setupProgramTest(t, true) + ctx := builder.ctx + l2info := builder.L2Info + l2client := builder.L2.Client + defer cleanup() + + builder.L2.ExecNode.ExecEngine.ClearWasmLruCache() + lruMetrics := builder.L2.ExecNode.ExecEngine.GetWasmLruCacheMetrics() + expectedLruMetrics := &programs.WasmLruCacheMetrics{} + if diff := cmp.Diff(lruMetrics, expectedLruMetrics); diff != "" { + t.Fatalf("lru cache metrics different than expected: %s", diff) + } + + lruCacheSize := uint32(500) + builder.L2.ExecNode.ExecEngine.ResizeWasmLruCache(lruCacheSize) + + // fallible wasm program will not be cached since its size is greater than lruCacheSize + fallibleAsmEstimateSizeKb := uint64(551) + fallibleProgramAddress := deployWasm(t, ctx, auth, l2client, rustFile("fallible")) + tx := l2info.PrepareTxTo("Owner", &fallibleProgramAddress, l2info.TransferGas, nil, []byte{0x01}) + Require(t, l2client.SendTransaction(ctx, tx)) + _, err := EnsureTxSucceeded(ctx, l2client, tx) + Require(t, err) + lruMetrics = builder.L2.ExecNode.ExecEngine.GetWasmLruCacheMetrics() + expectedLruMetrics = &programs.WasmLruCacheMetrics{} + if diff := cmp.Diff(lruMetrics, expectedLruMetrics); diff != "" { + t.Fatalf("lru cache metrics different than expected: %s", diff) + } + + // resize lru cache + lruCacheSize = uint32(1500) + builder.L2.ExecNode.ExecEngine.ResizeWasmLruCache(lruCacheSize) + + // fallible wasm program will be cached + tx = l2info.PrepareTxTo("Owner", &fallibleProgramAddress, l2info.TransferGas, nil, []byte{0x01}) + Require(t, l2client.SendTransaction(ctx, tx)) + _, err = EnsureTxSucceeded(ctx, l2client, tx) + Require(t, err) + lruMetrics = builder.L2.ExecNode.ExecEngine.GetWasmLruCacheMetrics() + expectedLruMetrics = &programs.WasmLruCacheMetrics{ + SizeKb: fallibleAsmEstimateSizeKb, + Count: 1, + } + if diff := cmp.Diff(lruMetrics, expectedLruMetrics); diff != "" { + t.Fatalf("lru cache metrics different than expected: %s", diff) + } + + // keccak wasm program will be cached + keccakAsmEstimateSizeKb := uint64(583) + keccakProgramAddress := deployWasm(t, ctx, auth, l2client, rustFile("keccak")) + tx = l2info.PrepareTxTo("Owner", &keccakProgramAddress, l2info.TransferGas, nil, []byte{0x01}) + Require(t, l2client.SendTransaction(ctx, tx)) + _, err = EnsureTxSucceeded(ctx, l2client, tx) + Require(t, err) + lruMetrics = builder.L2.ExecNode.ExecEngine.GetWasmLruCacheMetrics() + expectedLruMetrics = &programs.WasmLruCacheMetrics{ + SizeKb: fallibleAsmEstimateSizeKb + keccakAsmEstimateSizeKb, + Count: 2, + } + if diff := cmp.Diff(lruMetrics, expectedLruMetrics); diff != "" { + t.Fatalf("lru cache metrics different than expected: %s", diff) + } + + // math wasm program will be cached, but since (fallible + keccak + math) > lruCacheSize, fallible will be evicted + mathAsmEstimateSizeKb := uint64(560) + mathProgramAddress := deployWasm(t, ctx, auth, l2client, rustFile("math")) + tx = l2info.PrepareTxTo("Owner", &mathProgramAddress, l2info.TransferGas, nil, []byte{0x01}) + Require(t, l2client.SendTransaction(ctx, tx)) + _, err = EnsureTxSucceeded(ctx, l2client, tx) + Require(t, err) + lruMetrics = builder.L2.ExecNode.ExecEngine.GetWasmLruCacheMetrics() + expectedLruMetrics = &programs.WasmLruCacheMetrics{ + SizeKb: keccakAsmEstimateSizeKb + mathAsmEstimateSizeKb, + Count: 2, + } + if diff := cmp.Diff(lruMetrics, expectedLruMetrics); diff != "" { + t.Fatalf("lru cache metrics different than expected: %s", diff) + } +} From 0ab8ebeb6b00c161b578fb454a8cb686cb364740 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 20 Aug 2024 10:14:14 -0300 Subject: [PATCH 108/489] Removes -Wall from #cgo CFLAGS --- arbcompress/native.go | 2 +- arbos/programs/native.go | 2 +- arbos/programs/native_api.go | 2 +- arbos/programs/testconstants.go | 2 +- execution/gethexec/executionengine.go | 2 +- validator/server_arb/machine.go | 2 +- validator/server_arb/nitro_machine.go | 2 +- validator/server_arb/preimage_resolver.go | 2 +- validator/server_arb/prover_interface.go | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/arbcompress/native.go b/arbcompress/native.go index 8244010979..f7b8f0b8e0 100644 --- a/arbcompress/native.go +++ b/arbcompress/native.go @@ -7,7 +7,7 @@ package arbcompress /* -#cgo CFLAGS: -g -Wall -I${SRCDIR}/../target/include/ +#cgo CFLAGS: -g -I${SRCDIR}/../target/include/ #cgo LDFLAGS: ${SRCDIR}/../target/lib/libstylus.a -lm #include "arbitrator.h" */ diff --git a/arbos/programs/native.go b/arbos/programs/native.go index c10cc50f30..1147a29d2d 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -7,7 +7,7 @@ package programs /* -#cgo CFLAGS: -g -Wall -I../../target/include/ +#cgo CFLAGS: -g -I../../target/include/ #cgo LDFLAGS: ${SRCDIR}/../../target/lib/libstylus.a -ldl -lm #include "arbitrator.h" diff --git a/arbos/programs/native_api.go b/arbos/programs/native_api.go index 6fbb630ef3..6cecb8ef63 100644 --- a/arbos/programs/native_api.go +++ b/arbos/programs/native_api.go @@ -7,7 +7,7 @@ package programs /* -#cgo CFLAGS: -g -Wall -I../../target/include/ +#cgo CFLAGS: -g -I../../target/include/ #cgo LDFLAGS: ${SRCDIR}/../../target/lib/libstylus.a -ldl -lm #include "arbitrator.h" diff --git a/arbos/programs/testconstants.go b/arbos/programs/testconstants.go index 1ab0e6e93b..44f69a52de 100644 --- a/arbos/programs/testconstants.go +++ b/arbos/programs/testconstants.go @@ -9,7 +9,7 @@ package programs // This file exists because cgo isn't allowed in tests /* -#cgo CFLAGS: -g -Wall -I../../target/include/ +#cgo CFLAGS: -g -I../../target/include/ #include "arbitrator.h" */ import "C" diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index dce3639d02..789a53e8e4 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -7,7 +7,7 @@ package gethexec /* -#cgo CFLAGS: -g -Wall -I../../target/include/ +#cgo CFLAGS: -g -I../../target/include/ #cgo LDFLAGS: ${SRCDIR}/../../target/lib/libstylus.a -ldl -lm #include "arbitrator.h" */ diff --git a/validator/server_arb/machine.go b/validator/server_arb/machine.go index adca9695e2..1e73e6b212 100644 --- a/validator/server_arb/machine.go +++ b/validator/server_arb/machine.go @@ -4,7 +4,7 @@ package server_arb /* -#cgo CFLAGS: -g -Wall -I../../target/include/ +#cgo CFLAGS: -g -I../../target/include/ #include "arbitrator.h" ResolvedPreimage preimageResolverC(size_t context, uint8_t preimageType, const uint8_t* hash); diff --git a/validator/server_arb/nitro_machine.go b/validator/server_arb/nitro_machine.go index 2b2cb230b6..926b1e8930 100644 --- a/validator/server_arb/nitro_machine.go +++ b/validator/server_arb/nitro_machine.go @@ -4,7 +4,7 @@ package server_arb /* -#cgo CFLAGS: -g -Wall -I../../target/include/ +#cgo CFLAGS: -g -I../../target/include/ #include "arbitrator.h" #include */ diff --git a/validator/server_arb/preimage_resolver.go b/validator/server_arb/preimage_resolver.go index cd4ea40e28..f01d79f4dd 100644 --- a/validator/server_arb/preimage_resolver.go +++ b/validator/server_arb/preimage_resolver.go @@ -4,7 +4,7 @@ package server_arb /* -#cgo CFLAGS: -g -Wall -I../../target/include/ +#cgo CFLAGS: -g -I../../target/include/ #include "arbitrator.h" extern ResolvedPreimage preimageResolver(size_t context, uint8_t preimageType, const uint8_t* hash); diff --git a/validator/server_arb/prover_interface.go b/validator/server_arb/prover_interface.go index bdd81ed588..3010d2138d 100644 --- a/validator/server_arb/prover_interface.go +++ b/validator/server_arb/prover_interface.go @@ -4,7 +4,7 @@ package server_arb /* -#cgo CFLAGS: -g -Wall -I../target/include/ +#cgo CFLAGS: -g -I../target/include/ #cgo LDFLAGS: ${SRCDIR}/../../target/lib/libstylus.a -ldl -lm #include "arbitrator.h" #include From 708417bd1a4c69d2e805eeb9cad8dd2cda92919f Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 20 Aug 2024 10:23:58 -0300 Subject: [PATCH 109/489] Improves comments on stylus cache --- arbitrator/stylus/src/cache.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index 2ef674bf6e..e7b4776bcd 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -67,8 +67,8 @@ impl WeightScale for CustomWeightScale { fn weight(&self, _key: &CacheKey, val: &CacheItem) -> usize { let mut w = val.asm_size_estimate_kb.try_into().unwrap(); if w > 0 { - // clru defines the weight of an entry as the number returned by this function plus one. - // Therefore, we need to subtract one from the weight to get the actual weight of the entry. + // clru defines that each entry consumes (weight + 1) of the cache capacity. + // Since we only want to use the weight as the size of the entry, we need to subtract 1. w -= 1; } return w; @@ -196,6 +196,7 @@ impl InitCache { let cache = cache!(); let count = cache.lru.len(); return LruCacheMetrics{ + // add 1 to each entry to account that we subtracted 1 in the weight calculation size_kb: (cache.lru.weight() + count).try_into().unwrap(), count: count.try_into().unwrap(), } From 6af08f46806ef7cce24f0d0a859ecb705e584435 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 20 Aug 2024 11:20:41 -0300 Subject: [PATCH 110/489] Avoid multiple functions in programs to get metrics --- arbos/programs/native.go | 12 ++++-------- arbos/programs/programs.go | 2 +- arbos/programs/wasm.go | 2 +- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/arbos/programs/native.go b/arbos/programs/native.go index 1147a29d2d..eba16a694a 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -267,14 +267,6 @@ func callProgram( return data, err } -// exposes for testing -func GetMetrics() { - metrics := C.stylus_get_lru_cache_metrics() - - stylusLRUCacheSizeKbGauge.Update(int64(metrics.size_kb)) - stylusLRUCacheSizeCountGauge.Update(int64(metrics.count)) -} - //export handleReqImpl func handleReqImpl(apiId usize, req_type u32, data *rustSlice, costPtr *u64, out_response *C.GoSliceData, out_raw_data *C.GoSliceData) { api := getApi(apiId) @@ -335,6 +327,10 @@ type WasmLruCacheMetrics struct { // exported for testing func GetWasmLruCacheMetrics() *WasmLruCacheMetrics { metrics := C.stylus_get_lru_cache_metrics() + + stylusLRUCacheSizeKbGauge.Update(int64(metrics.size_kb)) + stylusLRUCacheSizeCountGauge.Update(int64(metrics.count)) + return &WasmLruCacheMetrics{ SizeKb: uint64(metrics.size_kb), Count: uint64(metrics.count), diff --git a/arbos/programs/programs.go b/arbos/programs/programs.go index 3909b0fceb..b324c98d0a 100644 --- a/arbos/programs/programs.go +++ b/arbos/programs/programs.go @@ -264,7 +264,7 @@ func (p Programs) CallProgram( maxGasToReturn := startingGas - evmCost contract.Gas = am.MinInt(contract.Gas, maxGasToReturn) } - GetMetrics() + GetWasmLruCacheMetrics() return ret, err } diff --git a/arbos/programs/wasm.go b/arbos/programs/wasm.go index 3889d47f34..849ed88f42 100644 --- a/arbos/programs/wasm.go +++ b/arbos/programs/wasm.go @@ -152,7 +152,7 @@ func callProgram( return retData, err } -func GetMetrics() {} +func GetWasmLruCacheMetrics() {} func CallProgramLoop( moduleHash common.Hash, From 9792757163c83b69559bc9b68b6d5b1dcf3e57a9 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 20 Aug 2024 11:54:21 -0300 Subject: [PATCH 111/489] Changes TestWasLruCache comparisons --- system_tests/program_test.go | 44 ++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/system_tests/program_test.go b/system_tests/program_test.go index 034e1c5fd3..0c48bf7661 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -30,7 +30,6 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" - "github.com/google/go-cmp/cmp" "github.com/offchainlabs/nitro/arbcompress" "github.com/offchainlabs/nitro/arbos/programs" "github.com/offchainlabs/nitro/arbos/util" @@ -2018,9 +2017,11 @@ func TestWasmLruCache(t *testing.T) { builder.L2.ExecNode.ExecEngine.ClearWasmLruCache() lruMetrics := builder.L2.ExecNode.ExecEngine.GetWasmLruCacheMetrics() - expectedLruMetrics := &programs.WasmLruCacheMetrics{} - if diff := cmp.Diff(lruMetrics, expectedLruMetrics); diff != "" { - t.Fatalf("lru cache metrics different than expected: %s", diff) + if lruMetrics.Count != 0 { + t.Fatalf("lruMetrics.Count, expected: %v, actual: %v", 0, lruMetrics.Count) + } + if lruMetrics.SizeKb != 0 { + t.Fatalf("lruMetrics.SizeKb, expected: %v, actual: %v", 0, lruMetrics.SizeKb) } lruCacheSize := uint32(500) @@ -2034,9 +2035,11 @@ func TestWasmLruCache(t *testing.T) { _, err := EnsureTxSucceeded(ctx, l2client, tx) Require(t, err) lruMetrics = builder.L2.ExecNode.ExecEngine.GetWasmLruCacheMetrics() - expectedLruMetrics = &programs.WasmLruCacheMetrics{} - if diff := cmp.Diff(lruMetrics, expectedLruMetrics); diff != "" { - t.Fatalf("lru cache metrics different than expected: %s", diff) + if lruMetrics.Count != 0 { + t.Fatalf("lruMetrics.Count, expected: %v, actual: %v", 0, lruMetrics.Count) + } + if lruMetrics.SizeKb != 0 { + t.Fatalf("lruMetrics.SizeKb, expected: %v, actual: %v", 0, lruMetrics.SizeKb) } // resize lru cache @@ -2049,12 +2052,11 @@ func TestWasmLruCache(t *testing.T) { _, err = EnsureTxSucceeded(ctx, l2client, tx) Require(t, err) lruMetrics = builder.L2.ExecNode.ExecEngine.GetWasmLruCacheMetrics() - expectedLruMetrics = &programs.WasmLruCacheMetrics{ - SizeKb: fallibleAsmEstimateSizeKb, - Count: 1, + if lruMetrics.Count != 1 { + t.Fatalf("lruMetrics.Count, expected: %v, actual: %v", 1, lruMetrics.Count) } - if diff := cmp.Diff(lruMetrics, expectedLruMetrics); diff != "" { - t.Fatalf("lru cache metrics different than expected: %s", diff) + if lruMetrics.SizeKb != fallibleAsmEstimateSizeKb { + t.Fatalf("lruMetrics.SizeKb, expected: %v, actual: %v", fallibleAsmEstimateSizeKb, lruMetrics.SizeKb) } // keccak wasm program will be cached @@ -2065,12 +2067,11 @@ func TestWasmLruCache(t *testing.T) { _, err = EnsureTxSucceeded(ctx, l2client, tx) Require(t, err) lruMetrics = builder.L2.ExecNode.ExecEngine.GetWasmLruCacheMetrics() - expectedLruMetrics = &programs.WasmLruCacheMetrics{ - SizeKb: fallibleAsmEstimateSizeKb + keccakAsmEstimateSizeKb, - Count: 2, + if lruMetrics.Count != 2 { + t.Fatalf("lruMetrics.Count, expected: %v, actual: %v", 2, lruMetrics.Count) } - if diff := cmp.Diff(lruMetrics, expectedLruMetrics); diff != "" { - t.Fatalf("lru cache metrics different than expected: %s", diff) + if lruMetrics.SizeKb != fallibleAsmEstimateSizeKb+keccakAsmEstimateSizeKb { + t.Fatalf("lruMetrics.SizeKb, expected: %v, actual: %v", fallibleAsmEstimateSizeKb+keccakAsmEstimateSizeKb, lruMetrics.SizeKb) } // math wasm program will be cached, but since (fallible + keccak + math) > lruCacheSize, fallible will be evicted @@ -2081,11 +2082,10 @@ func TestWasmLruCache(t *testing.T) { _, err = EnsureTxSucceeded(ctx, l2client, tx) Require(t, err) lruMetrics = builder.L2.ExecNode.ExecEngine.GetWasmLruCacheMetrics() - expectedLruMetrics = &programs.WasmLruCacheMetrics{ - SizeKb: keccakAsmEstimateSizeKb + mathAsmEstimateSizeKb, - Count: 2, + if lruMetrics.Count != 2 { + t.Fatalf("lruMetrics.Count, expected: %v, actual: %v", 2, lruMetrics.Count) } - if diff := cmp.Diff(lruMetrics, expectedLruMetrics); diff != "" { - t.Fatalf("lru cache metrics different than expected: %s", diff) + if lruMetrics.SizeKb != keccakAsmEstimateSizeKb+mathAsmEstimateSizeKb { + t.Fatalf("lruMetrics.SizeKb, expected: %v, actual: %v", keccakAsmEstimateSizeKb+mathAsmEstimateSizeKb, lruMetrics.SizeKb) } } From f65a08447023b7303becda0aa24f253d06694777 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 20 Aug 2024 11:55:25 -0300 Subject: [PATCH 112/489] Stylus LRU cache hits, misses, does not fit metrics --- arbitrator/stylus/src/cache.rs | 45 ++++++++++++++++++++++++++++++---- arbos/programs/native.go | 24 +++++++++++++----- 2 files changed, 58 insertions(+), 11 deletions(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index e7b4776bcd..fe04bf854d 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -23,9 +23,16 @@ macro_rules! cache { }; } +pub struct LruCounters { + pub hits: u64, + pub misses: u64, + pub does_not_fit: u64, +} + pub struct InitCache { long_term: HashMap, lru: CLruCache, + lru_counters: LruCounters, } #[derive(Clone, Copy, Hash, PartialEq, Eq)] @@ -79,6 +86,9 @@ impl WeightScale for CustomWeightScale { pub struct LruCacheMetrics { pub size_kb: u64, pub count: u64, + pub hits: u64, + pub misses: u64, + pub does_not_fit: u64, } impl InitCache { @@ -91,6 +101,11 @@ impl InitCache { Self { long_term: HashMap::new(), lru: CLruCache::with_config(CLruCacheConfig::new(NonZeroUsize::new(size).unwrap()).with_scale(CustomWeightScale)), + lru_counters: LruCounters { + hits: 0, + misses: 0, + does_not_fit: 0, + }, } } @@ -112,8 +127,11 @@ impl InitCache { // See if the item is in the LRU cache, promoting if so if let Some(item) = cache.lru.get(&key) { - return Some(item.data()); + let data = item.data(); + cache.lru_counters.hits += 1; + return Some(data); } + cache.lru_counters.misses += 1; None } @@ -153,7 +171,10 @@ impl InitCache { let mut cache = cache!(); if long_term_tag != Self::ARBOS_TAG { match cache.lru.put_with_weight(key, item) { - Err(_) => println!("Failed to insert into LRU cache, item too large"), + Err(_) => { + cache.lru_counters.does_not_fit += 1; + println!("Failed to insert into LRU cache, item too large"); + } Ok(_) => (), }; } else { @@ -193,13 +214,27 @@ impl InitCache { } pub fn get_lru_metrics() -> LruCacheMetrics { - let cache = cache!(); + let mut cache = cache!(); + let count = cache.lru.len(); - return LruCacheMetrics{ + let metrics = LruCacheMetrics{ // add 1 to each entry to account that we subtracted 1 in the weight calculation size_kb: (cache.lru.weight() + count).try_into().unwrap(), count: count.try_into().unwrap(), - } + + hits: cache.lru_counters.hits, + misses: cache.lru_counters.misses, + does_not_fit: cache.lru_counters.does_not_fit, + }; + + // empty counters + cache.lru_counters = LruCounters { + hits: 0, + misses: 0, + does_not_fit: 0, + }; + + return metrics } // only used for testing diff --git a/arbos/programs/native.go b/arbos/programs/native.go index eba16a694a..ef4de07ffa 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -47,8 +47,11 @@ type rustBytes = C.RustBytes type rustSlice = C.RustSlice var ( - stylusLRUCacheSizeKbGauge = metrics.NewRegisteredGauge("arb/arbos/stylus/cache/lru/size_kilobytes", nil) - stylusLRUCacheSizeCountGauge = metrics.NewRegisteredGauge("arb/arbos/stylus/cache/lru/count", nil) + stylusLRUCacheSizeKbGauge = metrics.NewRegisteredGauge("arb/arbos/stylus/cache/lru/size_kilobytes", nil) + stylusLRUCacheSizeCountGauge = metrics.NewRegisteredGauge("arb/arbos/stylus/cache/lru/count", nil) + stylusLRUCacheSizeHitsCounter = metrics.NewRegisteredCounter("arb/arbos/stylus/cache/lru/hits", nil) + stylusLRUCacheSizeMissesCounter = metrics.NewRegisteredCounter("arb/arbos/stylus/cache/lru/misses", nil) + stylusLRUCacheSizeDoesNotFitCounter = metrics.NewRegisteredCounter("arb/arbos/stylus/cache/lru/does_not_fit", nil) ) func activateProgram( @@ -320,8 +323,11 @@ func ResizeWasmLruCache(sizeKb uint32) { // exported for testing type WasmLruCacheMetrics struct { - SizeKb uint64 - Count uint64 + SizeKb uint64 + Count uint64 + Hits uint64 + Misses uint64 + DoesNotFit uint64 } // exported for testing @@ -330,10 +336,16 @@ func GetWasmLruCacheMetrics() *WasmLruCacheMetrics { stylusLRUCacheSizeKbGauge.Update(int64(metrics.size_kb)) stylusLRUCacheSizeCountGauge.Update(int64(metrics.count)) + stylusLRUCacheSizeHitsCounter.Inc(int64(metrics.hits)) + stylusLRUCacheSizeMissesCounter.Inc(int64(metrics.misses)) + stylusLRUCacheSizeDoesNotFitCounter.Inc(int64(metrics.does_not_fit)) return &WasmLruCacheMetrics{ - SizeKb: uint64(metrics.size_kb), - Count: uint64(metrics.count), + SizeKb: uint64(metrics.size_kb), + Count: uint64(metrics.count), + Hits: uint64(metrics.hits), + Misses: uint64(metrics.misses), + DoesNotFit: uint64(metrics.does_not_fit), } } From 6bf148d6c02473f68fdc2892980d1a51e5f09d8a Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 20 Aug 2024 12:10:37 -0300 Subject: [PATCH 113/489] Retrieves stylus cache metrics periodically --- arbos/programs/programs.go | 1 - execution/gethexec/executionengine.go | 10 ++++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/arbos/programs/programs.go b/arbos/programs/programs.go index b324c98d0a..32453dd529 100644 --- a/arbos/programs/programs.go +++ b/arbos/programs/programs.go @@ -264,7 +264,6 @@ func (p Programs) CallProgram( maxGasToReturn := startingGas - evmCost contract.Gas = am.MinInt(contract.Gas, maxGasToReturn) } - GetWasmLruCacheMetrics() return ret, err } diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index 789a53e8e4..5ee9e2202e 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -978,4 +978,14 @@ func (s *ExecutionEngine) Start(ctx_in context.Context) { } } }) + s.LaunchThread(func(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case <-time.After(time.Minute): + s.GetWasmLruCacheMetrics() + } + } + }) } From e2d9e8d24ba629f11769ba4b1eb5c5ee78291a03 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 20 Aug 2024 12:12:14 -0300 Subject: [PATCH 114/489] Simplifies WasmLruCacheMetrics --- arbos/programs/native.go | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/arbos/programs/native.go b/arbos/programs/native.go index ef4de07ffa..1a7b616f00 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -323,11 +323,8 @@ func ResizeWasmLruCache(sizeKb uint32) { // exported for testing type WasmLruCacheMetrics struct { - SizeKb uint64 - Count uint64 - Hits uint64 - Misses uint64 - DoesNotFit uint64 + SizeKb uint64 + Count uint64 } // exported for testing @@ -341,11 +338,8 @@ func GetWasmLruCacheMetrics() *WasmLruCacheMetrics { stylusLRUCacheSizeDoesNotFitCounter.Inc(int64(metrics.does_not_fit)) return &WasmLruCacheMetrics{ - SizeKb: uint64(metrics.size_kb), - Count: uint64(metrics.count), - Hits: uint64(metrics.hits), - Misses: uint64(metrics.misses), - DoesNotFit: uint64(metrics.does_not_fit), + SizeKb: uint64(metrics.size_kb), + Count: uint64(metrics.count), } } From e881d2bfdfbd93ef87521a11a913b43d28ca64c9 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 20 Aug 2024 12:16:34 -0300 Subject: [PATCH 115/489] Fixes clear_lru_cache --- arbitrator/stylus/src/cache.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index fe04bf854d..7e419c49b8 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -239,6 +239,12 @@ impl InitCache { // only used for testing pub fn clear_lru_cache() { - cache!().lru.clear(); + let mut cache = cache!(); + cache.lru.clear(); + cache.lru_counters = LruCounters { + hits: 0, + misses: 0, + does_not_fit: 0, + }; } } From 4ac5698fbe6b4234c9124c399f52496f03fb8239 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 20 Aug 2024 14:45:36 -0300 Subject: [PATCH 116/489] Fixes testcompile.go --- arbos/programs/testcompile.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arbos/programs/testcompile.go b/arbos/programs/testcompile.go index 1daf470620..57a5ce26b1 100644 --- a/arbos/programs/testcompile.go +++ b/arbos/programs/testcompile.go @@ -9,7 +9,7 @@ package programs // This file exists because cgo isn't allowed in tests /* -#cgo CFLAGS: -g -Wall -I../../target/include/ +#cgo CFLAGS: -g -I../../target/include/ #include "arbitrator.h" typedef uint16_t u16; @@ -244,6 +244,7 @@ func testCompileLoad() error { status := userStatus(C.stylus_call( goSlice(localAsm), + u32(1), goSlice(calldata), progParams.encode(), reqHandler, From 43e0bf64d479abb386a74b4b2b32cb7a3c7f2c36 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 20 Aug 2024 15:28:08 -0300 Subject: [PATCH 117/489] Adjusts default stylus lru cache capacity --- arbitrator/stylus/src/cache.rs | 2 +- execution/gethexec/blockchain.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index 7e419c49b8..ca5212a300 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -14,7 +14,7 @@ use std::hash::RandomState; use crate::target_cache::target_native; lazy_static! { - static ref INIT_CACHE: Mutex = Mutex::new(InitCache::new(256 * 10 * 1024)); + static ref INIT_CACHE: Mutex = Mutex::new(InitCache::new(256 * 1024)); } macro_rules! cache { diff --git a/execution/gethexec/blockchain.go b/execution/gethexec/blockchain.go index da6a22bdb4..21ac04d40b 100644 --- a/execution/gethexec/blockchain.go +++ b/execution/gethexec/blockchain.go @@ -75,7 +75,7 @@ var DefaultCachingConfig = CachingConfig{ SnapshotRestoreGasLimit: 300_000_000_000, MaxNumberOfBlocksToSkipStateSaving: 0, MaxAmountOfGasToSkipStateSaving: 0, - StylusLRUCacheSizeMb: 10, + StylusLRUCacheSizeMb: 256, StateScheme: rawdb.HashScheme, StateHistory: getStateHistory(DefaultSequencerConfig.MaxBlockSpeed), } From 516eceba14897523ecdd931d186fd298b9c6a3b0 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 20 Aug 2024 15:32:18 -0300 Subject: [PATCH 118/489] Updates comment in stylus cache weight computation --- arbitrator/stylus/src/cache.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index ca5212a300..431b021ffc 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -75,7 +75,7 @@ impl WeightScale for CustomWeightScale { let mut w = val.asm_size_estimate_kb.try_into().unwrap(); if w > 0 { // clru defines that each entry consumes (weight + 1) of the cache capacity. - // Since we only want to use the weight as the size of the entry, we need to subtract 1. + // We subtract 1 since we only want to use the weight as the size of the entry. w -= 1; } return w; @@ -220,6 +220,7 @@ impl InitCache { let metrics = LruCacheMetrics{ // add 1 to each entry to account that we subtracted 1 in the weight calculation size_kb: (cache.lru.weight() + count).try_into().unwrap(), + count: count.try_into().unwrap(), hits: cache.lru_counters.hits, From 0bf91916066934f2ce93798ecd393bb3d8a5aa6a Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 20 Aug 2024 15:33:32 -0300 Subject: [PATCH 119/489] Rename asm_size_estimate to asm_size_estimate_kb --- arbitrator/stylus/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arbitrator/stylus/src/lib.rs b/arbitrator/stylus/src/lib.rs index 576d041c28..ceb72440cb 100644 --- a/arbitrator/stylus/src/lib.rs +++ b/arbitrator/stylus/src/lib.rs @@ -319,12 +319,12 @@ pub extern "C" fn stylus_cache_lru_resize(size_kb: u32) { pub unsafe extern "C" fn stylus_cache_module( module: GoSliceData, module_hash: Bytes32, - asm_size_estimate: u32, + asm_size_estimate_kb: u32, version: u16, arbos_tag: u32, debug: bool, ) { - if let Err(error) = InitCache::insert(module_hash, module.slice(), asm_size_estimate, version, arbos_tag, debug) { + if let Err(error) = InitCache::insert(module_hash, module.slice(), asm_size_estimate_kb, version, arbos_tag, debug) { panic!("tried to cache invalid asm!: {error}"); } } From 38691b168fe9d4dc4f1c4b5a23fb40bf62c9ba4e Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 20 Aug 2024 15:36:41 -0300 Subject: [PATCH 120/489] Improves comments on why to empty counters in get_lru_metrics call --- arbitrator/stylus/src/cache.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index 431b021ffc..3683421a02 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -228,7 +228,9 @@ impl InitCache { does_not_fit: cache.lru_counters.does_not_fit, }; - // empty counters + // Empty counters. + // go side, which is the only consumer of this function besides tests, + // will read those counters and increment its own prometheus counters with them. cache.lru_counters = LruCounters { hits: 0, misses: 0, From 42631eab47fc2d4067c98b23b14939131e377509 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 20 Aug 2024 15:38:03 -0300 Subject: [PATCH 121/489] Rename asm_size_estimate to asm_size_estimate_kb --- arbitrator/stylus/src/lib.rs | 4 ++-- arbitrator/stylus/src/native.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/arbitrator/stylus/src/lib.rs b/arbitrator/stylus/src/lib.rs index ceb72440cb..25d8638764 100644 --- a/arbitrator/stylus/src/lib.rs +++ b/arbitrator/stylus/src/lib.rs @@ -256,7 +256,7 @@ pub unsafe extern "C" fn stylus_target_set( #[no_mangle] pub unsafe extern "C" fn stylus_call( module: GoSliceData, - asm_size_estimate: u32, + asm_size_estimate_kb: u32, calldata: GoSliceData, config: StylusConfig, req_handler: NativeRequestHandler, @@ -277,7 +277,7 @@ pub unsafe extern "C" fn stylus_call( let instance = unsafe { NativeInstance::deserialize_cached( module, - asm_size_estimate, + asm_size_estimate_kb, config.version, evm_api, evm_data, diff --git a/arbitrator/stylus/src/native.rs b/arbitrator/stylus/src/native.rs index d150d71ccc..7c2f7ab777 100644 --- a/arbitrator/stylus/src/native.rs +++ b/arbitrator/stylus/src/native.rs @@ -112,7 +112,7 @@ impl> NativeInstance { /// `module` must represent a valid module. pub unsafe fn deserialize_cached( module: &[u8], - asm_size_estimate: u32, + asm_size_estimate_kb: u32, version: u16, evm: E, evm_data: EvmData, @@ -130,7 +130,7 @@ impl> NativeInstance { long_term_tag = 0; } let (module, store) = - InitCache::insert(module_hash, module, asm_size_estimate, version, long_term_tag, debug)?; + InitCache::insert(module_hash, module, asm_size_estimate_kb, version, long_term_tag, debug)?; Self::from_module(module, store, env) } From fa9a932df424fe23031d1bf25e5c41f0e906d8fd Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 20 Aug 2024 15:50:48 -0300 Subject: [PATCH 122/489] Adds comment on periodic procedure that retrieves stylus lru cache metrics --- execution/gethexec/executionengine.go | 1 + 1 file changed, 1 insertion(+) diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index 5ee9e2202e..affdcec704 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -978,6 +978,7 @@ func (s *ExecutionEngine) Start(ctx_in context.Context) { } } }) + // periodically update stylus lru cache metrics s.LaunchThread(func(ctx context.Context) { for { select { From d9bf5848735214c7ad87345f06bf9d24390b3391 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 20 Aug 2024 15:53:35 -0300 Subject: [PATCH 123/489] Updates comment TestWasmLruCache --- system_tests/program_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system_tests/program_test.go b/system_tests/program_test.go index 0c48bf7661..c867aab824 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -2074,7 +2074,7 @@ func TestWasmLruCache(t *testing.T) { t.Fatalf("lruMetrics.SizeKb, expected: %v, actual: %v", fallibleAsmEstimateSizeKb+keccakAsmEstimateSizeKb, lruMetrics.SizeKb) } - // math wasm program will be cached, but since (fallible + keccak + math) > lruCacheSize, fallible will be evicted + // math wasm program will be cached, but fallible will be evicted since (fallible + keccak + math) > lruCacheSize mathAsmEstimateSizeKb := uint64(560) mathProgramAddress := deployWasm(t, ctx, auth, l2client, rustFile("math")) tx = l2info.PrepareTxTo("Owner", &mathProgramAddress, l2info.TransferGas, nil, []byte{0x01}) From ce4667625e4601f78b6af05a59225c9d3f5d79e0 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 20 Aug 2024 15:56:48 -0300 Subject: [PATCH 124/489] Renames StylusLRUCacheSizeMb to StylusLRUCacheSize --- arbnode/inbox_test.go | 2 +- execution/gethexec/blockchain.go | 6 +++--- execution/gethexec/node.go | 2 +- system_tests/common_test.go | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/arbnode/inbox_test.go b/arbnode/inbox_test.go index 42661c44a6..e0c97bb871 100644 --- a/arbnode/inbox_test.go +++ b/arbnode/inbox_test.go @@ -74,7 +74,7 @@ func NewTransactionStreamerForTest(t *testing.T, ownerAddress common.Address) (* } stylusTargetConfig := &gethexec.DefaultStylusTargetConfig Require(t, stylusTargetConfig.Validate()) // pre-processes config (i.a. parses wasmTargets) - if err := execEngine.Initialize(gethexec.DefaultCachingConfig.StylusLRUCacheSizeMb, &gethexec.DefaultStylusTargetConfig); err != nil { + if err := execEngine.Initialize(gethexec.DefaultCachingConfig.StylusLRUCacheSize, &gethexec.DefaultStylusTargetConfig); err != nil { Fail(t, err) } execSeq := &execClientWrapper{execEngine, t} diff --git a/execution/gethexec/blockchain.go b/execution/gethexec/blockchain.go index 21ac04d40b..e0e907683d 100644 --- a/execution/gethexec/blockchain.go +++ b/execution/gethexec/blockchain.go @@ -37,7 +37,7 @@ type CachingConfig struct { SnapshotRestoreGasLimit uint64 `koanf:"snapshot-restore-gas-limit"` MaxNumberOfBlocksToSkipStateSaving uint32 `koanf:"max-number-of-blocks-to-skip-state-saving"` MaxAmountOfGasToSkipStateSaving uint64 `koanf:"max-amount-of-gas-to-skip-state-saving"` - StylusLRUCacheSizeMb uint32 `koanf:"stylus-lru-cache-size"` + StylusLRUCacheSize uint32 `koanf:"stylus-lru-cache-size"` StateScheme string `koanf:"state-scheme"` StateHistory uint64 `koanf:"state-history"` } @@ -54,7 +54,7 @@ func CachingConfigAddOptions(prefix string, f *flag.FlagSet) { f.Uint64(prefix+".snapshot-restore-gas-limit", DefaultCachingConfig.SnapshotRestoreGasLimit, "maximum gas rolled back to recover snapshot") f.Uint32(prefix+".max-number-of-blocks-to-skip-state-saving", DefaultCachingConfig.MaxNumberOfBlocksToSkipStateSaving, "maximum number of blocks to skip state saving to persistent storage (archive node only) -- warning: this option seems to cause issues") f.Uint64(prefix+".max-amount-of-gas-to-skip-state-saving", DefaultCachingConfig.MaxAmountOfGasToSkipStateSaving, "maximum amount of gas in blocks to skip saving state to Persistent storage (archive node only) -- warning: this option seems to cause issues") - f.Uint32(prefix+".stylus-lru-cache-size", DefaultCachingConfig.StylusLRUCacheSizeMb, "capacity, in megabytes, of the LRU cache that keeps initialized stylus programs") + f.Uint32(prefix+".stylus-lru-cache-size", DefaultCachingConfig.StylusLRUCacheSize, "capacity, in megabytes, of the LRU cache that keeps initialized stylus programs") f.String(prefix+".state-scheme", DefaultCachingConfig.StateScheme, "scheme to use for state trie storage (hash, path)") f.Uint64(prefix+".state-history", DefaultCachingConfig.StateHistory, "number of recent blocks to retain state history for (path state-scheme only)") } @@ -75,7 +75,7 @@ var DefaultCachingConfig = CachingConfig{ SnapshotRestoreGasLimit: 300_000_000_000, MaxNumberOfBlocksToSkipStateSaving: 0, MaxAmountOfGasToSkipStateSaving: 0, - StylusLRUCacheSizeMb: 256, + StylusLRUCacheSize: 256, StateScheme: rawdb.HashScheme, StateHistory: getStateHistory(DefaultSequencerConfig.MaxBlockSpeed), } diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index b602dbd323..93bb254ed8 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -308,7 +308,7 @@ func (n *ExecutionNode) MarkFeedStart(to arbutil.MessageIndex) { func (n *ExecutionNode) Initialize(ctx context.Context) error { config := n.ConfigFetcher() - err := n.ExecEngine.Initialize(config.Caching.StylusLRUCacheSizeMb, &config.StylusTarget) + err := n.ExecEngine.Initialize(config.Caching.StylusLRUCacheSize, &config.StylusTarget) if err != nil { return fmt.Errorf("error initializing execution engine: %w", err) } diff --git a/system_tests/common_test.go b/system_tests/common_test.go index c1e9ff384b..363f1050fb 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -166,7 +166,7 @@ var TestCachingConfig = gethexec.CachingConfig{ SnapshotRestoreGasLimit: 300_000_000_000, MaxNumberOfBlocksToSkipStateSaving: 0, MaxAmountOfGasToSkipStateSaving: 0, - StylusLRUCacheSizeMb: 0, + StylusLRUCacheSize: 0, StateScheme: env.GetTestStateScheme(), } From 7cb4fa1ae84c66d9aa8f260478af75a8434b0d28 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 20 Aug 2024 16:07:00 -0300 Subject: [PATCH 125/489] Updates lru cache metrics data sizes --- arbitrator/stylus/src/cache.rs | 16 ++++++++-------- arbos/programs/native.go | 8 ++++---- system_tests/program_test.go | 6 +++--- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index 3683421a02..11ec55c6a7 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -24,9 +24,9 @@ macro_rules! cache { } pub struct LruCounters { - pub hits: u64, - pub misses: u64, - pub does_not_fit: u64, + pub hits: u32, + pub misses: u32, + pub does_not_fit: u32, } pub struct InitCache { @@ -84,11 +84,11 @@ impl WeightScale for CustomWeightScale { #[repr(C)] pub struct LruCacheMetrics { - pub size_kb: u64, - pub count: u64, - pub hits: u64, - pub misses: u64, - pub does_not_fit: u64, + pub size_kb: u32, + pub count: u32, + pub hits: u32, + pub misses: u32, + pub does_not_fit: u32, } impl InitCache { diff --git a/arbos/programs/native.go b/arbos/programs/native.go index 1a7b616f00..0f3e28287d 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -323,8 +323,8 @@ func ResizeWasmLruCache(sizeKb uint32) { // exported for testing type WasmLruCacheMetrics struct { - SizeKb uint64 - Count uint64 + SizeKb uint32 + Count uint32 } // exported for testing @@ -338,8 +338,8 @@ func GetWasmLruCacheMetrics() *WasmLruCacheMetrics { stylusLRUCacheSizeDoesNotFitCounter.Inc(int64(metrics.does_not_fit)) return &WasmLruCacheMetrics{ - SizeKb: uint64(metrics.size_kb), - Count: uint64(metrics.count), + SizeKb: uint32(metrics.size_kb), + Count: uint32(metrics.count), } } diff --git a/system_tests/program_test.go b/system_tests/program_test.go index c867aab824..108cec6647 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -2028,7 +2028,7 @@ func TestWasmLruCache(t *testing.T) { builder.L2.ExecNode.ExecEngine.ResizeWasmLruCache(lruCacheSize) // fallible wasm program will not be cached since its size is greater than lruCacheSize - fallibleAsmEstimateSizeKb := uint64(551) + fallibleAsmEstimateSizeKb := uint32(551) fallibleProgramAddress := deployWasm(t, ctx, auth, l2client, rustFile("fallible")) tx := l2info.PrepareTxTo("Owner", &fallibleProgramAddress, l2info.TransferGas, nil, []byte{0x01}) Require(t, l2client.SendTransaction(ctx, tx)) @@ -2060,7 +2060,7 @@ func TestWasmLruCache(t *testing.T) { } // keccak wasm program will be cached - keccakAsmEstimateSizeKb := uint64(583) + keccakAsmEstimateSizeKb := uint32(583) keccakProgramAddress := deployWasm(t, ctx, auth, l2client, rustFile("keccak")) tx = l2info.PrepareTxTo("Owner", &keccakProgramAddress, l2info.TransferGas, nil, []byte{0x01}) Require(t, l2client.SendTransaction(ctx, tx)) @@ -2075,7 +2075,7 @@ func TestWasmLruCache(t *testing.T) { } // math wasm program will be cached, but fallible will be evicted since (fallible + keccak + math) > lruCacheSize - mathAsmEstimateSizeKb := uint64(560) + mathAsmEstimateSizeKb := uint32(560) mathProgramAddress := deployWasm(t, ctx, auth, l2client, rustFile("math")) tx = l2info.PrepareTxTo("Owner", &mathProgramAddress, l2info.TransferGas, nil, []byte{0x01}) Require(t, l2client.SendTransaction(ctx, tx)) From 21217f1c5ce9261bdd525b5882424851b38260e0 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 20 Aug 2024 20:38:53 -0300 Subject: [PATCH 126/489] Simplifies weight computation by using saturating_sub --- arbitrator/stylus/src/cache.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index 11ec55c6a7..db199414df 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -72,13 +72,9 @@ impl CacheItem { struct CustomWeightScale; impl WeightScale for CustomWeightScale { fn weight(&self, _key: &CacheKey, val: &CacheItem) -> usize { - let mut w = val.asm_size_estimate_kb.try_into().unwrap(); - if w > 0 { - // clru defines that each entry consumes (weight + 1) of the cache capacity. - // We subtract 1 since we only want to use the weight as the size of the entry. - w -= 1; - } - return w; + // clru defines that each entry consumes (weight + 1) of the cache capacity. + // We subtract 1 since we only want to use the weight as the size of the entry. + val.asm_size_estimate_kb.saturating_sub(1).try_into().unwrap() } } From 7e6b7596ff5da5d707a11fcb238f5e4632e3d526 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 20 Aug 2024 20:43:47 -0300 Subject: [PATCH 127/489] Uses if let instead of match --- arbitrator/stylus/src/cache.rs | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index db199414df..6061a72cd2 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -166,12 +166,9 @@ impl InitCache { let data = item.data(); let mut cache = cache!(); if long_term_tag != Self::ARBOS_TAG { - match cache.lru.put_with_weight(key, item) { - Err(_) => { - cache.lru_counters.does_not_fit += 1; - println!("Failed to insert into LRU cache, item too large"); - } - Ok(_) => (), + if let Err(_) = cache.lru.put_with_weight(key, item) { + cache.lru_counters.does_not_fit += 1; + println!("Failed to insert into LRU cache, item too large"); }; } else { cache.long_term.insert(key, item); @@ -187,10 +184,9 @@ impl InitCache { let key = CacheKey::new(module_hash, version, debug); let mut cache = cache!(); if let Some(item) = cache.long_term.remove(&key) { - match cache.lru.put_with_weight(key, item) { - Err(_) => println!("Failed to insert into LRU cache, item too large"), - Ok(_) => (), - }; + if let Err(_) = cache.lru.put_with_weight(key, item) { + println!("Failed to insert into LRU cache, item too large"); + } } } @@ -202,10 +198,9 @@ impl InitCache { let cache = &mut *cache; for (key, item) in cache.long_term.drain() { // not all will fit, just a heuristic - match cache.lru.put_with_weight(key, item) { - Err(_) => println!("Failed to insert into LRU cache, item too large"), - Ok(_) => (), - }; + if let Err(_) = cache.lru.put_with_weight(key, item) { + println!("Failed to insert into LRU cache, item too large"); + } } } From fe2377c75357b7aa75fda26afda9e1399d60f27e Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 20 Aug 2024 20:44:19 -0300 Subject: [PATCH 128/489] Removes unused return --- arbitrator/stylus/src/cache.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index 6061a72cd2..033c65c84f 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -228,7 +228,7 @@ impl InitCache { does_not_fit: 0, }; - return metrics + metrics } // only used for testing From e439bc88c2e005c3abb11889d1f9851adf083636 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 20 Aug 2024 20:54:49 -0300 Subject: [PATCH 129/489] Creates a DOES_NOT_FIT_MSG static --- arbitrator/stylus/src/cache.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index 033c65c84f..831cc6f2d1 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -93,6 +93,8 @@ impl InitCache { // that will never modify long_term state const ARBOS_TAG: u32 = 1; + const DOES_NOT_FIT_MSG: &'static str = "Failed to insert into LRU cache, item too large"; + fn new(size: usize) -> Self { Self { long_term: HashMap::new(), @@ -168,7 +170,7 @@ impl InitCache { if long_term_tag != Self::ARBOS_TAG { if let Err(_) = cache.lru.put_with_weight(key, item) { cache.lru_counters.does_not_fit += 1; - println!("Failed to insert into LRU cache, item too large"); + println!("{}", Self::DOES_NOT_FIT_MSG); }; } else { cache.long_term.insert(key, item); @@ -185,7 +187,7 @@ impl InitCache { let mut cache = cache!(); if let Some(item) = cache.long_term.remove(&key) { if let Err(_) = cache.lru.put_with_weight(key, item) { - println!("Failed to insert into LRU cache, item too large"); + println!("{}", Self::DOES_NOT_FIT_MSG); } } } @@ -199,7 +201,7 @@ impl InitCache { for (key, item) in cache.long_term.drain() { // not all will fit, just a heuristic if let Err(_) = cache.lru.put_with_weight(key, item) { - println!("Failed to insert into LRU cache, item too large"); + println!("{}", Self::DOES_NOT_FIT_MSG); } } } From 281f608857b09bd72ba54bc8b216ae651d372aa6 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 20 Aug 2024 21:18:29 -0300 Subject: [PATCH 130/489] Uses is_err instead of let Err(_) --- arbitrator/stylus/src/cache.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index 831cc6f2d1..bdecdd724f 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -168,7 +168,7 @@ impl InitCache { let data = item.data(); let mut cache = cache!(); if long_term_tag != Self::ARBOS_TAG { - if let Err(_) = cache.lru.put_with_weight(key, item) { + if cache.lru.put_with_weight(key, item).is_err() { cache.lru_counters.does_not_fit += 1; println!("{}", Self::DOES_NOT_FIT_MSG); }; @@ -186,7 +186,7 @@ impl InitCache { let key = CacheKey::new(module_hash, version, debug); let mut cache = cache!(); if let Some(item) = cache.long_term.remove(&key) { - if let Err(_) = cache.lru.put_with_weight(key, item) { + if cache.lru.put_with_weight(key, item).is_err() { println!("{}", Self::DOES_NOT_FIT_MSG); } } @@ -200,7 +200,7 @@ impl InitCache { let cache = &mut *cache; for (key, item) in cache.long_term.drain() { // not all will fit, just a heuristic - if let Err(_) = cache.lru.put_with_weight(key, item) { + if cache.lru.put_with_weight(key, item).is_err() { println!("{}", Self::DOES_NOT_FIT_MSG); } } From a2f1b6082eb1007ae8c6f7f8fa6b2a686072943a Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 20 Aug 2024 21:31:12 -0300 Subject: [PATCH 131/489] Format rs files --- arbitrator/stylus/src/cache.rs | 22 ++++++++++++++++------ arbitrator/stylus/src/lib.rs | 9 ++++++++- arbitrator/stylus/src/native.rs | 10 ++++++++-- 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index bdecdd724f..ea88fe0fe9 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -2,14 +2,14 @@ // For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE use arbutil::Bytes32; +use clru::{CLruCache, CLruCacheConfig, WeightScale}; use eyre::Result; use lazy_static::lazy_static; -use clru::{CLruCache, CLruCacheConfig, WeightScale}; use parking_lot::Mutex; use prover::programs::config::CompileConfig; +use std::hash::RandomState; use std::{collections::HashMap, num::NonZeroUsize}; use wasmer::{Engine, Module, Store}; -use std::hash::RandomState; use crate::target_cache::target_native; @@ -61,7 +61,11 @@ struct CacheItem { impl CacheItem { fn new(module: Module, engine: Engine, asm_size_estimate_kb: u32) -> Self { - Self { module, engine, asm_size_estimate_kb } + Self { + module, + engine, + asm_size_estimate_kb, + } } fn data(&self) -> (Module, Store) { @@ -74,7 +78,10 @@ impl WeightScale for CustomWeightScale { fn weight(&self, _key: &CacheKey, val: &CacheItem) -> usize { // clru defines that each entry consumes (weight + 1) of the cache capacity. // We subtract 1 since we only want to use the weight as the size of the entry. - val.asm_size_estimate_kb.saturating_sub(1).try_into().unwrap() + val.asm_size_estimate_kb + .saturating_sub(1) + .try_into() + .unwrap() } } @@ -98,7 +105,10 @@ impl InitCache { fn new(size: usize) -> Self { Self { long_term: HashMap::new(), - lru: CLruCache::with_config(CLruCacheConfig::new(NonZeroUsize::new(size).unwrap()).with_scale(CustomWeightScale)), + lru: CLruCache::with_config( + CLruCacheConfig::new(NonZeroUsize::new(size).unwrap()) + .with_scale(CustomWeightScale), + ), lru_counters: LruCounters { hits: 0, misses: 0, @@ -210,7 +220,7 @@ impl InitCache { let mut cache = cache!(); let count = cache.lru.len(); - let metrics = LruCacheMetrics{ + let metrics = LruCacheMetrics { // add 1 to each entry to account that we subtracted 1 in the weight calculation size_kb: (cache.lru.weight() + count).try_into().unwrap(), diff --git a/arbitrator/stylus/src/lib.rs b/arbitrator/stylus/src/lib.rs index 25d8638764..6f7e207308 100644 --- a/arbitrator/stylus/src/lib.rs +++ b/arbitrator/stylus/src/lib.rs @@ -324,7 +324,14 @@ pub unsafe extern "C" fn stylus_cache_module( arbos_tag: u32, debug: bool, ) { - if let Err(error) = InitCache::insert(module_hash, module.slice(), asm_size_estimate_kb, version, arbos_tag, debug) { + if let Err(error) = InitCache::insert( + module_hash, + module.slice(), + asm_size_estimate_kb, + version, + arbos_tag, + debug, + ) { panic!("tried to cache invalid asm!: {error}"); } } diff --git a/arbitrator/stylus/src/native.rs b/arbitrator/stylus/src/native.rs index 7c2f7ab777..393cd951bd 100644 --- a/arbitrator/stylus/src/native.rs +++ b/arbitrator/stylus/src/native.rs @@ -129,8 +129,14 @@ impl> NativeInstance { if !env.evm_data.cached { long_term_tag = 0; } - let (module, store) = - InitCache::insert(module_hash, module, asm_size_estimate_kb, version, long_term_tag, debug)?; + let (module, store) = InitCache::insert( + module_hash, + module, + asm_size_estimate_kb, + version, + long_term_tag, + debug, + )?; Self::from_module(module, store, env) } From 761f84af54aa7135bc2801c8d8c0498e7152e465 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 21 Aug 2024 17:47:14 -0300 Subject: [PATCH 132/489] Relies on module.serialize to estimate asm size --- arbitrator/stylus/src/cache.rs | 29 ++++++++++++-------------- arbitrator/stylus/src/lib.rs | 8 ++----- arbitrator/stylus/src/native.rs | 2 -- arbos/programs/native.go | 28 ++++++++++++------------- arbos/programs/programs.go | 11 +++------- arbos/programs/testcompile.go | 1 - arbos/programs/wasm.go | 3 +-- execution/gethexec/executionengine.go | 6 +++--- system_tests/program_test.go | 30 +++++++++++++-------------- 9 files changed, 50 insertions(+), 68 deletions(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index ea88fe0fe9..5c29f61287 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -14,7 +14,7 @@ use wasmer::{Engine, Module, Store}; use crate::target_cache::target_native; lazy_static! { - static ref INIT_CACHE: Mutex = Mutex::new(InitCache::new(256 * 1024)); + static ref INIT_CACHE: Mutex = Mutex::new(InitCache::new(256 * 1024 * 1024)); } macro_rules! cache { @@ -56,15 +56,15 @@ impl CacheKey { struct CacheItem { module: Module, engine: Engine, - asm_size_estimate_kb: u32, + asm_size_estimate_bytes: usize, } impl CacheItem { - fn new(module: Module, engine: Engine, asm_size_estimate_kb: u32) -> Self { + fn new(module: Module, engine: Engine, asm_size_estimate_bytes: usize) -> Self { Self { module, engine, - asm_size_estimate_kb, + asm_size_estimate_bytes, } } @@ -78,16 +78,13 @@ impl WeightScale for CustomWeightScale { fn weight(&self, _key: &CacheKey, val: &CacheItem) -> usize { // clru defines that each entry consumes (weight + 1) of the cache capacity. // We subtract 1 since we only want to use the weight as the size of the entry. - val.asm_size_estimate_kb - .saturating_sub(1) - .try_into() - .unwrap() + val.asm_size_estimate_bytes.saturating_sub(1) } } #[repr(C)] pub struct LruCacheMetrics { - pub size_kb: u32, + pub size_bytes: u64, pub count: u32, pub hits: u32, pub misses: u32, @@ -102,11 +99,11 @@ impl InitCache { const DOES_NOT_FIT_MSG: &'static str = "Failed to insert into LRU cache, item too large"; - fn new(size: usize) -> Self { + fn new(size_bytes: usize) -> Self { Self { long_term: HashMap::new(), lru: CLruCache::with_config( - CLruCacheConfig::new(NonZeroUsize::new(size).unwrap()) + CLruCacheConfig::new(NonZeroUsize::new(size_bytes).unwrap()) .with_scale(CustomWeightScale), ), lru_counters: LruCounters { @@ -117,10 +114,10 @@ impl InitCache { } } - pub fn set_lru_size(size_kb: u32) { + pub fn set_lru_size(size_bytes: u64) { cache!() .lru - .resize(NonZeroUsize::new(size_kb.try_into().unwrap()).unwrap()) + .resize(NonZeroUsize::new(size_bytes.try_into().unwrap()).unwrap()) } /// Retrieves a cached value, updating items as necessary. @@ -148,7 +145,6 @@ impl InitCache { pub fn insert( module_hash: Bytes32, module: &[u8], - asm_size_estimate_kb: u32, version: u16, long_term_tag: u32, debug: bool, @@ -173,8 +169,9 @@ impl InitCache { let engine = CompileConfig::version(version, debug).engine(target_native()); let module = unsafe { Module::deserialize_unchecked(&engine, module)? }; + let asm_size_estimate_bytes = module.serialize()?.len(); - let item = CacheItem::new(module, engine, asm_size_estimate_kb); + let item = CacheItem::new(module, engine, asm_size_estimate_bytes); let data = item.data(); let mut cache = cache!(); if long_term_tag != Self::ARBOS_TAG { @@ -222,7 +219,7 @@ impl InitCache { let count = cache.lru.len(); let metrics = LruCacheMetrics { // add 1 to each entry to account that we subtracted 1 in the weight calculation - size_kb: (cache.lru.weight() + count).try_into().unwrap(), + size_bytes: (cache.lru.weight() + count).try_into().unwrap(), count: count.try_into().unwrap(), diff --git a/arbitrator/stylus/src/lib.rs b/arbitrator/stylus/src/lib.rs index 6f7e207308..18ff3900d3 100644 --- a/arbitrator/stylus/src/lib.rs +++ b/arbitrator/stylus/src/lib.rs @@ -256,7 +256,6 @@ pub unsafe extern "C" fn stylus_target_set( #[no_mangle] pub unsafe extern "C" fn stylus_call( module: GoSliceData, - asm_size_estimate_kb: u32, calldata: GoSliceData, config: StylusConfig, req_handler: NativeRequestHandler, @@ -277,7 +276,6 @@ pub unsafe extern "C" fn stylus_call( let instance = unsafe { NativeInstance::deserialize_cached( module, - asm_size_estimate_kb, config.version, evm_api, evm_data, @@ -304,8 +302,8 @@ pub unsafe extern "C" fn stylus_call( /// resize lru #[no_mangle] -pub extern "C" fn stylus_cache_lru_resize(size_kb: u32) { - InitCache::set_lru_size(size_kb); +pub extern "C" fn stylus_cache_lru_resize(size_bytes: u64) { + InitCache::set_lru_size(size_bytes); } /// Caches an activated user program. @@ -319,7 +317,6 @@ pub extern "C" fn stylus_cache_lru_resize(size_kb: u32) { pub unsafe extern "C" fn stylus_cache_module( module: GoSliceData, module_hash: Bytes32, - asm_size_estimate_kb: u32, version: u16, arbos_tag: u32, debug: bool, @@ -327,7 +324,6 @@ pub unsafe extern "C" fn stylus_cache_module( if let Err(error) = InitCache::insert( module_hash, module.slice(), - asm_size_estimate_kb, version, arbos_tag, debug, diff --git a/arbitrator/stylus/src/native.rs b/arbitrator/stylus/src/native.rs index 393cd951bd..b661435e52 100644 --- a/arbitrator/stylus/src/native.rs +++ b/arbitrator/stylus/src/native.rs @@ -112,7 +112,6 @@ impl> NativeInstance { /// `module` must represent a valid module. pub unsafe fn deserialize_cached( module: &[u8], - asm_size_estimate_kb: u32, version: u16, evm: E, evm_data: EvmData, @@ -132,7 +131,6 @@ impl> NativeInstance { let (module, store) = InitCache::insert( module_hash, module, - asm_size_estimate_kb, version, long_term_tag, debug, diff --git a/arbos/programs/native.go b/arbos/programs/native.go index 0f3e28287d..b817c55331 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -47,7 +47,7 @@ type rustBytes = C.RustBytes type rustSlice = C.RustSlice var ( - stylusLRUCacheSizeKbGauge = metrics.NewRegisteredGauge("arb/arbos/stylus/cache/lru/size_kilobytes", nil) + stylusLRUCacheSizeBytesGauge = metrics.NewRegisteredGauge("arb/arbos/stylus/cache/lru/size_bytes", nil) stylusLRUCacheSizeCountGauge = metrics.NewRegisteredGauge("arb/arbos/stylus/cache/lru/count", nil) stylusLRUCacheSizeHitsCounter = metrics.NewRegisteredCounter("arb/arbos/stylus/cache/lru/hits", nil) stylusLRUCacheSizeMissesCounter = metrics.NewRegisteredCounter("arb/arbos/stylus/cache/lru/misses", nil) @@ -220,7 +220,6 @@ func callProgram( address common.Address, moduleHash common.Hash, localAsm []byte, - asmSizeEstimateKb uint32, scope *vm.ScopeContext, interpreter *vm.EVMInterpreter, tracingInfo *util.TracingInfo, @@ -248,7 +247,6 @@ func callProgram( output := &rustBytes{} status := userStatus(C.stylus_call( goSlice(localAsm), - u32(asmSizeEstimateKb), goSlice(calldata), stylusParams.encode(), evmApi.cNative, @@ -291,55 +289,55 @@ func cacheProgram(db vm.StateDB, module common.Hash, program Program, addressFor panic("unable to recreate wasm") } tag := db.Database().WasmCacheTag() - state.CacheWasmRust(asm, module, program.asmSize(), program.version, tag, debug) + state.CacheWasmRust(asm, module, program.version, tag, debug) db.RecordCacheWasm(state.CacheWasm{ModuleHash: module, Version: program.version, Tag: tag, Debug: debug}) } } // Evicts a program in Rust. We write a record so that we can undo on revert, unless we don't need to (e.g. expired) // For gas estimation and eth_call, we ignore permanent updates and rely on Rust's LRU. -func evictProgram(db vm.StateDB, module common.Hash, asmSizeEstimateKb uint32, version uint16, debug bool, runMode core.MessageRunMode, forever bool) { +func evictProgram(db vm.StateDB, module common.Hash, version uint16, debug bool, runMode core.MessageRunMode, forever bool) { if runMode == core.MessageCommitMode { tag := db.Database().WasmCacheTag() state.EvictWasmRust(module, version, tag, debug) if !forever { - db.RecordEvictWasm(state.EvictWasm{ModuleHash: module, Version: version, Tag: tag, Debug: debug, AsmSizeEstimateKb: asmSizeEstimateKb}) + db.RecordEvictWasm(state.EvictWasm{ModuleHash: module, Version: version, Tag: tag, Debug: debug}) } } } func init() { - state.CacheWasmRust = func(asm []byte, moduleHash common.Hash, asmSizeEstimateKb uint32, version uint16, tag uint32, debug bool) { - C.stylus_cache_module(goSlice(asm), hashToBytes32(moduleHash), u32(asmSizeEstimateKb), u16(version), u32(tag), cbool(debug)) + state.CacheWasmRust = func(asm []byte, moduleHash common.Hash, version uint16, tag uint32, debug bool) { + C.stylus_cache_module(goSlice(asm), hashToBytes32(moduleHash), u16(version), u32(tag), cbool(debug)) } state.EvictWasmRust = func(moduleHash common.Hash, version uint16, tag uint32, debug bool) { C.stylus_evict_module(hashToBytes32(moduleHash), u16(version), u32(tag), cbool(debug)) } } -func ResizeWasmLruCache(sizeKb uint32) { - C.stylus_cache_lru_resize(u32(sizeKb)) +func ResizeWasmLruCache(sizeBytes uint64) { + C.stylus_cache_lru_resize(u64(sizeBytes)) } // exported for testing type WasmLruCacheMetrics struct { - SizeKb uint32 - Count uint32 + SizeBytes uint64 + Count uint32 } // exported for testing func GetWasmLruCacheMetrics() *WasmLruCacheMetrics { metrics := C.stylus_get_lru_cache_metrics() - stylusLRUCacheSizeKbGauge.Update(int64(metrics.size_kb)) + stylusLRUCacheSizeBytesGauge.Update(int64(metrics.size_bytes)) stylusLRUCacheSizeCountGauge.Update(int64(metrics.count)) stylusLRUCacheSizeHitsCounter.Inc(int64(metrics.hits)) stylusLRUCacheSizeMissesCounter.Inc(int64(metrics.misses)) stylusLRUCacheSizeDoesNotFitCounter.Inc(int64(metrics.does_not_fit)) return &WasmLruCacheMetrics{ - SizeKb: uint32(metrics.size_kb), - Count: uint32(metrics.count), + SizeBytes: uint64(metrics.size_bytes), + Count: uint32(metrics.count), } } diff --git a/arbos/programs/programs.go b/arbos/programs/programs.go index 32453dd529..b3f51ecba0 100644 --- a/arbos/programs/programs.go +++ b/arbos/programs/programs.go @@ -128,12 +128,7 @@ func (p Programs) ActivateProgram(evm *vm.EVM, address common.Address, runMode c return 0, codeHash, common.Hash{}, nil, true, err } - program, err := p.getActiveProgram(codeHash, time, params) - if err != nil { - return 0, codeHash, common.Hash{}, nil, true, err - } - - evictProgram(statedb, oldModuleHash, program.asmSize(), currentVersion, debugMode, runMode, expired) + evictProgram(statedb, oldModuleHash, currentVersion, debugMode, runMode, expired) } if err := p.moduleHashes.Set(codeHash, info.moduleHash); err != nil { return 0, codeHash, common.Hash{}, nil, true, err @@ -253,7 +248,7 @@ func (p Programs) CallProgram( if runmode == core.MessageCommitMode { arbos_tag = statedb.Database().WasmCacheTag() } - ret, err := callProgram(address, moduleHash, localAsm, program.asmEstimateKb.ToUint32(), scope, interpreter, tracingInfo, calldata, evmData, goParams, model, arbos_tag) + ret, err := callProgram(address, moduleHash, localAsm, scope, interpreter, tracingInfo, calldata, evmData, goParams, model, arbos_tag) if len(ret) > 0 && arbosVersion >= gethParams.ArbosVersion_StylusFixes { // Ensure that return data costs as least as much as it would in the EVM. evmCost := evmMemoryCost(uint64(len(ret))) @@ -439,7 +434,7 @@ func (p Programs) SetProgramCached( } cacheProgram(db, moduleHash, program, address, code, codeHash, params, debug, time, runMode) } else { - evictProgram(db, moduleHash, program.asmSize(), program.version, debug, runMode, expired) + evictProgram(db, moduleHash, program.version, debug, runMode, expired) } program.cached = cache return p.setProgram(codeHash, program) diff --git a/arbos/programs/testcompile.go b/arbos/programs/testcompile.go index 57a5ce26b1..615b0f3f72 100644 --- a/arbos/programs/testcompile.go +++ b/arbos/programs/testcompile.go @@ -244,7 +244,6 @@ func testCompileLoad() error { status := userStatus(C.stylus_call( goSlice(localAsm), - u32(1), goSlice(calldata), progParams.encode(), reqHandler, diff --git a/arbos/programs/wasm.go b/arbos/programs/wasm.go index 849ed88f42..504e3bf9fa 100644 --- a/arbos/programs/wasm.go +++ b/arbos/programs/wasm.go @@ -97,7 +97,7 @@ func activateProgram( // stub any non-consensus, Rust-side caching updates func cacheProgram(db vm.StateDB, module common.Hash, program Program, addressForLogging common.Address, code []byte, codeHash common.Hash, params *StylusParams, debug bool, time uint64, runMode core.MessageRunMode) { } -func evictProgram(db vm.StateDB, module common.Hash, asmSizeEstimate uint32, version uint16, debug bool, mode core.MessageRunMode, forever bool) { +func evictProgram(db vm.StateDB, module common.Hash, version uint16, debug bool, mode core.MessageRunMode, forever bool) { } //go:wasmimport programs new_program @@ -136,7 +136,6 @@ func callProgram( address common.Address, moduleHash common.Hash, _localAsm []byte, - _asmSizeEstimateKb uint32, scope *vm.ScopeContext, interpreter *vm.EVMInterpreter, tracingInfo *util.TracingInfo, diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index affdcec704..686e0f1a36 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -184,7 +184,7 @@ func populateStylusTargetCache(targetConfig *StylusTargetConfig) error { func (s *ExecutionEngine) Initialize(rustCacheSizeMb uint32, targetConfig *StylusTargetConfig) error { if rustCacheSizeMb != 0 { - s.ResizeWasmLruCache(arbmath.SaturatingUMul(rustCacheSizeMb, 1024)) + s.ResizeWasmLruCache(arbmath.SaturatingUMul(uint64(rustCacheSizeMb), 1024*1024)) } if err := populateStylusTargetCache(targetConfig); err != nil { return fmt.Errorf("error populating stylus target cache: %w", err) @@ -193,8 +193,8 @@ func (s *ExecutionEngine) Initialize(rustCacheSizeMb uint32, targetConfig *Stylu } // exported for testing -func (s *ExecutionEngine) ResizeWasmLruCache(sizeKb uint32) { - programs.ResizeWasmLruCache(sizeKb) +func (s *ExecutionEngine) ResizeWasmLruCache(sizeBytes uint64) { + programs.ResizeWasmLruCache(sizeBytes) } // exported for testing diff --git a/system_tests/program_test.go b/system_tests/program_test.go index 108cec6647..0dd65a6bf6 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -2020,15 +2020,15 @@ func TestWasmLruCache(t *testing.T) { if lruMetrics.Count != 0 { t.Fatalf("lruMetrics.Count, expected: %v, actual: %v", 0, lruMetrics.Count) } - if lruMetrics.SizeKb != 0 { - t.Fatalf("lruMetrics.SizeKb, expected: %v, actual: %v", 0, lruMetrics.SizeKb) + if lruMetrics.SizeBytes != 0 { + t.Fatalf("lruMetrics.SizeBytes, expected: %v, actual: %v", 0, lruMetrics.SizeBytes) } - lruCacheSize := uint32(500) + lruCacheSize := uint64(100_000) builder.L2.ExecNode.ExecEngine.ResizeWasmLruCache(lruCacheSize) // fallible wasm program will not be cached since its size is greater than lruCacheSize - fallibleAsmEstimateSizeKb := uint32(551) + fallibleAsmEstimateSizeBytes := uint64(118_624) fallibleProgramAddress := deployWasm(t, ctx, auth, l2client, rustFile("fallible")) tx := l2info.PrepareTxTo("Owner", &fallibleProgramAddress, l2info.TransferGas, nil, []byte{0x01}) Require(t, l2client.SendTransaction(ctx, tx)) @@ -2038,12 +2038,12 @@ func TestWasmLruCache(t *testing.T) { if lruMetrics.Count != 0 { t.Fatalf("lruMetrics.Count, expected: %v, actual: %v", 0, lruMetrics.Count) } - if lruMetrics.SizeKb != 0 { - t.Fatalf("lruMetrics.SizeKb, expected: %v, actual: %v", 0, lruMetrics.SizeKb) + if lruMetrics.SizeBytes != 0 { + t.Fatalf("lruMetrics.SizeBytes, expected: %v, actual: %v", 0, lruMetrics.SizeBytes) } // resize lru cache - lruCacheSize = uint32(1500) + lruCacheSize = uint64(350_000) builder.L2.ExecNode.ExecEngine.ResizeWasmLruCache(lruCacheSize) // fallible wasm program will be cached @@ -2055,12 +2055,12 @@ func TestWasmLruCache(t *testing.T) { if lruMetrics.Count != 1 { t.Fatalf("lruMetrics.Count, expected: %v, actual: %v", 1, lruMetrics.Count) } - if lruMetrics.SizeKb != fallibleAsmEstimateSizeKb { - t.Fatalf("lruMetrics.SizeKb, expected: %v, actual: %v", fallibleAsmEstimateSizeKb, lruMetrics.SizeKb) + if lruMetrics.SizeBytes != fallibleAsmEstimateSizeBytes { + t.Fatalf("lruMetrics.SizeBytes, expected: %v, actual: %v", fallibleAsmEstimateSizeBytes, lruMetrics.SizeBytes) } // keccak wasm program will be cached - keccakAsmEstimateSizeKb := uint32(583) + keccakAsmEstimateSizeBytes := uint64(175_496) keccakProgramAddress := deployWasm(t, ctx, auth, l2client, rustFile("keccak")) tx = l2info.PrepareTxTo("Owner", &keccakProgramAddress, l2info.TransferGas, nil, []byte{0x01}) Require(t, l2client.SendTransaction(ctx, tx)) @@ -2070,12 +2070,12 @@ func TestWasmLruCache(t *testing.T) { if lruMetrics.Count != 2 { t.Fatalf("lruMetrics.Count, expected: %v, actual: %v", 2, lruMetrics.Count) } - if lruMetrics.SizeKb != fallibleAsmEstimateSizeKb+keccakAsmEstimateSizeKb { - t.Fatalf("lruMetrics.SizeKb, expected: %v, actual: %v", fallibleAsmEstimateSizeKb+keccakAsmEstimateSizeKb, lruMetrics.SizeKb) + if lruMetrics.SizeBytes != fallibleAsmEstimateSizeBytes+keccakAsmEstimateSizeBytes { + t.Fatalf("lruMetrics.SizeBytes, expected: %v, actual: %v", fallibleAsmEstimateSizeBytes+keccakAsmEstimateSizeBytes, lruMetrics.SizeBytes) } // math wasm program will be cached, but fallible will be evicted since (fallible + keccak + math) > lruCacheSize - mathAsmEstimateSizeKb := uint32(560) + mathAsmEstimateSizeBytes := uint64(131_672) mathProgramAddress := deployWasm(t, ctx, auth, l2client, rustFile("math")) tx = l2info.PrepareTxTo("Owner", &mathProgramAddress, l2info.TransferGas, nil, []byte{0x01}) Require(t, l2client.SendTransaction(ctx, tx)) @@ -2085,7 +2085,7 @@ func TestWasmLruCache(t *testing.T) { if lruMetrics.Count != 2 { t.Fatalf("lruMetrics.Count, expected: %v, actual: %v", 2, lruMetrics.Count) } - if lruMetrics.SizeKb != keccakAsmEstimateSizeKb+mathAsmEstimateSizeKb { - t.Fatalf("lruMetrics.SizeKb, expected: %v, actual: %v", keccakAsmEstimateSizeKb+mathAsmEstimateSizeKb, lruMetrics.SizeKb) + if lruMetrics.SizeBytes != keccakAsmEstimateSizeBytes+mathAsmEstimateSizeBytes { + t.Fatalf("lruMetrics.SizeBytes, expected: %v, actual: %v", keccakAsmEstimateSizeBytes+mathAsmEstimateSizeBytes, lruMetrics.SizeBytes) } } From 21664e9b94cbffc1779e37050e48a180770f4a4f Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 21 Aug 2024 19:22:32 -0300 Subject: [PATCH 133/489] Renames resize to set capacity in stylus cache --- arbitrator/stylus/src/cache.rs | 4 ++-- arbitrator/stylus/src/lib.rs | 6 +++--- arbnode/inbox_test.go | 2 +- arbos/programs/native.go | 4 ++-- execution/gethexec/blockchain.go | 6 +++--- execution/gethexec/executionengine.go | 10 +++++----- execution/gethexec/node.go | 2 +- system_tests/common_test.go | 2 +- system_tests/program_test.go | 10 +++++----- 9 files changed, 23 insertions(+), 23 deletions(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index 5c29f61287..aeefd9c8bd 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -114,10 +114,10 @@ impl InitCache { } } - pub fn set_lru_size(size_bytes: u64) { + pub fn set_lru_capacity(capacity_bytes: u64) { cache!() .lru - .resize(NonZeroUsize::new(size_bytes.try_into().unwrap()).unwrap()) + .resize(NonZeroUsize::new(capacity_bytes.try_into().unwrap()).unwrap()) } /// Retrieves a cached value, updating items as necessary. diff --git a/arbitrator/stylus/src/lib.rs b/arbitrator/stylus/src/lib.rs index 18ff3900d3..791557dd0c 100644 --- a/arbitrator/stylus/src/lib.rs +++ b/arbitrator/stylus/src/lib.rs @@ -300,10 +300,10 @@ pub unsafe extern "C" fn stylus_call( status } -/// resize lru +/// set lru cache capacity #[no_mangle] -pub extern "C" fn stylus_cache_lru_resize(size_bytes: u64) { - InitCache::set_lru_size(size_bytes); +pub extern "C" fn stylus_set_cache_lru_capacity(capacity_bytes: u64) { + InitCache::set_lru_capacity(capacity_bytes); } /// Caches an activated user program. diff --git a/arbnode/inbox_test.go b/arbnode/inbox_test.go index e0c97bb871..e588ef399b 100644 --- a/arbnode/inbox_test.go +++ b/arbnode/inbox_test.go @@ -74,7 +74,7 @@ func NewTransactionStreamerForTest(t *testing.T, ownerAddress common.Address) (* } stylusTargetConfig := &gethexec.DefaultStylusTargetConfig Require(t, stylusTargetConfig.Validate()) // pre-processes config (i.a. parses wasmTargets) - if err := execEngine.Initialize(gethexec.DefaultCachingConfig.StylusLRUCacheSize, &gethexec.DefaultStylusTargetConfig); err != nil { + if err := execEngine.Initialize(gethexec.DefaultCachingConfig.StylusLRUCacheCapacity, &gethexec.DefaultStylusTargetConfig); err != nil { Fail(t, err) } execSeq := &execClientWrapper{execEngine, t} diff --git a/arbos/programs/native.go b/arbos/programs/native.go index b817c55331..97b1f435ed 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -315,8 +315,8 @@ func init() { } } -func ResizeWasmLruCache(sizeBytes uint64) { - C.stylus_cache_lru_resize(u64(sizeBytes)) +func SetWasmLruCacheCapacity(capacityBytes uint64) { + C.stylus_set_cache_lru_capacity(u64(capacityBytes)) } // exported for testing diff --git a/execution/gethexec/blockchain.go b/execution/gethexec/blockchain.go index e0e907683d..1bf618bba3 100644 --- a/execution/gethexec/blockchain.go +++ b/execution/gethexec/blockchain.go @@ -37,7 +37,7 @@ type CachingConfig struct { SnapshotRestoreGasLimit uint64 `koanf:"snapshot-restore-gas-limit"` MaxNumberOfBlocksToSkipStateSaving uint32 `koanf:"max-number-of-blocks-to-skip-state-saving"` MaxAmountOfGasToSkipStateSaving uint64 `koanf:"max-amount-of-gas-to-skip-state-saving"` - StylusLRUCacheSize uint32 `koanf:"stylus-lru-cache-size"` + StylusLRUCacheCapacity uint32 `koanf:"stylus-lru-cache-capacity"` StateScheme string `koanf:"state-scheme"` StateHistory uint64 `koanf:"state-history"` } @@ -54,7 +54,7 @@ func CachingConfigAddOptions(prefix string, f *flag.FlagSet) { f.Uint64(prefix+".snapshot-restore-gas-limit", DefaultCachingConfig.SnapshotRestoreGasLimit, "maximum gas rolled back to recover snapshot") f.Uint32(prefix+".max-number-of-blocks-to-skip-state-saving", DefaultCachingConfig.MaxNumberOfBlocksToSkipStateSaving, "maximum number of blocks to skip state saving to persistent storage (archive node only) -- warning: this option seems to cause issues") f.Uint64(prefix+".max-amount-of-gas-to-skip-state-saving", DefaultCachingConfig.MaxAmountOfGasToSkipStateSaving, "maximum amount of gas in blocks to skip saving state to Persistent storage (archive node only) -- warning: this option seems to cause issues") - f.Uint32(prefix+".stylus-lru-cache-size", DefaultCachingConfig.StylusLRUCacheSize, "capacity, in megabytes, of the LRU cache that keeps initialized stylus programs") + f.Uint32(prefix+".stylus-lru-cache-capacity", DefaultCachingConfig.StylusLRUCacheCapacity, "capacity, in megabytes, of the LRU cache that keeps initialized stylus programs") f.String(prefix+".state-scheme", DefaultCachingConfig.StateScheme, "scheme to use for state trie storage (hash, path)") f.Uint64(prefix+".state-history", DefaultCachingConfig.StateHistory, "number of recent blocks to retain state history for (path state-scheme only)") } @@ -75,7 +75,7 @@ var DefaultCachingConfig = CachingConfig{ SnapshotRestoreGasLimit: 300_000_000_000, MaxNumberOfBlocksToSkipStateSaving: 0, MaxAmountOfGasToSkipStateSaving: 0, - StylusLRUCacheSize: 256, + StylusLRUCacheCapacity: 256, StateScheme: rawdb.HashScheme, StateHistory: getStateHistory(DefaultSequencerConfig.MaxBlockSpeed), } diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index 686e0f1a36..5b5c9a8938 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -182,9 +182,9 @@ func populateStylusTargetCache(targetConfig *StylusTargetConfig) error { return nil } -func (s *ExecutionEngine) Initialize(rustCacheSizeMb uint32, targetConfig *StylusTargetConfig) error { - if rustCacheSizeMb != 0 { - s.ResizeWasmLruCache(arbmath.SaturatingUMul(uint64(rustCacheSizeMb), 1024*1024)) +func (s *ExecutionEngine) Initialize(rustCacheCapacityMb uint32, targetConfig *StylusTargetConfig) error { + if rustCacheCapacityMb != 0 { + s.SetWasmLruCacheCapacity(arbmath.SaturatingUMul(uint64(rustCacheCapacityMb), 1024*1024)) } if err := populateStylusTargetCache(targetConfig); err != nil { return fmt.Errorf("error populating stylus target cache: %w", err) @@ -193,8 +193,8 @@ func (s *ExecutionEngine) Initialize(rustCacheSizeMb uint32, targetConfig *Stylu } // exported for testing -func (s *ExecutionEngine) ResizeWasmLruCache(sizeBytes uint64) { - programs.ResizeWasmLruCache(sizeBytes) +func (s *ExecutionEngine) SetWasmLruCacheCapacity(capacityBytes uint64) { + programs.SetWasmLruCacheCapacity(capacityBytes) } // exported for testing diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index 93bb254ed8..8fbbdc9762 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -308,7 +308,7 @@ func (n *ExecutionNode) MarkFeedStart(to arbutil.MessageIndex) { func (n *ExecutionNode) Initialize(ctx context.Context) error { config := n.ConfigFetcher() - err := n.ExecEngine.Initialize(config.Caching.StylusLRUCacheSize, &config.StylusTarget) + err := n.ExecEngine.Initialize(config.Caching.StylusLRUCacheCapacity, &config.StylusTarget) if err != nil { return fmt.Errorf("error initializing execution engine: %w", err) } diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 363f1050fb..d4a12d536a 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -166,7 +166,7 @@ var TestCachingConfig = gethexec.CachingConfig{ SnapshotRestoreGasLimit: 300_000_000_000, MaxNumberOfBlocksToSkipStateSaving: 0, MaxAmountOfGasToSkipStateSaving: 0, - StylusLRUCacheSize: 0, + StylusLRUCacheCapacity: 0, StateScheme: env.GetTestStateScheme(), } diff --git a/system_tests/program_test.go b/system_tests/program_test.go index 0dd65a6bf6..7aa80f43ee 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -2024,8 +2024,8 @@ func TestWasmLruCache(t *testing.T) { t.Fatalf("lruMetrics.SizeBytes, expected: %v, actual: %v", 0, lruMetrics.SizeBytes) } - lruCacheSize := uint64(100_000) - builder.L2.ExecNode.ExecEngine.ResizeWasmLruCache(lruCacheSize) + lruCacheCapacity := uint64(100_000) + builder.L2.ExecNode.ExecEngine.SetWasmLruCacheCapacity(lruCacheCapacity) // fallible wasm program will not be cached since its size is greater than lruCacheSize fallibleAsmEstimateSizeBytes := uint64(118_624) @@ -2042,9 +2042,9 @@ func TestWasmLruCache(t *testing.T) { t.Fatalf("lruMetrics.SizeBytes, expected: %v, actual: %v", 0, lruMetrics.SizeBytes) } - // resize lru cache - lruCacheSize = uint64(350_000) - builder.L2.ExecNode.ExecEngine.ResizeWasmLruCache(lruCacheSize) + // set new lru cache capacity + lruCacheCapacity = uint64(350_000) + builder.L2.ExecNode.ExecEngine.SetWasmLruCacheCapacity(lruCacheCapacity) // fallible wasm program will be cached tx = l2info.PrepareTxTo("Owner", &fallibleProgramAddress, l2info.TransferGas, nil, []byte{0x01}) From b7a949c92666143e136a70eca727e5cbb7731e8f Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 22 Aug 2024 09:55:43 -0300 Subject: [PATCH 134/489] Don't hardcode wasm sizes in TestWasmLruCache --- arbitrator/stylus/src/cache.rs | 15 ++++-- arbitrator/stylus/src/lib.rs | 24 ++++++---- arbos/programs/native.go | 5 ++ system_tests/program_test.go | 85 +++++++++++++++++++++++++--------- 4 files changed, 97 insertions(+), 32 deletions(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index aeefd9c8bd..c2aaddb6e5 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -91,6 +91,17 @@ pub struct LruCacheMetrics { pub does_not_fit: u32, } +pub fn deserialize_module( + module: &[u8], + version: u16, + debug: bool, +) -> Result<(Module, Engine, usize)> { + let engine = CompileConfig::version(version, debug).engine(target_native()); + let module = unsafe { Module::deserialize_unchecked(&engine, module)? }; + let asm_size_estimate_bytes = module.serialize()?.len(); + Ok((module, engine, asm_size_estimate_bytes)) +} + impl InitCache { // current implementation only has one tag that stores to the long_term // future implementations might have more, but 0 is a reserved tag @@ -167,9 +178,7 @@ impl InitCache { } drop(cache); - let engine = CompileConfig::version(version, debug).engine(target_native()); - let module = unsafe { Module::deserialize_unchecked(&engine, module)? }; - let asm_size_estimate_bytes = module.serialize()?.len(); + let (module, engine, asm_size_estimate_bytes) = deserialize_module(module, version, debug)?; let item = CacheItem::new(module, engine, asm_size_estimate_bytes); let data = item.data(); diff --git a/arbitrator/stylus/src/lib.rs b/arbitrator/stylus/src/lib.rs index 791557dd0c..e5a80f5ee5 100644 --- a/arbitrator/stylus/src/lib.rs +++ b/arbitrator/stylus/src/lib.rs @@ -11,7 +11,7 @@ use arbutil::{ format::DebugBytes, Bytes32, }; -use cache::{InitCache, LruCacheMetrics}; +use cache::{deserialize_module, InitCache, LruCacheMetrics}; use evm_api::NativeRequestHandler; use eyre::ErrReport; use native::NativeInstance; @@ -321,13 +321,7 @@ pub unsafe extern "C" fn stylus_cache_module( arbos_tag: u32, debug: bool, ) { - if let Err(error) = InitCache::insert( - module_hash, - module.slice(), - version, - arbos_tag, - debug, - ) { + if let Err(error) = InitCache::insert(module_hash, module.slice(), version, arbos_tag, debug) { panic!("tried to cache invalid asm!: {error}"); } } @@ -373,3 +367,17 @@ pub extern "C" fn stylus_get_lru_cache_metrics() -> LruCacheMetrics { pub extern "C" fn stylus_clear_lru_cache() { InitCache::clear_lru_cache() } + +/// Gets asm estimate size. +/// Only used for testing purposes. +#[no_mangle] +pub unsafe extern "C" fn stylus_get_asm_size_estimate_bytes( + module: GoSliceData, + version: u16, + debug: bool, +) -> u64 { + match deserialize_module(module.slice(), version, debug) { + Err(error) => panic!("tried to get invalid asm!: {error}"), + Ok((_, _, asm_size_estimate_bytes)) => asm_size_estimate_bytes.try_into().unwrap(), + } +} diff --git a/arbos/programs/native.go b/arbos/programs/native.go index 97b1f435ed..33ecf7d744 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -346,6 +346,11 @@ func ClearWasmLruCache() { C.stylus_clear_lru_cache() } +// exported for testing +func GetAsmSizeEstimateBytes(module []byte, version uint16, debug bool) uint64 { + return uint64(C.stylus_get_asm_size_estimate_bytes(goSlice(module), u16(version), cbool(debug))) +} + const DefaultTargetDescriptionArm = "arm64-linux-unknown+neon" const DefaultTargetDescriptionX86 = "x86_64-linux-unknown+sse4.2+lzcnt+bmi" diff --git a/system_tests/program_test.go b/system_tests/program_test.go index 7aa80f43ee..faa1649df9 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -2008,6 +2008,45 @@ func checkWasmStoreContent(t *testing.T, wasmDb ethdb.KeyValueStore, targets []s } } +func deployWasmAndGetSizeEstimateBytes( + t *testing.T, + builder *NodeBuilder, + auth bind.TransactOpts, + wasmName string, +) (common.Address, uint64) { + ctx := builder.ctx + l2client := builder.L2.Client + + wasm, _ := readWasmFile(t, rustFile(wasmName)) + arbWasm, err := pgen.NewArbWasm(types.ArbWasmAddress, l2client) + Require(t, err, ", wasmName:", wasmName) + + programAddress := deployContract(t, ctx, auth, l2client, wasm) + tx, err := arbWasm.ActivateProgram(&auth, programAddress) + Require(t, err, ", wasmName:", wasmName) + receipt, err := EnsureTxSucceeded(ctx, l2client, tx) + Require(t, err, ", wasmName:", wasmName) + + if len(receipt.Logs) != 1 { + Fatal(t, "expected 1 log while activating, got ", len(receipt.Logs), ", wasmName:", wasmName) + } + log, err := arbWasm.ParseProgramActivated(*receipt.Logs[0]) + Require(t, err, ", wasmName:", wasmName) + + statedb, err := builder.L2.ExecNode.Backend.ArbInterface().BlockChain().State() + Require(t, err, ", wasmName:", wasmName) + + module, err := statedb.TryGetActivatedAsm(rawdb.LocalTarget(), log.ModuleHash) + Require(t, err, ", wasmName:", wasmName) + + asmSizeEstimateBytes := programs.GetAsmSizeEstimateBytes(module, log.Version, true) + if asmSizeEstimateBytes == 0 { + // just a sanity check + Fatal(t, "asmSizeEstimateBytes is 0, wasmName:", wasmName) + } + return programAddress, asmSizeEstimateBytes +} + func TestWasmLruCache(t *testing.T) { builder, auth, cleanup := setupProgramTest(t, true) ctx := builder.ctx @@ -2015,6 +2054,19 @@ func TestWasmLruCache(t *testing.T) { l2client := builder.L2.Client defer cleanup() + auth.GasLimit = 32000000 + auth.Value = oneEth + + fallibleProgramAddress, fallibleAsmSizeEstimateBytes := deployWasmAndGetSizeEstimateBytes(t, builder, auth, "fallible") + keccakProgramAddress, keccakAsmSizeEstimateBytes := deployWasmAndGetSizeEstimateBytes(t, builder, auth, "keccak") + mathProgramAddress, mathAsmSizeEstimateBytes := deployWasmAndGetSizeEstimateBytes(t, builder, auth, "math") + t.Log( + "asmSizeEstimateBytes, ", + "fallible:", fallibleAsmSizeEstimateBytes, + "keccak:", keccakAsmSizeEstimateBytes, + "math:", mathAsmSizeEstimateBytes, + ) + builder.L2.ExecNode.ExecEngine.ClearWasmLruCache() lruMetrics := builder.L2.ExecNode.ExecEngine.GetWasmLruCacheMetrics() if lruMetrics.Count != 0 { @@ -2024,12 +2076,8 @@ func TestWasmLruCache(t *testing.T) { t.Fatalf("lruMetrics.SizeBytes, expected: %v, actual: %v", 0, lruMetrics.SizeBytes) } - lruCacheCapacity := uint64(100_000) - builder.L2.ExecNode.ExecEngine.SetWasmLruCacheCapacity(lruCacheCapacity) - - // fallible wasm program will not be cached since its size is greater than lruCacheSize - fallibleAsmEstimateSizeBytes := uint64(118_624) - fallibleProgramAddress := deployWasm(t, ctx, auth, l2client, rustFile("fallible")) + builder.L2.ExecNode.ExecEngine.SetWasmLruCacheCapacity(fallibleAsmSizeEstimateBytes - 1) + // fallible wasm program will not be cached since its size is greater than lru cache capacity tx := l2info.PrepareTxTo("Owner", &fallibleProgramAddress, l2info.TransferGas, nil, []byte{0x01}) Require(t, l2client.SendTransaction(ctx, tx)) _, err := EnsureTxSucceeded(ctx, l2client, tx) @@ -2042,10 +2090,9 @@ func TestWasmLruCache(t *testing.T) { t.Fatalf("lruMetrics.SizeBytes, expected: %v, actual: %v", 0, lruMetrics.SizeBytes) } - // set new lru cache capacity - lruCacheCapacity = uint64(350_000) - builder.L2.ExecNode.ExecEngine.SetWasmLruCacheCapacity(lruCacheCapacity) - + builder.L2.ExecNode.ExecEngine.SetWasmLruCacheCapacity( + fallibleAsmSizeEstimateBytes + keccakAsmSizeEstimateBytes + mathAsmSizeEstimateBytes - 1, + ) // fallible wasm program will be cached tx = l2info.PrepareTxTo("Owner", &fallibleProgramAddress, l2info.TransferGas, nil, []byte{0x01}) Require(t, l2client.SendTransaction(ctx, tx)) @@ -2055,13 +2102,11 @@ func TestWasmLruCache(t *testing.T) { if lruMetrics.Count != 1 { t.Fatalf("lruMetrics.Count, expected: %v, actual: %v", 1, lruMetrics.Count) } - if lruMetrics.SizeBytes != fallibleAsmEstimateSizeBytes { - t.Fatalf("lruMetrics.SizeBytes, expected: %v, actual: %v", fallibleAsmEstimateSizeBytes, lruMetrics.SizeBytes) + if lruMetrics.SizeBytes != fallibleAsmSizeEstimateBytes { + t.Fatalf("lruMetrics.SizeBytes, expected: %v, actual: %v", fallibleAsmSizeEstimateBytes, lruMetrics.SizeBytes) } // keccak wasm program will be cached - keccakAsmEstimateSizeBytes := uint64(175_496) - keccakProgramAddress := deployWasm(t, ctx, auth, l2client, rustFile("keccak")) tx = l2info.PrepareTxTo("Owner", &keccakProgramAddress, l2info.TransferGas, nil, []byte{0x01}) Require(t, l2client.SendTransaction(ctx, tx)) _, err = EnsureTxSucceeded(ctx, l2client, tx) @@ -2070,13 +2115,11 @@ func TestWasmLruCache(t *testing.T) { if lruMetrics.Count != 2 { t.Fatalf("lruMetrics.Count, expected: %v, actual: %v", 2, lruMetrics.Count) } - if lruMetrics.SizeBytes != fallibleAsmEstimateSizeBytes+keccakAsmEstimateSizeBytes { - t.Fatalf("lruMetrics.SizeBytes, expected: %v, actual: %v", fallibleAsmEstimateSizeBytes+keccakAsmEstimateSizeBytes, lruMetrics.SizeBytes) + if lruMetrics.SizeBytes != fallibleAsmSizeEstimateBytes+keccakAsmSizeEstimateBytes { + t.Fatalf("lruMetrics.SizeBytes, expected: %v, actual: %v", fallibleAsmSizeEstimateBytes+keccakAsmSizeEstimateBytes, lruMetrics.SizeBytes) } - // math wasm program will be cached, but fallible will be evicted since (fallible + keccak + math) > lruCacheSize - mathAsmEstimateSizeBytes := uint64(131_672) - mathProgramAddress := deployWasm(t, ctx, auth, l2client, rustFile("math")) + // math wasm program will be cached, but fallible will be evicted since (fallible + keccak + math) > lruCacheCapacity tx = l2info.PrepareTxTo("Owner", &mathProgramAddress, l2info.TransferGas, nil, []byte{0x01}) Require(t, l2client.SendTransaction(ctx, tx)) _, err = EnsureTxSucceeded(ctx, l2client, tx) @@ -2085,7 +2128,7 @@ func TestWasmLruCache(t *testing.T) { if lruMetrics.Count != 2 { t.Fatalf("lruMetrics.Count, expected: %v, actual: %v", 2, lruMetrics.Count) } - if lruMetrics.SizeBytes != keccakAsmEstimateSizeBytes+mathAsmEstimateSizeBytes { - t.Fatalf("lruMetrics.SizeBytes, expected: %v, actual: %v", keccakAsmEstimateSizeBytes+mathAsmEstimateSizeBytes, lruMetrics.SizeBytes) + if lruMetrics.SizeBytes != keccakAsmSizeEstimateBytes+mathAsmSizeEstimateBytes { + t.Fatalf("lruMetrics.SizeBytes, expected: %v, actual: %v", keccakAsmSizeEstimateBytes+mathAsmSizeEstimateBytes, lruMetrics.SizeBytes) } } From bbed30a3bc8e628f926f0da6aabbb4c0ef7ad679 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 22 Aug 2024 10:02:00 -0300 Subject: [PATCH 135/489] Removes unnecessary calls to ExecutionEngine regarding Wasm LRU cache --- arbos/programs/native.go | 5 ++--- execution/gethexec/executionengine.go | 19 ++----------------- system_tests/program_test.go | 18 +++++++++--------- 3 files changed, 13 insertions(+), 29 deletions(-) diff --git a/arbos/programs/native.go b/arbos/programs/native.go index 33ecf7d744..dd5cf648e9 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -325,7 +325,6 @@ type WasmLruCacheMetrics struct { Count uint32 } -// exported for testing func GetWasmLruCacheMetrics() *WasmLruCacheMetrics { metrics := C.stylus_get_lru_cache_metrics() @@ -341,12 +340,12 @@ func GetWasmLruCacheMetrics() *WasmLruCacheMetrics { } } -// exported for testing +// Used for testing func ClearWasmLruCache() { C.stylus_clear_lru_cache() } -// exported for testing +// Used for testing func GetAsmSizeEstimateBytes(module []byte, version uint16, debug bool) uint64 { return uint64(C.stylus_get_asm_size_estimate_bytes(goSlice(module), u16(version), cbool(debug))) } diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index 5b5c9a8938..2e6fff5c96 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -184,7 +184,7 @@ func populateStylusTargetCache(targetConfig *StylusTargetConfig) error { func (s *ExecutionEngine) Initialize(rustCacheCapacityMb uint32, targetConfig *StylusTargetConfig) error { if rustCacheCapacityMb != 0 { - s.SetWasmLruCacheCapacity(arbmath.SaturatingUMul(uint64(rustCacheCapacityMb), 1024*1024)) + programs.SetWasmLruCacheCapacity(arbmath.SaturatingUMul(uint64(rustCacheCapacityMb), 1024*1024)) } if err := populateStylusTargetCache(targetConfig); err != nil { return fmt.Errorf("error populating stylus target cache: %w", err) @@ -192,21 +192,6 @@ func (s *ExecutionEngine) Initialize(rustCacheCapacityMb uint32, targetConfig *S return nil } -// exported for testing -func (s *ExecutionEngine) SetWasmLruCacheCapacity(capacityBytes uint64) { - programs.SetWasmLruCacheCapacity(capacityBytes) -} - -// exported for testing -func (s *ExecutionEngine) GetWasmLruCacheMetrics() *programs.WasmLruCacheMetrics { - return programs.GetWasmLruCacheMetrics() -} - -// exported for testing -func (s *ExecutionEngine) ClearWasmLruCache() { - programs.ClearWasmLruCache() -} - func (s *ExecutionEngine) SetRecorder(recorder *BlockRecorder) { if s.Started() { panic("trying to set recorder after start") @@ -985,7 +970,7 @@ func (s *ExecutionEngine) Start(ctx_in context.Context) { case <-ctx.Done(): return case <-time.After(time.Minute): - s.GetWasmLruCacheMetrics() + programs.GetWasmLruCacheMetrics() } } }) diff --git a/system_tests/program_test.go b/system_tests/program_test.go index faa1649df9..b943d20d7e 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -2040,8 +2040,8 @@ func deployWasmAndGetSizeEstimateBytes( Require(t, err, ", wasmName:", wasmName) asmSizeEstimateBytes := programs.GetAsmSizeEstimateBytes(module, log.Version, true) + // just a sanity check if asmSizeEstimateBytes == 0 { - // just a sanity check Fatal(t, "asmSizeEstimateBytes is 0, wasmName:", wasmName) } return programAddress, asmSizeEstimateBytes @@ -2067,8 +2067,8 @@ func TestWasmLruCache(t *testing.T) { "math:", mathAsmSizeEstimateBytes, ) - builder.L2.ExecNode.ExecEngine.ClearWasmLruCache() - lruMetrics := builder.L2.ExecNode.ExecEngine.GetWasmLruCacheMetrics() + programs.ClearWasmLruCache() + lruMetrics := programs.GetWasmLruCacheMetrics() if lruMetrics.Count != 0 { t.Fatalf("lruMetrics.Count, expected: %v, actual: %v", 0, lruMetrics.Count) } @@ -2076,13 +2076,13 @@ func TestWasmLruCache(t *testing.T) { t.Fatalf("lruMetrics.SizeBytes, expected: %v, actual: %v", 0, lruMetrics.SizeBytes) } - builder.L2.ExecNode.ExecEngine.SetWasmLruCacheCapacity(fallibleAsmSizeEstimateBytes - 1) + programs.SetWasmLruCacheCapacity(fallibleAsmSizeEstimateBytes - 1) // fallible wasm program will not be cached since its size is greater than lru cache capacity tx := l2info.PrepareTxTo("Owner", &fallibleProgramAddress, l2info.TransferGas, nil, []byte{0x01}) Require(t, l2client.SendTransaction(ctx, tx)) _, err := EnsureTxSucceeded(ctx, l2client, tx) Require(t, err) - lruMetrics = builder.L2.ExecNode.ExecEngine.GetWasmLruCacheMetrics() + lruMetrics = programs.GetWasmLruCacheMetrics() if lruMetrics.Count != 0 { t.Fatalf("lruMetrics.Count, expected: %v, actual: %v", 0, lruMetrics.Count) } @@ -2090,7 +2090,7 @@ func TestWasmLruCache(t *testing.T) { t.Fatalf("lruMetrics.SizeBytes, expected: %v, actual: %v", 0, lruMetrics.SizeBytes) } - builder.L2.ExecNode.ExecEngine.SetWasmLruCacheCapacity( + programs.SetWasmLruCacheCapacity( fallibleAsmSizeEstimateBytes + keccakAsmSizeEstimateBytes + mathAsmSizeEstimateBytes - 1, ) // fallible wasm program will be cached @@ -2098,7 +2098,7 @@ func TestWasmLruCache(t *testing.T) { Require(t, l2client.SendTransaction(ctx, tx)) _, err = EnsureTxSucceeded(ctx, l2client, tx) Require(t, err) - lruMetrics = builder.L2.ExecNode.ExecEngine.GetWasmLruCacheMetrics() + lruMetrics = programs.GetWasmLruCacheMetrics() if lruMetrics.Count != 1 { t.Fatalf("lruMetrics.Count, expected: %v, actual: %v", 1, lruMetrics.Count) } @@ -2111,7 +2111,7 @@ func TestWasmLruCache(t *testing.T) { Require(t, l2client.SendTransaction(ctx, tx)) _, err = EnsureTxSucceeded(ctx, l2client, tx) Require(t, err) - lruMetrics = builder.L2.ExecNode.ExecEngine.GetWasmLruCacheMetrics() + lruMetrics = programs.GetWasmLruCacheMetrics() if lruMetrics.Count != 2 { t.Fatalf("lruMetrics.Count, expected: %v, actual: %v", 2, lruMetrics.Count) } @@ -2124,7 +2124,7 @@ func TestWasmLruCache(t *testing.T) { Require(t, l2client.SendTransaction(ctx, tx)) _, err = EnsureTxSucceeded(ctx, l2client, tx) Require(t, err) - lruMetrics = builder.L2.ExecNode.ExecEngine.GetWasmLruCacheMetrics() + lruMetrics = programs.GetWasmLruCacheMetrics() if lruMetrics.Count != 2 { t.Fatalf("lruMetrics.Count, expected: %v, actual: %v", 2, lruMetrics.Count) } From 17a62dad358217859c210920945d45cb9b8949db Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 22 Aug 2024 10:30:24 -0300 Subject: [PATCH 136/489] Removes unsafe from stylus_get_asm_size_estimate_bytes --- arbitrator/stylus/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arbitrator/stylus/src/lib.rs b/arbitrator/stylus/src/lib.rs index e5a80f5ee5..b38bc3044b 100644 --- a/arbitrator/stylus/src/lib.rs +++ b/arbitrator/stylus/src/lib.rs @@ -371,7 +371,7 @@ pub extern "C" fn stylus_clear_lru_cache() { /// Gets asm estimate size. /// Only used for testing purposes. #[no_mangle] -pub unsafe extern "C" fn stylus_get_asm_size_estimate_bytes( +pub extern "C" fn stylus_get_asm_size_estimate_bytes( module: GoSliceData, version: u16, debug: bool, From 8a3d6efe382a210a1fb0452ac678e4c70bd9f7c1 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 22 Aug 2024 11:01:36 -0300 Subject: [PATCH 137/489] Format rs files --- arbitrator/stylus/src/native.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/arbitrator/stylus/src/native.rs b/arbitrator/stylus/src/native.rs index b661435e52..7a82314fbc 100644 --- a/arbitrator/stylus/src/native.rs +++ b/arbitrator/stylus/src/native.rs @@ -128,13 +128,8 @@ impl> NativeInstance { if !env.evm_data.cached { long_term_tag = 0; } - let (module, store) = InitCache::insert( - module_hash, - module, - version, - long_term_tag, - debug, - )?; + let (module, store) = + InitCache::insert(module_hash, module, version, long_term_tag, debug)?; Self::from_module(module, store, env) } From eaa04db3708c58d8d3f32c571752f240844e4bda Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 28 Aug 2024 11:47:30 -0300 Subject: [PATCH 138/489] From println to eprintln in stylus cache --- arbitrator/stylus/src/cache.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index c2aaddb6e5..8f54b0cb10 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -186,7 +186,7 @@ impl InitCache { if long_term_tag != Self::ARBOS_TAG { if cache.lru.put_with_weight(key, item).is_err() { cache.lru_counters.does_not_fit += 1; - println!("{}", Self::DOES_NOT_FIT_MSG); + eprintln!("{}", Self::DOES_NOT_FIT_MSG); }; } else { cache.long_term.insert(key, item); @@ -203,7 +203,7 @@ impl InitCache { let mut cache = cache!(); if let Some(item) = cache.long_term.remove(&key) { if cache.lru.put_with_weight(key, item).is_err() { - println!("{}", Self::DOES_NOT_FIT_MSG); + eprintln!("{}", Self::DOES_NOT_FIT_MSG); } } } @@ -217,7 +217,7 @@ impl InitCache { for (key, item) in cache.long_term.drain() { // not all will fit, just a heuristic if cache.lru.put_with_weight(key, item).is_err() { - println!("{}", Self::DOES_NOT_FIT_MSG); + eprintln!("{}", Self::DOES_NOT_FIT_MSG); } } } From 1449a7b7fbea4c9e0dbe3df4befc7b37f354ae1d Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 28 Aug 2024 14:59:36 -0300 Subject: [PATCH 139/489] Renames asm_size_estimate_bytes to lru_entry_size_estimate_bytes --- arbitrator/stylus/src/cache.rs | 4 ++-- arbitrator/stylus/src/lib.rs | 6 ++--- arbos/programs/native.go | 4 ++-- system_tests/program_test.go | 40 +++++++++++++++++----------------- 4 files changed, 27 insertions(+), 27 deletions(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index 8f54b0cb10..cc0cf5344a 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -98,8 +98,8 @@ pub fn deserialize_module( ) -> Result<(Module, Engine, usize)> { let engine = CompileConfig::version(version, debug).engine(target_native()); let module = unsafe { Module::deserialize_unchecked(&engine, module)? }; - let asm_size_estimate_bytes = module.serialize()?.len(); - Ok((module, engine, asm_size_estimate_bytes)) + let lru_entry_size_estimate_bytes = module.serialize()?.len(); + Ok((module, engine, lru_entry_size_estimate_bytes)) } impl InitCache { diff --git a/arbitrator/stylus/src/lib.rs b/arbitrator/stylus/src/lib.rs index b38bc3044b..fc2e1dc6ea 100644 --- a/arbitrator/stylus/src/lib.rs +++ b/arbitrator/stylus/src/lib.rs @@ -368,16 +368,16 @@ pub extern "C" fn stylus_clear_lru_cache() { InitCache::clear_lru_cache() } -/// Gets asm estimate size. +/// Gets lru entry size in bytes. /// Only used for testing purposes. #[no_mangle] -pub extern "C" fn stylus_get_asm_size_estimate_bytes( +pub extern "C" fn stylus_get_lru_entry_size_estimate_bytes( module: GoSliceData, version: u16, debug: bool, ) -> u64 { match deserialize_module(module.slice(), version, debug) { Err(error) => panic!("tried to get invalid asm!: {error}"), - Ok((_, _, asm_size_estimate_bytes)) => asm_size_estimate_bytes.try_into().unwrap(), + Ok((_, _, lru_entry_size_estimate_bytes)) => lru_entry_size_estimate_bytes.try_into().unwrap(), } } diff --git a/arbos/programs/native.go b/arbos/programs/native.go index dd5cf648e9..57290dfa1e 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -346,8 +346,8 @@ func ClearWasmLruCache() { } // Used for testing -func GetAsmSizeEstimateBytes(module []byte, version uint16, debug bool) uint64 { - return uint64(C.stylus_get_asm_size_estimate_bytes(goSlice(module), u16(version), cbool(debug))) +func GetLruEntrySizeEstimateBytes(module []byte, version uint16, debug bool) uint64 { + return uint64(C.stylus_get_lru_entry_size_estimate_bytes(goSlice(module), u16(version), cbool(debug))) } const DefaultTargetDescriptionArm = "arm64-linux-unknown+neon" diff --git a/system_tests/program_test.go b/system_tests/program_test.go index b943d20d7e..1cbbf268f1 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -2008,7 +2008,7 @@ func checkWasmStoreContent(t *testing.T, wasmDb ethdb.KeyValueStore, targets []s } } -func deployWasmAndGetSizeEstimateBytes( +func deployWasmAndGetLruEntrySizeEstimateBytes( t *testing.T, builder *NodeBuilder, auth bind.TransactOpts, @@ -2039,12 +2039,12 @@ func deployWasmAndGetSizeEstimateBytes( module, err := statedb.TryGetActivatedAsm(rawdb.LocalTarget(), log.ModuleHash) Require(t, err, ", wasmName:", wasmName) - asmSizeEstimateBytes := programs.GetAsmSizeEstimateBytes(module, log.Version, true) + lruEntrySizeEstimateBytes := programs.GetLruEntrySizeEstimateBytes(module, log.Version, true) // just a sanity check - if asmSizeEstimateBytes == 0 { - Fatal(t, "asmSizeEstimateBytes is 0, wasmName:", wasmName) + if lruEntrySizeEstimateBytes == 0 { + Fatal(t, "lruEntrySizeEstimateBytes is 0, wasmName:", wasmName) } - return programAddress, asmSizeEstimateBytes + return programAddress, lruEntrySizeEstimateBytes } func TestWasmLruCache(t *testing.T) { @@ -2057,14 +2057,14 @@ func TestWasmLruCache(t *testing.T) { auth.GasLimit = 32000000 auth.Value = oneEth - fallibleProgramAddress, fallibleAsmSizeEstimateBytes := deployWasmAndGetSizeEstimateBytes(t, builder, auth, "fallible") - keccakProgramAddress, keccakAsmSizeEstimateBytes := deployWasmAndGetSizeEstimateBytes(t, builder, auth, "keccak") - mathProgramAddress, mathAsmSizeEstimateBytes := deployWasmAndGetSizeEstimateBytes(t, builder, auth, "math") + fallibleProgramAddress, fallibleLruEntrySizeEstimateBytes := deployWasmAndGetLruEntrySizeEstimateBytes(t, builder, auth, "fallible") + keccakProgramAddress, keccakLruEntrySizeEstimateBytes := deployWasmAndGetLruEntrySizeEstimateBytes(t, builder, auth, "keccak") + mathProgramAddress, mathLruEntrySizeEstimateBytes := deployWasmAndGetLruEntrySizeEstimateBytes(t, builder, auth, "math") t.Log( - "asmSizeEstimateBytes, ", - "fallible:", fallibleAsmSizeEstimateBytes, - "keccak:", keccakAsmSizeEstimateBytes, - "math:", mathAsmSizeEstimateBytes, + "lruEntrySizeEstimateBytes, ", + "fallible:", fallibleLruEntrySizeEstimateBytes, + "keccak:", keccakLruEntrySizeEstimateBytes, + "math:", mathLruEntrySizeEstimateBytes, ) programs.ClearWasmLruCache() @@ -2076,7 +2076,7 @@ func TestWasmLruCache(t *testing.T) { t.Fatalf("lruMetrics.SizeBytes, expected: %v, actual: %v", 0, lruMetrics.SizeBytes) } - programs.SetWasmLruCacheCapacity(fallibleAsmSizeEstimateBytes - 1) + programs.SetWasmLruCacheCapacity(fallibleLruEntrySizeEstimateBytes - 1) // fallible wasm program will not be cached since its size is greater than lru cache capacity tx := l2info.PrepareTxTo("Owner", &fallibleProgramAddress, l2info.TransferGas, nil, []byte{0x01}) Require(t, l2client.SendTransaction(ctx, tx)) @@ -2091,7 +2091,7 @@ func TestWasmLruCache(t *testing.T) { } programs.SetWasmLruCacheCapacity( - fallibleAsmSizeEstimateBytes + keccakAsmSizeEstimateBytes + mathAsmSizeEstimateBytes - 1, + fallibleLruEntrySizeEstimateBytes + keccakLruEntrySizeEstimateBytes + mathLruEntrySizeEstimateBytes - 1, ) // fallible wasm program will be cached tx = l2info.PrepareTxTo("Owner", &fallibleProgramAddress, l2info.TransferGas, nil, []byte{0x01}) @@ -2102,8 +2102,8 @@ func TestWasmLruCache(t *testing.T) { if lruMetrics.Count != 1 { t.Fatalf("lruMetrics.Count, expected: %v, actual: %v", 1, lruMetrics.Count) } - if lruMetrics.SizeBytes != fallibleAsmSizeEstimateBytes { - t.Fatalf("lruMetrics.SizeBytes, expected: %v, actual: %v", fallibleAsmSizeEstimateBytes, lruMetrics.SizeBytes) + if lruMetrics.SizeBytes != fallibleLruEntrySizeEstimateBytes { + t.Fatalf("lruMetrics.SizeBytes, expected: %v, actual: %v", fallibleLruEntrySizeEstimateBytes, lruMetrics.SizeBytes) } // keccak wasm program will be cached @@ -2115,8 +2115,8 @@ func TestWasmLruCache(t *testing.T) { if lruMetrics.Count != 2 { t.Fatalf("lruMetrics.Count, expected: %v, actual: %v", 2, lruMetrics.Count) } - if lruMetrics.SizeBytes != fallibleAsmSizeEstimateBytes+keccakAsmSizeEstimateBytes { - t.Fatalf("lruMetrics.SizeBytes, expected: %v, actual: %v", fallibleAsmSizeEstimateBytes+keccakAsmSizeEstimateBytes, lruMetrics.SizeBytes) + if lruMetrics.SizeBytes != fallibleLruEntrySizeEstimateBytes+keccakLruEntrySizeEstimateBytes { + t.Fatalf("lruMetrics.SizeBytes, expected: %v, actual: %v", fallibleLruEntrySizeEstimateBytes+keccakLruEntrySizeEstimateBytes, lruMetrics.SizeBytes) } // math wasm program will be cached, but fallible will be evicted since (fallible + keccak + math) > lruCacheCapacity @@ -2128,7 +2128,7 @@ func TestWasmLruCache(t *testing.T) { if lruMetrics.Count != 2 { t.Fatalf("lruMetrics.Count, expected: %v, actual: %v", 2, lruMetrics.Count) } - if lruMetrics.SizeBytes != keccakAsmSizeEstimateBytes+mathAsmSizeEstimateBytes { - t.Fatalf("lruMetrics.SizeBytes, expected: %v, actual: %v", keccakAsmSizeEstimateBytes+mathAsmSizeEstimateBytes, lruMetrics.SizeBytes) + if lruMetrics.SizeBytes != keccakLruEntrySizeEstimateBytes+mathLruEntrySizeEstimateBytes { + t.Fatalf("lruMetrics.SizeBytes, expected: %v, actual: %v", keccakLruEntrySizeEstimateBytes+mathLruEntrySizeEstimateBytes, lruMetrics.SizeBytes) } } From ba9e9d1c2cc5da358679081dc4456208dfac1fcc Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 28 Aug 2024 15:26:46 -0300 Subject: [PATCH 140/489] Adds extra bytes to lru_entry_size_bytes to account for overheads --- arbitrator/stylus/src/cache.rs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index cc0cf5344a..0bbeabcb4c 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -56,15 +56,15 @@ impl CacheKey { struct CacheItem { module: Module, engine: Engine, - asm_size_estimate_bytes: usize, + entry_size_estimate_bytes: usize, } impl CacheItem { - fn new(module: Module, engine: Engine, asm_size_estimate_bytes: usize) -> Self { + fn new(module: Module, engine: Engine, entry_size_estimate_bytes: usize) -> Self { Self { module, engine, - asm_size_estimate_bytes, + entry_size_estimate_bytes, } } @@ -78,7 +78,7 @@ impl WeightScale for CustomWeightScale { fn weight(&self, _key: &CacheKey, val: &CacheItem) -> usize { // clru defines that each entry consumes (weight + 1) of the cache capacity. // We subtract 1 since we only want to use the weight as the size of the entry. - val.asm_size_estimate_bytes.saturating_sub(1) + val.entry_size_estimate_bytes.saturating_sub(1) } } @@ -98,8 +98,12 @@ pub fn deserialize_module( ) -> Result<(Module, Engine, usize)> { let engine = CompileConfig::version(version, debug).engine(target_native()); let module = unsafe { Module::deserialize_unchecked(&engine, module)? }; - let lru_entry_size_estimate_bytes = module.serialize()?.len(); - Ok((module, engine, lru_entry_size_estimate_bytes)) + + let asm_size_estimate_bytes = module.serialize()?.len(); + // add 128 bytes for the cache item overhead + let entry_size_estimate_bytes = asm_size_estimate_bytes + 128; + + Ok((module, engine, entry_size_estimate_bytes)) } impl InitCache { @@ -178,9 +182,9 @@ impl InitCache { } drop(cache); - let (module, engine, asm_size_estimate_bytes) = deserialize_module(module, version, debug)?; + let (module, engine, entry_size_estimate_bytes) = deserialize_module(module, version, debug)?; - let item = CacheItem::new(module, engine, asm_size_estimate_bytes); + let item = CacheItem::new(module, engine, entry_size_estimate_bytes); let data = item.data(); let mut cache = cache!(); if long_term_tag != Self::ARBOS_TAG { From 5a82f007b6529033c95c981b9ad837ee622c0efe Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 28 Aug 2024 15:45:25 -0300 Subject: [PATCH 141/489] Format rs files --- arbitrator/stylus/src/cache.rs | 3 ++- arbitrator/stylus/src/lib.rs | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index 0bbeabcb4c..c1fdaaccee 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -182,7 +182,8 @@ impl InitCache { } drop(cache); - let (module, engine, entry_size_estimate_bytes) = deserialize_module(module, version, debug)?; + let (module, engine, entry_size_estimate_bytes) = + deserialize_module(module, version, debug)?; let item = CacheItem::new(module, engine, entry_size_estimate_bytes); let data = item.data(); diff --git a/arbitrator/stylus/src/lib.rs b/arbitrator/stylus/src/lib.rs index fc2e1dc6ea..32094752fc 100644 --- a/arbitrator/stylus/src/lib.rs +++ b/arbitrator/stylus/src/lib.rs @@ -378,6 +378,8 @@ pub extern "C" fn stylus_get_lru_entry_size_estimate_bytes( ) -> u64 { match deserialize_module(module.slice(), version, debug) { Err(error) => panic!("tried to get invalid asm!: {error}"), - Ok((_, _, lru_entry_size_estimate_bytes)) => lru_entry_size_estimate_bytes.try_into().unwrap(), + Ok((_, _, lru_entry_size_estimate_bytes)) => { + lru_entry_size_estimate_bytes.try_into().unwrap() + } } } From f144eb17addcc2b45ccf889347296ad9caf1a5da Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 29 Aug 2024 10:30:47 -0300 Subject: [PATCH 142/489] Renames rustCacheCapacityMb to rustCacheCapacityMB --- execution/gethexec/executionengine.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index 2e6fff5c96..4527c5f85e 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -182,9 +182,9 @@ func populateStylusTargetCache(targetConfig *StylusTargetConfig) error { return nil } -func (s *ExecutionEngine) Initialize(rustCacheCapacityMb uint32, targetConfig *StylusTargetConfig) error { - if rustCacheCapacityMb != 0 { - programs.SetWasmLruCacheCapacity(arbmath.SaturatingUMul(uint64(rustCacheCapacityMb), 1024*1024)) +func (s *ExecutionEngine) Initialize(rustCacheCapacityMB uint32, targetConfig *StylusTargetConfig) error { + if rustCacheCapacityMB != 0 { + programs.SetWasmLruCacheCapacity(arbmath.SaturatingUMul(uint64(rustCacheCapacityMB), 1024*1024)) } if err := populateStylusTargetCache(targetConfig); err != nil { return fmt.Errorf("error populating stylus target cache: %w", err) From 19ef7eaf78ceb07ad6eb5ddf3a7df39b8e920f95 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Fri, 6 Sep 2024 09:56:11 -0500 Subject: [PATCH 143/489] Revert "Fix ArbWASM precompile minInitGas() method from reverting" --- contracts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts b/contracts index a5c2d54561..f7894d3a6d 160000 --- a/contracts +++ b/contracts @@ -1 +1 @@ -Subproject commit a5c2d54561c2547e05d6118d6f072882c5fc7a9d +Subproject commit f7894d3a6d4035ba60f51a7f1334f0f2d4f02dce From 077213c4593ade8d5afac8eb54db5dd467cd033b Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Fri, 6 Sep 2024 11:39:09 -0500 Subject: [PATCH 144/489] Bump contracts to latest develop --- contracts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts b/contracts index f7894d3a6d..23fc796282 160000 --- a/contracts +++ b/contracts @@ -1 +1 @@ -Subproject commit f7894d3a6d4035ba60f51a7f1334f0f2d4f02dce +Subproject commit 23fc79628292aa5d604d449fed48937ae7faeb2f From b3918e5db50952a5a32ac093edbc66304110e5dc Mon Sep 17 00:00:00 2001 From: Joshua Colvin Date: Fri, 6 Sep 2024 12:42:20 -0700 Subject: [PATCH 145/489] Add buildspec.yml Move buildspec for AWS codebuild into nitro repo --- .github/buildspec.yml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/buildspec.yml diff --git a/.github/buildspec.yml b/.github/buildspec.yml new file mode 100644 index 0000000000..9b6503bb5d --- /dev/null +++ b/.github/buildspec.yml @@ -0,0 +1,36 @@ +version: 0.2 + +phases: + pre_build: + commands: + - git submodule update --init + - echo Logging in to Dockerhub.... + - docker login --username $DOCKERHUB_USERNAME --password $DOCKERHUB_PASSWORD + - aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin $REPOSITORY_URI + - COMMIT_HASH=$(git rev-parse --short=7 HEAD || echo "latest") + - VERSION_TAG=$(git tag --points-at HEAD | sed '/-/!s/$/_/' | sort -rV | sed 's/_$//' | head -n 1 | grep ^ || git show -s --pretty=%D | sed 's/, /\n/g' | grep -v '^origin/' |grep -v '^grafted\|HEAD\|master\|main$' || echo "dev") + - NITRO_VERSION=${VERSION_TAG}-${COMMIT_HASH} + - IMAGE_TAG=${NITRO_VERSION} + - NITRO_DATETIME=$(git show -s --date=iso-strict --format=%cd) + - NITRO_MODIFIED="false" + - echo ${NITRO_VERSION} > ./.nitro-tag.txt + build: + commands: + - echo Build started on `date` + - echo Building the Docker image ${NITRO_VERSION}... + - DOCKER_BUILDKIT=1 docker build . -t nitro-node-slim --target nitro-node-slim --build-arg version=$NITRO_VERSION --build-arg datetime=$NITRO_DATETIME --build-arg modified=$NITRO_MODIFIED + - DOCKER_BUILDKIT=1 docker build . -t nitro-node --target nitro-node --build-arg version=$NITRO_VERSION --build-arg datetime=$NITRO_DATETIME --build-arg modified=$NITRO_MODIFIED + - DOCKER_BUILDKIT=1 docker build . -t nitro-node-dev --target nitro-node-dev --build-arg version=$NITRO_VERSION --build-arg datetime=$NITRO_DATETIME --build-arg modified=$NITRO_MODIFIED + - DOCKER_BUILDKIT=1 docker build . -t nitro-node-validator --target nitro-node-validator --build-arg version=$NITRO_VERSION --build-arg datetime=$NITRO_DATETIME --build-arg modified=$NITRO_MODIFIED + - docker tag nitro-node:latest $REPOSITORY_URI:$IMAGE_TAG-$ARCH_TAG + - docker tag nitro-node-slim:latest $REPOSITORY_URI:$IMAGE_TAG-slim-$ARCH_TAG + - docker tag nitro-node-dev:latest $REPOSITORY_URI:$IMAGE_TAG-dev-$ARCH_TAG + - docker tag nitro-node-validator:latest $REPOSITORY_URI:$IMAGE_TAG-validator-$ARCH_TAG + post_build: + commands: + - echo Build completed on `date` + - echo pushing to repo + - docker push $REPOSITORY_URI:$IMAGE_TAG-$ARCH_TAG + - docker push $REPOSITORY_URI:$IMAGE_TAG-slim-$ARCH_TAG + - docker push $REPOSITORY_URI:$IMAGE_TAG-dev-$ARCH_TAG + - docker push $REPOSITORY_URI:$IMAGE_TAG-validator-$ARCH_TAG From 79d5a813c5b1471d08ccabd2e3187833ea3213fe Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Sat, 7 Sep 2024 10:43:54 -0500 Subject: [PATCH 146/489] Be more precise when selecting firstMsg in the batch poster --- arbnode/batch_poster.go | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/arbnode/batch_poster.go b/arbnode/batch_poster.go index 6b4b95f8e0..ea41b52574 100644 --- a/arbnode/batch_poster.go +++ b/arbnode/batch_poster.go @@ -1176,12 +1176,6 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) // There's nothing after the newest batch, therefore batch posting was not required return false, nil } - firstMsg, err := b.streamer.GetMessage(batchPosition.MessageCount) - if err != nil { - return false, err - } - // #nosec G115 - firstMsgTime := time.Unix(int64(firstMsg.Message.Header.Timestamp), 0) lastPotentialMsg, err := b.streamer.GetMessage(msgCount - 1) if err != nil { @@ -1189,7 +1183,7 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) } config := b.config() - forcePostBatch := config.MaxDelay <= 0 || time.Since(firstMsgTime) >= config.MaxDelay + forcePostBatch := config.MaxDelay <= 0 var l1BoundMaxBlockNumber uint64 = math.MaxUint64 var l1BoundMaxTimestamp uint64 = math.MaxUint64 @@ -1257,6 +1251,8 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) } } + var firstNonDelayedMsg *arbostypes.MessageWithMetadata + var firstUsefulMsg *arbostypes.MessageWithMetadata for b.building.msgCount < msgCount { msg, err := b.streamer.GetMessage(b.building.msgCount) if err != nil { @@ -1299,6 +1295,9 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) forcePostBatch = true } b.building.haveUsefulMessage = true + if firstUsefulMsg == nil { + firstUsefulMsg = msg + } break } if config.CheckBatchCorrectness { @@ -1309,13 +1308,28 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) } if msg.Message.Header.Kind != arbostypes.L1MessageType_BatchPostingReport { b.building.haveUsefulMessage = true + if firstUsefulMsg == nil { + firstUsefulMsg = msg + } + } + if !isDelayed && firstNonDelayedMsg == nil { + firstNonDelayedMsg = msg } b.building.msgCount++ } - if hasL1Bound && config.ReorgResistanceMargin > 0 { - firstMsgBlockNumber := firstMsg.Message.Header.BlockNumber - firstMsgTimeStamp := firstMsg.Message.Header.Timestamp + firstUsefulMsgTime := time.Now() + if firstUsefulMsg != nil { + // #nosec G115 + firstUsefulMsgTime = time.Unix(int64(firstUsefulMsg.Message.Header.Timestamp), 0) + if time.Since(firstUsefulMsgTime) >= config.MaxDelay { + forcePostBatch = true + } + } + + if firstNonDelayedMsg != nil && hasL1Bound && config.ReorgResistanceMargin > 0 { + firstMsgBlockNumber := firstNonDelayedMsg.Message.Header.BlockNumber + firstMsgTimeStamp := firstNonDelayedMsg.Message.Header.Timestamp batchNearL1BoundMinBlockNumber := firstMsgBlockNumber <= arbmath.SaturatingUAdd(l1BoundMinBlockNumber, uint64(config.ReorgResistanceMargin/ethPosBlockTime)) batchNearL1BoundMinTimestamp := firstMsgTimeStamp <= arbmath.SaturatingUAdd(l1BoundMinTimestamp, uint64(config.ReorgResistanceMargin/time.Second)) if batchNearL1BoundMinTimestamp || batchNearL1BoundMinBlockNumber { @@ -1463,7 +1477,7 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) } tx, err := b.dataPoster.PostTransaction(ctx, - firstMsgTime, + firstUsefulMsgTime, nonce, newMeta, b.seqInboxAddr, From 7591f1c5a9625aaced402f3e5160217c6633a665 Mon Sep 17 00:00:00 2001 From: Pepper Lebeck-Jobe Date: Sun, 8 Sep 2024 02:11:49 +0200 Subject: [PATCH 147/489] Temporarily increase the time for bridging balances This change allows the test to pass even on a heavily-loaded mac. Related to: NIT-2768 --- system_tests/common_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 457dae0910..caf749788c 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -618,7 +618,7 @@ func BridgeBalance( break } TransferBalance(t, "Faucet", "User", big.NewInt(1), l1info, l1client, ctx) - if i > 20 { + if i > 200 { Fatal(t, "bridging failed") } <-time.After(time.Millisecond * 100) From bf7b6ac85058feab3e42f9f85edeb5618935b02b Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Sun, 8 Sep 2024 11:13:31 -0500 Subject: [PATCH 148/489] Resolve parent chain block by hash when resolving delayed messages --- arbnode/delayed.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arbnode/delayed.go b/arbnode/delayed.go index c166aa2b90..082f0ecf9d 100644 --- a/arbnode/delayed.go +++ b/arbnode/delayed.go @@ -215,7 +215,7 @@ func (b *DelayedBridge) logsToDeliveredMessages(ctx context.Context, logs []type } messages := make([]*DelayedInboxMessage, 0, len(logs)) - var lastParentChainBlockNumber uint64 + var lastParentChainBlockHash common.Hash var lastL1BlockNumber uint64 for _, parsedLog := range parsedLogs { msgKey := common.BigToHash(parsedLog.MessageIndex) @@ -228,17 +228,17 @@ func (b *DelayedBridge) logsToDeliveredMessages(ctx context.Context, logs []type } requestId := common.BigToHash(parsedLog.MessageIndex) - parentChainBlockNumber := parsedLog.Raw.BlockNumber + parentChainBlockHash := parsedLog.Raw.BlockHash var l1BlockNumber uint64 - if lastParentChainBlockNumber == parentChainBlockNumber && lastParentChainBlockNumber > 0 { + if lastParentChainBlockHash == parentChainBlockHash && lastParentChainBlockHash != (common.Hash{}) { l1BlockNumber = lastL1BlockNumber } else { - var err error - l1BlockNumber, err = arbutil.CorrespondingL1BlockNumber(ctx, b.client, parentChainBlockNumber) + parentChainHeader, err := b.client.HeaderByHash(ctx, parentChainBlockHash) if err != nil { return nil, err } - lastParentChainBlockNumber = parentChainBlockNumber + l1BlockNumber = arbutil.ParentHeaderToL1BlockNumber(parentChainHeader) + lastParentChainBlockHash = parentChainBlockHash lastL1BlockNumber = l1BlockNumber } msg := &DelayedInboxMessage{ From 070a43a0274891f1c562b0c1489e9af6e7b25b4f Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Mon, 9 Sep 2024 13:34:52 +0530 Subject: [PATCH 149/489] update dataposter test to support *ethclient.Client --- arbnode/dataposter/data_poster.go | 4 +- arbnode/dataposter/dataposter_test.go | 63 ++++++++++++--------------- arbutil/wait_for_l1.go | 15 ------- staker/rollup_watcher.go | 6 +-- staker/txbuilder/builder.go | 4 +- 5 files changed, 34 insertions(+), 58 deletions(-) diff --git a/arbnode/dataposter/data_poster.go b/arbnode/dataposter/data_poster.go index 6a483929b2..8a9d695073 100644 --- a/arbnode/dataposter/data_poster.go +++ b/arbnode/dataposter/data_poster.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/consensus/misc/eip4844" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto/kzg4844" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" @@ -39,7 +40,6 @@ import ( "github.com/offchainlabs/nitro/arbnode/dataposter/noop" "github.com/offchainlabs/nitro/arbnode/dataposter/slice" "github.com/offchainlabs/nitro/arbnode/dataposter/storage" - "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/util/arbmath" "github.com/offchainlabs/nitro/util/blobs" "github.com/offchainlabs/nitro/util/headerreader" @@ -69,7 +69,7 @@ var ( type DataPoster struct { stopwaiter.StopWaiter headerReader *headerreader.HeaderReader - client arbutil.L1Interface + client *ethclient.Client auth *bind.TransactOpts signer signerFn config ConfigFetcher diff --git a/arbnode/dataposter/dataposter_test.go b/arbnode/dataposter/dataposter_test.go index 7f2f61c07e..5c8ab68541 100644 --- a/arbnode/dataposter/dataposter_test.go +++ b/arbnode/dataposter/dataposter_test.go @@ -2,17 +2,18 @@ package dataposter import ( "context" + "errors" "fmt" "math/big" "testing" "time" "github.com/Knetic/govaluate" - "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi/bind" - "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" @@ -152,46 +153,36 @@ func TestMaxFeeCapFormulaCalculation(t *testing.T) { } } -type stubL1Client struct { +type stubL1ClientInterface struct { senderNonce uint64 suggestedGasTipCap *big.Int - - // Define most of the required methods that aren't used by feeAndTipCaps - backends.SimulatedBackend -} - -func (c *stubL1Client) NonceAt(ctx context.Context, account common.Address, blockNumber *big.Int) (uint64, error) { - return c.senderNonce, nil -} - -func (c *stubL1Client) SuggestGasTipCap(ctx context.Context) (*big.Int, error) { - return c.suggestedGasTipCap, nil -} - -// Not used but we need to define -func (c *stubL1Client) BlockNumber(ctx context.Context) (uint64, error) { - return 0, nil -} - -func (c *stubL1Client) CallContractAtHash(ctx context.Context, msg ethereum.CallMsg, blockHash common.Hash) ([]byte, error) { - return []byte{}, nil } -func (c *stubL1Client) CodeAtHash(ctx context.Context, address common.Address, blockHash common.Hash) ([]byte, error) { - return []byte{}, nil +func (c *stubL1ClientInterface) CallContext(ctx_in context.Context, result interface{}, method string, args ...interface{}) error { + switch method { + case "eth_getTransactionCount": + ptr, ok := result.(*hexutil.Uint64) + if !ok { + return errors.New("result is not a *hexutil.Uint64") + } + *ptr = hexutil.Uint64(c.senderNonce) + case "eth_maxPriorityFeePerGas": + ptr, ok := result.(*hexutil.Big) + if !ok { + return errors.New("result is not a *hexutil.Big") + } + *ptr = hexutil.Big(*c.suggestedGasTipCap) + } + return nil } -func (c *stubL1Client) ChainID(ctx context.Context) (*big.Int, error) { +func (c *stubL1ClientInterface) EthSubscribe(ctx context.Context, channel interface{}, args ...interface{}) (*rpc.ClientSubscription, error) { return nil, nil } - -func (c *stubL1Client) Client() rpc.ClientInterface { +func (c *stubL1ClientInterface) BatchCallContext(ctx context.Context, b []rpc.BatchElem) error { return nil } - -func (c *stubL1Client) TransactionSender(ctx context.Context, tx *types.Transaction, block common.Hash, index uint) (common.Address, error) { - return common.Address{}, nil -} +func (c *stubL1ClientInterface) Close() {} func TestFeeAndTipCaps_EnoughBalance_NoBacklog_NoUnconfirmed_BlobTx(t *testing.T) { conf := func() *DataPosterConfig { @@ -223,10 +214,10 @@ func TestFeeAndTipCaps_EnoughBalance_NoBacklog_NoUnconfirmed_BlobTx(t *testing.T extraBacklog: func() uint64 { return 0 }, balance: big.NewInt(0).Mul(big.NewInt(params.Ether), big.NewInt(10)), usingNoOpStorage: false, - client: &stubL1Client{ + client: ethclient.NewClient(&stubL1ClientInterface{ senderNonce: 1, suggestedGasTipCap: big.NewInt(2 * params.GWei), - }, + }), auth: &bind.TransactOpts{ From: common.Address{}, }, @@ -354,10 +345,10 @@ func TestFeeAndTipCaps_RBF_RisingBlobFee_FallingBaseFee(t *testing.T) { extraBacklog: func() uint64 { return 0 }, balance: big.NewInt(0).Mul(big.NewInt(params.Ether), big.NewInt(10)), usingNoOpStorage: false, - client: &stubL1Client{ + client: ethclient.NewClient(&stubL1ClientInterface{ senderNonce: 1, suggestedGasTipCap: big.NewInt(2 * params.GWei), - }, + }), auth: &bind.TransactOpts{ From: common.Address{}, }, diff --git a/arbutil/wait_for_l1.go b/arbutil/wait_for_l1.go index 49eea6af78..80dd356b24 100644 --- a/arbutil/wait_for_l1.go +++ b/arbutil/wait_for_l1.go @@ -10,27 +10,12 @@ import ( "math/big" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/ethclient" - "github.com/ethereum/go-ethereum/rpc" ) -type L1Interface interface { - bind.ContractBackend - bind.BlockHashContractCaller - ethereum.ChainReader - ethereum.ChainStateReader - ethereum.TransactionReader - TransactionSender(ctx context.Context, tx *types.Transaction, block common.Hash, index uint) (common.Address, error) - BlockNumber(ctx context.Context) (uint64, error) - PendingCallContract(ctx context.Context, msg ethereum.CallMsg) ([]byte, error) - ChainID(ctx context.Context) (*big.Int, error) - Client() rpc.ClientInterface -} - func SendTxAsCall(ctx context.Context, client *ethclient.Client, tx *types.Transaction, from common.Address, blockNum *big.Int, unlimitedGas bool) ([]byte, error) { var gas uint64 if unlimitedGas { diff --git a/staker/rollup_watcher.go b/staker/rollup_watcher.go index 9c51e659f5..a7221d335d 100644 --- a/staker/rollup_watcher.go +++ b/staker/rollup_watcher.go @@ -48,18 +48,18 @@ type RollupWatcher struct { *rollupgen.RollupUserLogic address common.Address fromBlock *big.Int - client WatcherL1Interface + client RollupWatcherL1Interface baseCallOpts bind.CallOpts unSupportedL3Method atomic.Bool } -type WatcherL1Interface interface { +type RollupWatcherL1Interface interface { bind.ContractBackend HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) FilterLogs(ctx context.Context, q ethereum.FilterQuery) ([]types.Log, error) } -func NewRollupWatcher(address common.Address, client WatcherL1Interface, callOpts bind.CallOpts) (*RollupWatcher, error) { +func NewRollupWatcher(address common.Address, client RollupWatcherL1Interface, callOpts bind.CallOpts) (*RollupWatcher, error) { con, err := rollupgen.NewRollupUserLogic(address, client) if err != nil { return nil, err diff --git a/staker/txbuilder/builder.go b/staker/txbuilder/builder.go index fa40c6a816..f52b03a781 100644 --- a/staker/txbuilder/builder.go +++ b/staker/txbuilder/builder.go @@ -27,8 +27,8 @@ type ValidatorWalletInterface interface { // Builder combines any transactions sent to it via SendTransaction into one batch, // which is then sent to the validator wallet. // This lets the validator make multiple atomic transactions. -// This inherits from an eth client so it can be used as an L1Interface, -// where it transparently intercepts calls to SendTransaction and queues them for the next batch. +// This inherits from an ethclient.Client so it can be used to transparently +// intercept calls to SendTransaction and queue them for the next batch. type Builder struct { *ethclient.Client transactions []*types.Transaction From c8226b22cb8ea332a5951ea8fef0b214c648e972 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 9 Sep 2024 11:04:46 -0300 Subject: [PATCH 150/489] Gives preference to ipv4 when connecting to feed --- broadcastclient/broadcastclient.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/broadcastclient/broadcastclient.go b/broadcastclient/broadcastclient.go index 7d27c57fe9..4e97ca8cd0 100644 --- a/broadcastclient/broadcastclient.go +++ b/broadcastclient/broadcastclient.go @@ -280,6 +280,18 @@ func (bc *BroadcastClient) connect(ctx context.Context, nextSeqNum arbutil.Messa MinVersion: tls.VersionTLS12, }, Extensions: extensions, + NetDial: func(ctx context.Context, network, addr string) (net.Conn, error) { + var netDialer net.Dialer + // For tcp connections, prefer IPv4 over IPv6 to avoid rate limiting issues + if network == "tcp" { + conn, err := netDialer.DialContext(ctx, "tcp4", addr) + if err == nil { + return conn, nil + } + return netDialer.DialContext(ctx, "tcp6", addr) + } + return netDialer.DialContext(ctx, network, addr) + }, } if bc.isShuttingDown() { From e950d4bc0d7f531544d245c2b0b397342707fbe3 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Mon, 9 Sep 2024 11:46:00 -0500 Subject: [PATCH 151/489] Store firstUsefulMsg and firstNonDelayedMsg in buildingBatch --- arbnode/batch_poster.go | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/arbnode/batch_poster.go b/arbnode/batch_poster.go index ea41b52574..072b3254ce 100644 --- a/arbnode/batch_poster.go +++ b/arbnode/batch_poster.go @@ -716,12 +716,14 @@ type batchSegments struct { } type buildingBatch struct { - segments *batchSegments - startMsgCount arbutil.MessageIndex - msgCount arbutil.MessageIndex - haveUsefulMessage bool - use4844 bool - muxBackend *simulatedMuxBackend + segments *batchSegments + startMsgCount arbutil.MessageIndex + msgCount arbutil.MessageIndex + haveUsefulMessage bool + use4844 bool + muxBackend *simulatedMuxBackend + firstNonDelayedMsg *arbostypes.MessageWithMetadata + firstUsefulMsg *arbostypes.MessageWithMetadata } func newBatchSegments(firstDelayed uint64, config *BatchPosterConfig, backlog uint64, use4844 bool) *batchSegments { @@ -1251,8 +1253,6 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) } } - var firstNonDelayedMsg *arbostypes.MessageWithMetadata - var firstUsefulMsg *arbostypes.MessageWithMetadata for b.building.msgCount < msgCount { msg, err := b.streamer.GetMessage(b.building.msgCount) if err != nil { @@ -1295,8 +1295,8 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) forcePostBatch = true } b.building.haveUsefulMessage = true - if firstUsefulMsg == nil { - firstUsefulMsg = msg + if b.building.firstUsefulMsg == nil { + b.building.firstUsefulMsg = msg } break } @@ -1308,28 +1308,28 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) } if msg.Message.Header.Kind != arbostypes.L1MessageType_BatchPostingReport { b.building.haveUsefulMessage = true - if firstUsefulMsg == nil { - firstUsefulMsg = msg + if b.building.firstUsefulMsg == nil { + b.building.firstUsefulMsg = msg } } - if !isDelayed && firstNonDelayedMsg == nil { - firstNonDelayedMsg = msg + if !isDelayed && b.building.firstNonDelayedMsg == nil { + b.building.firstNonDelayedMsg = msg } b.building.msgCount++ } firstUsefulMsgTime := time.Now() - if firstUsefulMsg != nil { + if b.building.firstUsefulMsg != nil { // #nosec G115 - firstUsefulMsgTime = time.Unix(int64(firstUsefulMsg.Message.Header.Timestamp), 0) + firstUsefulMsgTime = time.Unix(int64(b.building.firstUsefulMsg.Message.Header.Timestamp), 0) if time.Since(firstUsefulMsgTime) >= config.MaxDelay { forcePostBatch = true } } - if firstNonDelayedMsg != nil && hasL1Bound && config.ReorgResistanceMargin > 0 { - firstMsgBlockNumber := firstNonDelayedMsg.Message.Header.BlockNumber - firstMsgTimeStamp := firstNonDelayedMsg.Message.Header.Timestamp + if b.building.firstNonDelayedMsg != nil && hasL1Bound && config.ReorgResistanceMargin > 0 { + firstMsgBlockNumber := b.building.firstNonDelayedMsg.Message.Header.BlockNumber + firstMsgTimeStamp := b.building.firstNonDelayedMsg.Message.Header.Timestamp batchNearL1BoundMinBlockNumber := firstMsgBlockNumber <= arbmath.SaturatingUAdd(l1BoundMinBlockNumber, uint64(config.ReorgResistanceMargin/ethPosBlockTime)) batchNearL1BoundMinTimestamp := firstMsgTimeStamp <= arbmath.SaturatingUAdd(l1BoundMinTimestamp, uint64(config.ReorgResistanceMargin/time.Second)) if batchNearL1BoundMinTimestamp || batchNearL1BoundMinBlockNumber { From 8b9f4ae319d11630af5a5ff53ff770071f26691b Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Mon, 9 Sep 2024 12:09:46 -0500 Subject: [PATCH 152/489] Improve block validator test stability --- system_tests/block_validator_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/system_tests/block_validator_test.go b/system_tests/block_validator_test.go index eef6c29b7a..cca5572f4e 100644 --- a/system_tests/block_validator_test.go +++ b/system_tests/block_validator_test.go @@ -63,7 +63,6 @@ func testBlockValidatorSimple(t *testing.T, opts Options) { var delayEvery int if opts.workloadLoops > 1 { - l1NodeConfigA.BatchPoster.MaxDelay = time.Millisecond * 500 delayEvery = opts.workloadLoops / 3 } From 36dc6fc827fd6475c582e8a93433e6b33124c8f8 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Mon, 9 Sep 2024 12:34:23 -0600 Subject: [PATCH 153/489] fix activateProgramInternal --- arbos/programs/native.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arbos/programs/native.go b/arbos/programs/native.go index 377e25a31e..e834888b47 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -108,10 +108,10 @@ func activateProgramInternal( } results := make(chan result, len(targets)) for _, target := range targets { + target := target if target == rawdb.TargetWavm { results <- result{target, module, nil} } else { - target := target go func() { output := &rustBytes{} status_asm := C.stylus_compile( From 01e1c4af5b4dcd3f05c364bccc1f29e409ef25e9 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Mon, 9 Sep 2024 16:04:34 -0500 Subject: [PATCH 154/489] Fix new G115 lints in golangci-lint v1.61.0 --- arbnode/batch_poster.go | 13 +++++++--- arbnode/dataposter/data_poster.go | 10 ++++---- arbnode/dataposter/dataposter_test.go | 4 +-- arbnode/dataposter/dbstorage/storage.go | 4 +-- arbnode/dataposter/storage_test.go | 17 +++++++++--- arbnode/delayed_sequencer.go | 2 ++ arbnode/inbox_reader.go | 1 + arbnode/transaction_streamer.go | 17 +++++++----- arbos/activate_test.go | 1 + arbos/addressSet/addressSet.go | 1 + arbos/addressTable/addressTable.go | 1 + arbos/arbosState/initialization_test.go | 1 + arbos/arbosState/initialize.go | 4 +-- arbos/l1pricing_test.go | 2 ++ arbos/merkleAccumulator/merkleAccumulator.go | 1 + arbos/programs/data_pricer.go | 4 +-- arbos/retryable_test.go | 3 +++ arbos/retryables/retryable.go | 4 ++- arbos/storage/storage.go | 9 ++++--- arbutil/block_message_relation.go | 1 + blocks_reexecutor/blocks_reexecutor.go | 1 + broadcastclient/broadcastclient_test.go | 2 ++ broadcaster/broadcaster.go | 1 + broadcaster/message/message.go | 1 + cmd/datool/datool.go | 3 +++ cmd/nitro/init.go | 1 + cmd/nitro/nitro.go | 4 +++ cmd/pruning/pruning.go | 1 + das/aggregator.go | 1 + das/dasRpcClient.go | 1 + das/dasRpcServer.go | 4 +-- das/das_test.go | 2 ++ das/db_storage_service.go | 1 + das/factory.go | 8 +----- das/fallback_storage_service.go | 1 + das/local_file_storage_service.go | 1 + das/local_file_storage_service_test.go | 13 ++++++++++ das/redis_storage_service_test.go | 1 + das/redundant_storage_test.go | 1 + das/restful_server_test.go | 1 + das/rpc_aggregator.go | 2 +- das/s3_storage_service_test.go | 1 + das/simple_das_reader_aggregator_test.go | 4 +++ das/store_signing_test.go | 1 + das/syncing_fallback_storage.go | 1 + execution/gethexec/api.go | 4 +++ execution/gethexec/blockchain.go | 1 + execution/gethexec/sequencer.go | 4 +-- execution/nodeInterface/NodeInterface.go | 3 +++ precompiles/precompile.go | 3 +++ precompiles/precompile_test.go | 2 ++ relay/relay_stress_test.go | 1 + staker/challenge-cache/cache_test.go | 15 ++++++----- staker/challenge_manager.go | 1 + system_tests/bloom_test.go | 6 +++++ system_tests/common_test.go | 2 +- system_tests/estimation_test.go | 2 +- system_tests/nodeinterface_test.go | 2 ++ system_tests/outbox_test.go | 3 +++ system_tests/recreatestate_rpc_test.go | 5 ++++ system_tests/seqfeed_test.go | 4 +-- system_tests/seqinbox_test.go | 3 +++ system_tests/stylus_trace_test.go | 1 + util/arbmath/bips.go | 27 ++++++++++++++------ util/arbmath/math.go | 7 +++-- util/arbmath/math_test.go | 1 + util/headerreader/blob_client.go | 1 + util/merkletree/merkleEventProof_test.go | 1 + validator/server_arb/machine_cache.go | 11 ++++---- validator/server_jit/jit_machine.go | 1 + wavmio/stub.go | 13 +++++----- 71 files changed, 207 insertions(+), 75 deletions(-) diff --git a/arbnode/batch_poster.go b/arbnode/batch_poster.go index 6b4b95f8e0..44b360e76e 100644 --- a/arbnode/batch_poster.go +++ b/arbnode/batch_poster.go @@ -168,7 +168,7 @@ type BatchPosterConfig struct { L1BlockBound string `koanf:"l1-block-bound" reload:"hot"` L1BlockBoundBypass time.Duration `koanf:"l1-block-bound-bypass" reload:"hot"` UseAccessLists bool `koanf:"use-access-lists" reload:"hot"` - GasEstimateBaseFeeMultipleBips arbmath.Bips `koanf:"gas-estimate-base-fee-multiple-bips"` + GasEstimateBaseFeeMultipleBips arbmath.UBips `koanf:"gas-estimate-base-fee-multiple-bips"` Dangerous BatchPosterDangerousConfig `koanf:"dangerous"` ReorgResistanceMargin time.Duration `koanf:"reorg-resistance-margin" reload:"hot"` CheckBatchCorrectness bool `koanf:"check-batch-correctness"` @@ -253,7 +253,7 @@ var DefaultBatchPosterConfig = BatchPosterConfig{ L1BlockBoundBypass: time.Hour, UseAccessLists: true, RedisLock: redislock.DefaultCfg, - GasEstimateBaseFeeMultipleBips: arbmath.OneInBips * 3 / 2, + GasEstimateBaseFeeMultipleBips: arbmath.OneInUBips * 3 / 2, ReorgResistanceMargin: 10 * time.Minute, CheckBatchCorrectness: true, } @@ -285,7 +285,7 @@ var TestBatchPosterConfig = BatchPosterConfig{ L1BlockBound: "", L1BlockBoundBypass: time.Hour, UseAccessLists: true, - GasEstimateBaseFeeMultipleBips: arbmath.OneInBips * 3 / 2, + GasEstimateBaseFeeMultipleBips: arbmath.OneInUBips * 3 / 2, CheckBatchCorrectness: true, } @@ -1035,7 +1035,7 @@ func (b *BatchPoster) estimateGas(ctx context.Context, sequencerMessage []byte, if err != nil { return 0, err } - maxFeePerGas := arbmath.BigMulByBips(latestHeader.BaseFee, config.GasEstimateBaseFeeMultipleBips) + maxFeePerGas := arbmath.BigMulByUBips(latestHeader.BaseFee, config.GasEstimateBaseFeeMultipleBips) if useNormalEstimation { _, realBlobHashes, err := blobs.ComputeCommitmentsAndHashes(realBlobs) if err != nil { @@ -1250,7 +1250,9 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) l1BoundMinTimestamp = arbmath.SaturatingUSub(latestHeader.Time, arbmath.BigToUintSaturating(maxTimeVariationDelaySeconds)) if config.L1BlockBoundBypass > 0 { + // #nosec G115 blockNumberWithPadding := arbmath.SaturatingUAdd(latestBlockNumber, uint64(config.L1BlockBoundBypass/ethPosBlockTime)) + // #nosec G115 timestampWithPadding := arbmath.SaturatingUAdd(latestHeader.Time, uint64(config.L1BlockBoundBypass/time.Second)) l1BoundMinBlockNumberWithBypass = arbmath.SaturatingUSub(blockNumberWithPadding, arbmath.BigToUintSaturating(maxTimeVariationDelayBlocks)) l1BoundMinTimestampWithBypass = arbmath.SaturatingUSub(timestampWithPadding, arbmath.BigToUintSaturating(maxTimeVariationDelaySeconds)) @@ -1316,7 +1318,9 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) if hasL1Bound && config.ReorgResistanceMargin > 0 { firstMsgBlockNumber := firstMsg.Message.Header.BlockNumber firstMsgTimeStamp := firstMsg.Message.Header.Timestamp + // #nosec G115 batchNearL1BoundMinBlockNumber := firstMsgBlockNumber <= arbmath.SaturatingUAdd(l1BoundMinBlockNumber, uint64(config.ReorgResistanceMargin/ethPosBlockTime)) + // #nosec G115 batchNearL1BoundMinTimestamp := firstMsgTimeStamp <= arbmath.SaturatingUAdd(l1BoundMinTimestamp, uint64(config.ReorgResistanceMargin/time.Second)) if batchNearL1BoundMinTimestamp || batchNearL1BoundMinBlockNumber { log.Error( @@ -1361,6 +1365,7 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) batchPosterDAFailureCounter.Inc(1) return false, fmt.Errorf("%w: nonce changed from %d to %d while creating batch", storage.ErrStorageRace, nonce, gotNonce) } + // #nosec G115 sequencerMsg, err = b.dapWriter.Store(ctx, sequencerMsg, uint64(time.Now().Add(config.DASRetentionPeriod).Unix()), config.DisableDapFallbackStoreDataOnChain) if err != nil { batchPosterDAFailureCounter.Inc(1) diff --git a/arbnode/dataposter/data_poster.go b/arbnode/dataposter/data_poster.go index 6a483929b2..31b9a983db 100644 --- a/arbnode/dataposter/data_poster.go +++ b/arbnode/dataposter/data_poster.go @@ -638,11 +638,11 @@ func (p *DataPoster) feeAndTipCaps(ctx context.Context, nonce uint64, gasLimit u if config.MaxFeeBidMultipleBips > 0 { // Limit the fee caps to be no greater than max(MaxFeeBidMultipleBips, minRbf) - maxNonBlobFee := arbmath.BigMulByBips(currentNonBlobFee, config.MaxFeeBidMultipleBips) + maxNonBlobFee := arbmath.BigMulByUBips(currentNonBlobFee, config.MaxFeeBidMultipleBips) if lastTx != nil { maxNonBlobFee = arbmath.BigMax(maxNonBlobFee, arbmath.BigMulByBips(lastTx.GasFeeCap(), minRbfIncrease)) } - maxBlobFee := arbmath.BigMulByBips(currentBlobFee, config.MaxFeeBidMultipleBips) + maxBlobFee := arbmath.BigMulByUBips(currentBlobFee, config.MaxFeeBidMultipleBips) if lastTx != nil && lastTx.BlobGasFeeCap() != nil { maxBlobFee = arbmath.BigMax(maxBlobFee, arbmath.BigMulByBips(lastTx.BlobGasFeeCap(), minRbfIncrease)) } @@ -1241,7 +1241,7 @@ type DataPosterConfig struct { MinBlobTxTipCapGwei float64 `koanf:"min-blob-tx-tip-cap-gwei" reload:"hot"` MaxTipCapGwei float64 `koanf:"max-tip-cap-gwei" reload:"hot"` MaxBlobTxTipCapGwei float64 `koanf:"max-blob-tx-tip-cap-gwei" reload:"hot"` - MaxFeeBidMultipleBips arbmath.Bips `koanf:"max-fee-bid-multiple-bips" reload:"hot"` + MaxFeeBidMultipleBips arbmath.UBips `koanf:"max-fee-bid-multiple-bips" reload:"hot"` NonceRbfSoftConfs uint64 `koanf:"nonce-rbf-soft-confs" reload:"hot"` AllocateMempoolBalance bool `koanf:"allocate-mempool-balance" reload:"hot"` UseDBStorage bool `koanf:"use-db-storage"` @@ -1345,7 +1345,7 @@ var DefaultDataPosterConfig = DataPosterConfig{ MinBlobTxTipCapGwei: 1, // default geth minimum, and relays aren't likely to accept lower values given propagation time MaxTipCapGwei: 1.2, MaxBlobTxTipCapGwei: 1, // lower than normal because 4844 rbf is a minimum of a 2x - MaxFeeBidMultipleBips: arbmath.OneInBips * 10, + MaxFeeBidMultipleBips: arbmath.OneInUBips * 10, NonceRbfSoftConfs: 1, AllocateMempoolBalance: true, UseDBStorage: true, @@ -1380,7 +1380,7 @@ var TestDataPosterConfig = DataPosterConfig{ MinBlobTxTipCapGwei: 1, MaxTipCapGwei: 5, MaxBlobTxTipCapGwei: 1, - MaxFeeBidMultipleBips: arbmath.OneInBips * 10, + MaxFeeBidMultipleBips: arbmath.OneInUBips * 10, NonceRbfSoftConfs: 1, AllocateMempoolBalance: true, UseDBStorage: false, diff --git a/arbnode/dataposter/dataposter_test.go b/arbnode/dataposter/dataposter_test.go index 7f2f61c07e..d2c49427be 100644 --- a/arbnode/dataposter/dataposter_test.go +++ b/arbnode/dataposter/dataposter_test.go @@ -204,7 +204,7 @@ func TestFeeAndTipCaps_EnoughBalance_NoBacklog_NoUnconfirmed_BlobTx(t *testing.T MinBlobTxTipCapGwei: 1, MaxTipCapGwei: 5, MaxBlobTxTipCapGwei: 10, - MaxFeeBidMultipleBips: arbmath.OneInBips * 10, + MaxFeeBidMultipleBips: arbmath.OneInUBips * 10, AllocateMempoolBalance: true, UrgencyGwei: 2., @@ -335,7 +335,7 @@ func TestFeeAndTipCaps_RBF_RisingBlobFee_FallingBaseFee(t *testing.T) { MinBlobTxTipCapGwei: 1, MaxTipCapGwei: 5, MaxBlobTxTipCapGwei: 10, - MaxFeeBidMultipleBips: arbmath.OneInBips * 10, + MaxFeeBidMultipleBips: arbmath.OneInUBips * 10, AllocateMempoolBalance: true, UrgencyGwei: 2., diff --git a/arbnode/dataposter/dbstorage/storage.go b/arbnode/dataposter/dbstorage/storage.go index 37ebfa5099..6a6cd3cfa4 100644 --- a/arbnode/dataposter/dbstorage/storage.go +++ b/arbnode/dataposter/dbstorage/storage.go @@ -95,11 +95,11 @@ func (s *Storage) PruneAll(ctx context.Context) error { if err != nil { return fmt.Errorf("pruning all keys: %w", err) } - until, err := strconv.Atoi(string(idx)) + until, err := strconv.ParseUint(string(idx), 10, 64) if err != nil { return fmt.Errorf("converting last item index bytes to integer: %w", err) } - return s.Prune(ctx, uint64(until+1)) + return s.Prune(ctx, until+1) } func (s *Storage) Prune(ctx context.Context, until uint64) error { diff --git a/arbnode/dataposter/storage_test.go b/arbnode/dataposter/storage_test.go index 8934d92b45..c6316caea7 100644 --- a/arbnode/dataposter/storage_test.go +++ b/arbnode/dataposter/storage_test.go @@ -72,24 +72,29 @@ func newRedisStorage(ctx context.Context, t *testing.T, encF storage.EncoderDeco func valueOf(t *testing.T, i int) *storage.QueuedTransaction { t.Helper() + // #nosec G115 meta, err := rlp.EncodeToBytes(storage.BatchPosterPosition{DelayedMessageCount: uint64(i)}) if err != nil { t.Fatalf("Encoding batch poster position, error: %v", err) } return &storage.QueuedTransaction{ FullTx: types.NewTransaction( + // #nosec G115 uint64(i), common.Address{}, big.NewInt(int64(i)), + // #nosec G115 uint64(i), big.NewInt(int64(i)), []byte{byte(i)}), Meta: meta, DeprecatedData: types.DynamicFeeTx{ - ChainID: big.NewInt(int64(i)), - Nonce: uint64(i), - GasTipCap: big.NewInt(int64(i)), - GasFeeCap: big.NewInt(int64(i)), + ChainID: big.NewInt(int64(i)), + // #nosec G115 + Nonce: uint64(i), + GasTipCap: big.NewInt(int64(i)), + GasFeeCap: big.NewInt(int64(i)), + // #nosec G115 Gas: uint64(i), Value: big.NewInt(int64(i)), Data: []byte{byte(i % 8)}, @@ -113,6 +118,7 @@ func values(t *testing.T, from, to int) []*storage.QueuedTransaction { func initStorage(ctx context.Context, t *testing.T, s QueueStorage) QueueStorage { t.Helper() for i := 0; i < 20; i++ { + // #nosec G115 if err := s.Put(ctx, uint64(i), nil, valueOf(t, i)); err != nil { t.Fatalf("Error putting a key/value: %v", err) } @@ -153,6 +159,7 @@ func TestPruneAll(t *testing.T) { s := newLevelDBStorage(t, func() storage.EncoderDecoderInterface { return &storage.EncoderDecoder{} }) ctx := context.Background() for i := 0; i < 20; i++ { + // #nosec G115 if err := s.Put(ctx, uint64(i), nil, valueOf(t, i)); err != nil { t.Fatalf("Error putting a key/value: %v", err) } @@ -236,6 +243,7 @@ func TestLast(t *testing.T) { ctx := context.Background() for i := 0; i < cnt; i++ { val := valueOf(t, i) + // #nosec G115 if err := s.Put(ctx, uint64(i), nil, val); err != nil { t.Fatalf("Error putting a key/value: %v", err) } @@ -255,6 +263,7 @@ func TestLast(t *testing.T) { for i := 0; i < cnt-1; i++ { prev := valueOf(t, i) newVal := valueOf(t, cnt+i) + // #nosec G115 if err := s.Put(ctx, uint64(i), prev, newVal); err != nil { t.Fatalf("Error putting a key/value: %v, prev: %v, new: %v", err, prev, newVal) } diff --git a/arbnode/delayed_sequencer.go b/arbnode/delayed_sequencer.go index 4f18531a76..b29a66dd05 100644 --- a/arbnode/delayed_sequencer.go +++ b/arbnode/delayed_sequencer.go @@ -121,6 +121,7 @@ func (d *DelayedSequencer) sequenceWithoutLockout(ctx context.Context, lastBlock if currentNum < config.FinalizeDistance { return nil } + // #nosec G115 finalized = uint64(currentNum - config.FinalizeDistance) } @@ -189,6 +190,7 @@ func (d *DelayedSequencer) sequenceWithoutLockout(ctx context.Context, lastBlock return fmt.Errorf("inbox reader at delayed message %v db accumulator %v doesn't match delayed bridge accumulator %v at L1 block %v", pos-1, lastDelayedAcc, delayedBridgeAcc, finalized) } for i, msg := range messages { + // #nosec G115 err = d.exec.SequenceDelayedMessage(msg, startPos+uint64(i)) if err != nil { return err diff --git a/arbnode/inbox_reader.go b/arbnode/inbox_reader.go index fd050b5f67..c596cfa9b0 100644 --- a/arbnode/inbox_reader.go +++ b/arbnode/inbox_reader.go @@ -542,6 +542,7 @@ func (r *InboxReader) run(ctx context.Context, hadError bool) error { } else { from = arbmath.BigAddByUint(to, 1) } + // #nosec G115 haveMessages := uint64(len(delayedMessages) + len(sequencerBatches)) if haveMessages <= (config.TargetMessagesRead / 2) { blocksToFetch += (blocksToFetch + 4) / 5 diff --git a/arbnode/transaction_streamer.go b/arbnode/transaction_streamer.go index a5bab8342f..24a0564b97 100644 --- a/arbnode/transaction_streamer.go +++ b/arbnode/transaction_streamer.go @@ -279,6 +279,7 @@ func (s *TransactionStreamer) reorg(batch ethdb.Batch, count arbutil.MessageInde return err } config := s.config() + // #nosec G115 maxResequenceMsgCount := count + arbutil.MessageIndex(config.MaxReorgResequenceDepth) if config.MaxReorgResequenceDepth >= 0 && maxResequenceMsgCount < targetMsgCount { log.Error( @@ -388,6 +389,7 @@ func (s *TransactionStreamer) reorg(batch ethdb.Batch, count arbutil.MessageInde } for i := 0; i < len(messagesResults); i++ { + // #nosec G115 pos := count + arbutil.MessageIndex(i) err = s.storeResult(pos, *messagesResults[i], batch) if err != nil { @@ -680,7 +682,7 @@ func (s *TransactionStreamer) AddMessagesAndEndBatch(pos arbutil.MessageIndex, m if err != nil { return err } - if dups == len(messages) { + if dups == uint64(len(messages)) { return endBatch(batch) } // cant keep reorg lock when catching insertionMutex. @@ -715,10 +717,10 @@ func (s *TransactionStreamer) countDuplicateMessages( pos arbutil.MessageIndex, messages []arbostypes.MessageWithMetadataAndBlockHash, batch *ethdb.Batch, -) (int, bool, *arbostypes.MessageWithMetadata, error) { - curMsg := 0 +) (uint64, bool, *arbostypes.MessageWithMetadata, error) { + var curMsg uint64 for { - if len(messages) == curMsg { + if uint64(len(messages)) == curMsg { break } key := dbKey(messagePrefix, uint64(pos)) @@ -818,7 +820,7 @@ func (s *TransactionStreamer) addMessagesAndEndBatchImpl(messageStartPos arbutil broadcastStartPos := arbutil.MessageIndex(s.broadcasterQueuedMessagesPos.Load()) if messagesAreConfirmed { - var duplicates int + var duplicates uint64 var err error duplicates, confirmedReorg, oldMsg, err = s.countDuplicateMessages(messageStartPos, messages, &batch) if err != nil { @@ -857,7 +859,7 @@ func (s *TransactionStreamer) addMessagesAndEndBatchImpl(messageStartPos arbutil var feedReorg bool if !hasNewConfirmedMessages { - var duplicates int + var duplicates uint64 var err error duplicates, feedReorg, oldMsg, err = s.countDuplicateMessages(messageStartPos, messages, nil) if err != nil { @@ -889,6 +891,7 @@ func (s *TransactionStreamer) addMessagesAndEndBatchImpl(messageStartPos arbutil // Validate delayed message counts of remaining messages for i, msg := range messages { + // #nosec G115 msgPos := messageStartPos + arbutil.MessageIndex(i) diff := msg.MessageWithMeta.DelayedMessagesRead - lastDelayedRead if diff != 0 && diff != 1 { @@ -924,6 +927,7 @@ func (s *TransactionStreamer) addMessagesAndEndBatchImpl(messageStartPos arbutil // Check if new messages were added at the end of cache, if they were, then dont remove those particular messages if len(s.broadcasterQueuedMessages) > cacheClearLen { s.broadcasterQueuedMessages = s.broadcasterQueuedMessages[cacheClearLen:] + // #nosec G115 s.broadcasterQueuedMessagesPos.Store(uint64(broadcastStartPos) + uint64(cacheClearLen)) } else { s.broadcasterQueuedMessages = s.broadcasterQueuedMessages[:0] @@ -1044,6 +1048,7 @@ func (s *TransactionStreamer) writeMessages(pos arbutil.MessageIndex, messages [ batch = s.db.NewBatch() } for i, msg := range messages { + // #nosec G115 err := s.writeMessage(pos+arbutil.MessageIndex(i), msg, batch) if err != nil { return err diff --git a/arbos/activate_test.go b/arbos/activate_test.go index 55440bb208..a89a38639a 100644 --- a/arbos/activate_test.go +++ b/arbos/activate_test.go @@ -20,6 +20,7 @@ func TestActivationDataFee(t *testing.T) { rand.Seed(time.Now().UTC().UnixNano()) state, _ := arbosState.NewArbosMemoryBackedArbOSState() pricer := state.Programs().DataPricer() + // #nosec G115 time := uint64(time.Now().Unix()) assert := func(cond bool) { diff --git a/arbos/addressSet/addressSet.go b/arbos/addressSet/addressSet.go index 1f09ff1440..156f36e7e7 100644 --- a/arbos/addressSet/addressSet.go +++ b/arbos/addressSet/addressSet.go @@ -79,6 +79,7 @@ func (as *AddressSet) AllMembers(maxNumToReturn uint64) ([]common.Address, error } ret := make([]common.Address, size) for i := range ret { + // #nosec G115 sba := as.backingStorage.OpenStorageBackedAddress(uint64(i + 1)) ret[i], err = sba.Get() if err != nil { diff --git a/arbos/addressTable/addressTable.go b/arbos/addressTable/addressTable.go index 566c71b689..6ae271060d 100644 --- a/arbos/addressTable/addressTable.go +++ b/arbos/addressTable/addressTable.go @@ -103,6 +103,7 @@ func (atab *AddressTable) Decompress(buf []byte) (common.Address, uint64, error) return common.Address{}, 0, err } if len(input) == 20 { + // #nosec G115 numBytesRead := uint64(rd.Size() - int64(rd.Len())) return common.BytesToAddress(input), numBytesRead, nil } else { diff --git a/arbos/arbosState/initialization_test.go b/arbos/arbosState/initialization_test.go index b0fe1d0dac..5e605b8bd2 100644 --- a/arbos/arbosState/initialization_test.go +++ b/arbos/arbosState/initialization_test.go @@ -126,6 +126,7 @@ func checkAddressTable(arbState *ArbosState, addrTable []common.Address, t *test Fail(t) } for i, addr := range addrTable { + // #nosec G115 res, exists, err := atab.LookupIndex(uint64(i)) Require(t, err) if !exists { diff --git a/arbos/arbosState/initialize.go b/arbos/arbosState/initialize.go index 56fa579c15..427bdc3087 100644 --- a/arbos/arbosState/initialize.go +++ b/arbos/arbosState/initialize.go @@ -108,7 +108,7 @@ func InitializeArbosInDatabase(db ethdb.Database, cacheConfig *core.CacheConfig, if err != nil { return common.Hash{}, err } - for i := 0; addressReader.More(); i++ { + for i := uint64(0); addressReader.More(); i++ { addr, err := addressReader.GetNext() if err != nil { return common.Hash{}, err @@ -117,7 +117,7 @@ func InitializeArbosInDatabase(db ethdb.Database, cacheConfig *core.CacheConfig, if err != nil { return common.Hash{}, err } - if uint64(i) != slot { + if i != slot { return common.Hash{}, errors.New("address table slot mismatch") } } diff --git a/arbos/l1pricing_test.go b/arbos/l1pricing_test.go index 1cda4b3d82..6f9e3ecb35 100644 --- a/arbos/l1pricing_test.go +++ b/arbos/l1pricing_test.go @@ -279,7 +279,9 @@ func _testL1PriceEquilibration(t *testing.T, initialL1BasefeeEstimate *big.Int, evm.StateDB, evm, 3, + // #nosec G115 uint64(10*(i+1)), + // #nosec G115 uint64(10*(i+1)+5), bpAddr, arbmath.BigMulByUint(equilibriumL1BasefeeEstimate, unitsToAdd), diff --git a/arbos/merkleAccumulator/merkleAccumulator.go b/arbos/merkleAccumulator/merkleAccumulator.go index 2e060c5840..e62303e5fd 100644 --- a/arbos/merkleAccumulator/merkleAccumulator.go +++ b/arbos/merkleAccumulator/merkleAccumulator.go @@ -97,6 +97,7 @@ func (acc *MerkleAccumulator) GetPartials() ([]*common.Hash, error) { } partials := make([]*common.Hash, CalcNumPartials(size)) for i := range partials { + // #nosec G115 p, err := acc.getPartial(uint64(i)) if err != nil { return nil, err diff --git a/arbos/programs/data_pricer.go b/arbos/programs/data_pricer.go index ed7c98556d..d82aa81f04 100644 --- a/arbos/programs/data_pricer.go +++ b/arbos/programs/data_pricer.go @@ -83,8 +83,8 @@ func (p *DataPricer) UpdateModel(tempBytes uint32, time uint64) (*big.Int, error } exponent := arbmath.OneInBips * arbmath.Bips(demand) / arbmath.Bips(inertia) - multiplier := arbmath.ApproxExpBasisPoints(exponent, 12).Uint64() - costPerByte := arbmath.SaturatingUMul(uint64(minPrice), multiplier) / 10000 + multiplier := arbmath.ApproxExpBasisPoints(exponent, 12) + costPerByte := arbmath.UintSaturatingMulByBips(uint64(minPrice), multiplier) costInWei := arbmath.SaturatingUMul(costPerByte, uint64(tempBytes)) return arbmath.UintToBig(costInWei), nil } diff --git a/arbos/retryable_test.go b/arbos/retryable_test.go index ddb88348dd..2eccaea6c2 100644 --- a/arbos/retryable_test.go +++ b/arbos/retryable_test.go @@ -38,6 +38,7 @@ func TestRetryableLifecycle(t *testing.T) { retryableState := state.RetryableState() lifetime := uint64(retryables.RetryableLifetimeSeconds) + // #nosec G115 timestampAtCreation := uint64(rand.Int63n(1 << 16)) timeoutAtCreation := timestampAtCreation + lifetime currentTime := timeoutAtCreation @@ -57,6 +58,7 @@ func TestRetryableLifecycle(t *testing.T) { checkQueueSize := func(expected int, message string) { timeoutQueueSize, err := retryableState.TimeoutQueue.Size() Require(t, err) + // #nosec G115 if timeoutQueueSize != uint64(expected) { Fail(t, currentTime, message, timeoutQueueSize) } @@ -167,6 +169,7 @@ func TestRetryableCleanup(t *testing.T) { callvalue := big.NewInt(0) calldata := testhelpers.RandomizeSlice(make([]byte, rand.Intn(1<<12))) + // #nosec G115 timeout := uint64(rand.Int63n(1 << 16)) timestamp := 2 * timeout diff --git a/arbos/retryables/retryable.go b/arbos/retryables/retryable.go index e1cfe48bcf..5938244782 100644 --- a/arbos/retryables/retryable.go +++ b/arbos/retryables/retryable.go @@ -367,5 +367,7 @@ func RetryableEscrowAddress(ticketId common.Hash) common.Address { } func RetryableSubmissionFee(calldataLengthInBytes int, l1BaseFee *big.Int) *big.Int { - return arbmath.BigMulByUint(l1BaseFee, uint64(1400+6*calldataLengthInBytes)) + // This can't overflow because calldataLengthInBytes would need to be 3 exabytes + // #nosec G115 + return arbmath.BigMulByUint(l1BaseFee, 1400+6*uint64(calldataLengthInBytes)) } diff --git a/arbos/storage/storage.go b/arbos/storage/storage.go index 352726778d..bc16491af0 100644 --- a/arbos/storage/storage.go +++ b/arbos/storage/storage.go @@ -322,11 +322,11 @@ func (s *Storage) Burner() burn.Burner { } func (s *Storage) Keccak(data ...[]byte) ([]byte, error) { - byteCount := 0 + var byteCount uint64 for _, part := range data { - byteCount += len(part) + byteCount += uint64(len(part)) } - cost := 30 + 6*arbmath.WordsForBytes(uint64(byteCount)) + cost := 30 + 6*arbmath.WordsForBytes(byteCount) if err := s.burner.Burn(cost); err != nil { return nil, err } @@ -420,6 +420,7 @@ func (sbu *StorageBackedInt64) Get() (int64, error) { } func (sbu *StorageBackedInt64) Set(value int64) error { + // #nosec G115 return sbu.StorageSlot.Set(util.UintToHash(uint64(value))) // see implementation note above } @@ -456,7 +457,7 @@ func (sbu *StorageBackedUBips) Get() (arbmath.UBips, error) { } func (sbu *StorageBackedUBips) Set(bips arbmath.UBips) error { - return sbu.backing.Set(bips.Uint64()) + return sbu.backing.Set(uint64(bips)) } type StorageBackedUint16 struct { diff --git a/arbutil/block_message_relation.go b/arbutil/block_message_relation.go index e164cf2619..dcf4c86084 100644 --- a/arbutil/block_message_relation.go +++ b/arbutil/block_message_relation.go @@ -11,6 +11,7 @@ func BlockNumberToMessageCount(blockNumber uint64, genesisBlockNumber uint64) Me // Block number must correspond to a message count, meaning it may not be less than -1 func SignedBlockNumberToMessageCount(blockNumber int64, genesisBlockNumber uint64) MessageIndex { + // #nosec G115 return MessageIndex(uint64(blockNumber+1) - genesisBlockNumber) } diff --git a/blocks_reexecutor/blocks_reexecutor.go b/blocks_reexecutor/blocks_reexecutor.go index f7cc0d8c72..b43999a7db 100644 --- a/blocks_reexecutor/blocks_reexecutor.go +++ b/blocks_reexecutor/blocks_reexecutor.go @@ -113,6 +113,7 @@ func New(c *Config, blockchain *core.BlockChain, fatalErrChan chan error) *Block } // Divide work equally among available threads when BlocksPerThread is zero if c.BlocksPerThread == 0 { + // #nosec G115 work := (end - start) / uint64(c.Room) if work > 0 { blocksPerThread = work diff --git a/broadcastclient/broadcastclient_test.go b/broadcastclient/broadcastclient_test.go index 44b48192ab..a499628cd5 100644 --- a/broadcastclient/broadcastclient_test.go +++ b/broadcastclient/broadcastclient_test.go @@ -105,6 +105,7 @@ func testReceiveMessages(t *testing.T, clientCompression bool, serverCompression go func() { for i := 0; i < messageCount; i++ { + // #nosec G115 Require(t, b.BroadcastSingle(arbostypes.TestMessageWithMetadataAndRequestId, arbutil.MessageIndex(i), nil)) } }() @@ -156,6 +157,7 @@ func TestInvalidSignature(t *testing.T) { go func() { for i := 0; i < messageCount; i++ { + // #nosec G115 Require(t, b.BroadcastSingle(arbostypes.TestMessageWithMetadataAndRequestId, arbutil.MessageIndex(i), nil)) } }() diff --git a/broadcaster/broadcaster.go b/broadcaster/broadcaster.go index 397698635a..4fe8657bfa 100644 --- a/broadcaster/broadcaster.go +++ b/broadcaster/broadcaster.go @@ -104,6 +104,7 @@ func (b *Broadcaster) BroadcastMessages( }() var feedMessages []*m.BroadcastFeedMessage for i, msg := range messagesWithBlockHash { + // #nosec G115 bfm, err := b.NewBroadcastFeedMessage(msg.MessageWithMeta, seq+arbutil.MessageIndex(i), msg.BlockHash) if err != nil { return err diff --git a/broadcaster/message/message.go b/broadcaster/message/message.go index aca9598754..1e26e6da5e 100644 --- a/broadcaster/message/message.go +++ b/broadcaster/message/message.go @@ -41,6 +41,7 @@ type BroadcastFeedMessage struct { } func (m *BroadcastFeedMessage) Size() uint64 { + // #nosec G115 return uint64(len(m.Signature) + len(m.Message.Message.L2msg) + 160) } diff --git a/cmd/datool/datool.go b/cmd/datool/datool.go index ba60cbbd4d..f791d8cbc4 100644 --- a/cmd/datool/datool.go +++ b/cmd/datool/datool.go @@ -166,8 +166,10 @@ func startClientStore(args []string) error { if err != nil { return err } + // #nosec G115 cert, err = client.Store(ctx, message, uint64(time.Now().Add(config.DASRetentionPeriod).Unix())) } else if len(config.Message) > 0 { + // #nosec G115 cert, err = client.Store(ctx, []byte(config.Message), uint64(time.Now().Add(config.DASRetentionPeriod).Unix())) } else { return errors.New("--message or --random-message-size must be specified") @@ -363,6 +365,7 @@ func dumpKeyset(args []string) error { return err } + // #nosec G115 keysetHash, keysetBytes, err := das.KeysetHashFromServices(services, uint64(config.Keyset.AssumedHonest)) if err != nil { return err diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index a8463a7d21..fc59f2d231 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -876,6 +876,7 @@ func testUpdateTxIndex(chainDb ethdb.Database, chainConfig *params.ChainConfig, localWg.Add(1) go func() { batch := chainDb.NewBatch() + // #nosec G115 for blockNum := uint64(thread); blockNum <= lastBlock; blockNum += uint64(threads) { blockHash := rawdb.ReadCanonicalHash(chainDb, blockNum) block := rawdb.ReadBlock(chainDb, blockHash, blockNum) diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index 146a0049e7..e66d99b56e 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -892,6 +892,7 @@ func ParseNode(ctx context.Context, args []string) (*NodeConfig, *genericconf.Wa l2ChainInfoIpfsDownloadPath := k.String("chain.info-ipfs-download-path") l2ChainInfoFiles := k.Strings("chain.info-files") l2ChainInfoJson := k.String("chain.info-json") + // #nosec G115 err = applyChainParameters(ctx, k, uint64(l2ChainId), l2ChainName, l2ChainInfoFiles, l2ChainInfoJson, l2ChainInfoIpfsUrl, l2ChainInfoIpfsDownloadPath) if err != nil { return nil, nil, err @@ -1037,13 +1038,16 @@ func applyChainParameters(ctx context.Context, k *koanf.Koanf, chainId uint64, c func initReorg(initConfig conf.InitConfig, chainConfig *params.ChainConfig, inboxTracker *arbnode.InboxTracker) error { var batchCount uint64 if initConfig.ReorgToBatch >= 0 { + // #nosec G115 batchCount = uint64(initConfig.ReorgToBatch) + 1 } else { var messageIndex arbutil.MessageIndex if initConfig.ReorgToMessageBatch >= 0 { + // #nosec G115 messageIndex = arbutil.MessageIndex(initConfig.ReorgToMessageBatch) } else if initConfig.ReorgToBlockBatch > 0 { genesis := chainConfig.ArbitrumChainParams.GenesisBlockNum + // #nosec G115 blockNum := uint64(initConfig.ReorgToBlockBatch) if blockNum < genesis { return fmt.Errorf("ReorgToBlockBatch %d before genesis %d", blockNum, genesis) diff --git a/cmd/pruning/pruning.go b/cmd/pruning/pruning.go index 096bb4b1ae..6fc7741478 100644 --- a/cmd/pruning/pruning.go +++ b/cmd/pruning/pruning.go @@ -212,6 +212,7 @@ func findImportantRoots(ctx context.Context, chainDb ethdb.Database, stack *node } if meta.ParentChainBlock <= l1BlockNum { signedBlockNum := arbutil.MessageCountToBlockNumber(meta.MessageCount, genesisNum) + // #nosec G115 blockNum := uint64(signedBlockNum) l2Hash := rawdb.ReadCanonicalHash(chainDb, blockNum) l2Header := rawdb.ReadHeader(chainDb, l2Hash, blockNum) diff --git a/das/aggregator.go b/das/aggregator.go index e8972447ad..e7460fa371 100644 --- a/das/aggregator.go +++ b/das/aggregator.go @@ -130,6 +130,7 @@ func NewAggregatorWithSeqInboxCaller( seqInboxCaller *bridgegen.SequencerInboxCaller, ) (*Aggregator, error) { + // #nosec G115 keysetHash, keysetBytes, err := KeysetHashFromServices(services, uint64(config.RPCAggregator.AssumedHonest)) if err != nil { return nil, err diff --git a/das/dasRpcClient.go b/das/dasRpcClient.go index 635696bdab..7d48ed796d 100644 --- a/das/dasRpcClient.go +++ b/das/dasRpcClient.go @@ -58,6 +58,7 @@ func NewDASRPCClient(target string, signer signature.DataSignerFunc, maxStoreChu } func (c *DASRPCClient) Store(ctx context.Context, message []byte, timeout uint64) (*daprovider.DataAvailabilityCertificate, error) { + // #nosec G115 timestamp := uint64(time.Now().Unix()) nChunks := uint64(len(message)) / c.chunkSize lastChunkSize := uint64(len(message)) % c.chunkSize diff --git a/das/dasRpcServer.go b/das/dasRpcServer.go index d14766cc7e..bb1be0384e 100644 --- a/das/dasRpcServer.go +++ b/das/dasRpcServer.go @@ -153,7 +153,7 @@ type SendChunkResult struct { type batch struct { chunks [][]byte expectedChunks uint64 - seenChunks atomic.Int64 + seenChunks atomic.Uint64 expectedChunkSize, expectedSize uint64 timeout uint64 startTime time.Time @@ -248,7 +248,7 @@ func (b *batchBuilder) close(id uint64) ([]byte, uint64, time.Time, error) { return nil, 0, time.Time{}, fmt.Errorf("unknown batch(%d)", id) } - if batch.expectedChunks != uint64(batch.seenChunks.Load()) { + if batch.expectedChunks != batch.seenChunks.Load() { return nil, 0, time.Time{}, fmt.Errorf("incomplete batch(%d): got %d/%d chunks", id, batch.seenChunks.Load(), batch.expectedChunks) } diff --git a/das/das_test.go b/das/das_test.go index 179734c8b1..4971d454e5 100644 --- a/das/das_test.go +++ b/das/das_test.go @@ -55,6 +55,7 @@ func testDASStoreRetrieveMultipleInstances(t *testing.T, storageType string) { Require(t, err, "no das") var daReader DataAvailabilityServiceReader = storageService + // #nosec G115 timeout := uint64(time.Now().Add(time.Hour * 24).Unix()) messageSaved := []byte("hello world") cert, err := daWriter.Store(firstCtx, messageSaved, timeout) @@ -146,6 +147,7 @@ func testDASMissingMessage(t *testing.T, storageType string) { var daReader DataAvailabilityServiceReader = storageService messageSaved := []byte("hello world") + // #nosec G115 timeout := uint64(time.Now().Add(time.Hour * 24).Unix()) cert, err := daWriter.Store(ctx, messageSaved, timeout) Require(t, err, "Error storing message") diff --git a/das/db_storage_service.go b/das/db_storage_service.go index 1d9e5348d4..74bf12b927 100644 --- a/das/db_storage_service.go +++ b/das/db_storage_service.go @@ -267,6 +267,7 @@ func (dbs *DBStorageService) String() string { func (dbs *DBStorageService) HealthCheck(ctx context.Context) error { testData := []byte("Test-Data") + // #nosec G115 err := dbs.Put(ctx, testData, uint64(time.Now().Add(time.Minute).Unix())) if err != nil { return err diff --git a/das/factory.go b/das/factory.go index 5742a39479..7f696912b3 100644 --- a/das/factory.go +++ b/das/factory.go @@ -7,7 +7,6 @@ import ( "context" "errors" "fmt" - "math" "github.com/ethereum/go-ethereum/common" @@ -187,12 +186,7 @@ func CreateDAComponentsForDaserver( dasLifecycleManager.Register(restAgg) syncConf := &config.RestAggregator.SyncToStorage - var retentionPeriodSeconds uint64 - if uint64(syncConf.RetentionPeriod) == math.MaxUint64 { - retentionPeriodSeconds = math.MaxUint64 - } else { - retentionPeriodSeconds = uint64(syncConf.RetentionPeriod.Seconds()) - } + retentionPeriodSeconds := uint64(syncConf.RetentionPeriod.Seconds()) if syncConf.Eager { if l1Reader == nil || seqInboxAddress == nil { diff --git a/das/fallback_storage_service.go b/das/fallback_storage_service.go index 49f961da60..0a451678d0 100644 --- a/das/fallback_storage_service.go +++ b/das/fallback_storage_service.go @@ -85,6 +85,7 @@ func (f *FallbackStorageService) GetByHash(ctx context.Context, key common.Hash) } if dastree.ValidHash(key, data) { putErr := f.StorageService.Put( + // #nosec G115 ctx, data, arbmath.SaturatingUAdd(uint64(time.Now().Unix()), f.backupRetentionSeconds), ) if putErr != nil && !f.ignoreRetentionWriteErrors { diff --git a/das/local_file_storage_service.go b/das/local_file_storage_service.go index ce86786718..5e64c34b10 100644 --- a/das/local_file_storage_service.go +++ b/das/local_file_storage_service.go @@ -377,6 +377,7 @@ func migrate(fl *flatLayout, tl *trieLayout) error { return err } + // #nosec G115 expiryPath := tl.expiryPath(batch.key, uint64(batch.expiry.Unix())) if err = createHardLink(newPath, expiryPath); err != nil { return err diff --git a/das/local_file_storage_service_test.go b/das/local_file_storage_service_test.go index 01b999f356..8a36664670 100644 --- a/das/local_file_storage_service_test.go +++ b/das/local_file_storage_service_test.go @@ -78,6 +78,7 @@ func TestMigrationNoExpiry(t *testing.T) { Require(t, err) s.enableLegacyLayout = true + // #nosec G115 now := uint64(time.Now().Unix()) err = s.Put(ctx, []byte("a"), now+1) @@ -121,14 +122,19 @@ func TestMigrationExpiry(t *testing.T) { now := time.Now() // Use increments of expiry divisor in order to span multiple by-expiry-timestamp dirs + // #nosec G115 err = s.Put(ctx, []byte("a"), uint64(now.Add(-2*time.Second*expiryDivisor).Unix())) Require(t, err) + // #nosec G115 err = s.Put(ctx, []byte("b"), uint64(now.Add(-1*time.Second*expiryDivisor).Unix())) Require(t, err) + // #nosec G115 err = s.Put(ctx, []byte("c"), uint64(now.Add(time.Second*expiryDivisor).Unix())) Require(t, err) + // #nosec G115 err = s.Put(ctx, []byte("d"), uint64(now.Add(time.Second*expiryDivisor).Unix())) Require(t, err) + // #nosec G115 err = s.Put(ctx, []byte("e"), uint64(now.Add(2*time.Second*expiryDivisor).Unix())) Require(t, err) @@ -171,19 +177,26 @@ func TestExpiryDuplicates(t *testing.T) { now := time.Now() // Use increments of expiry divisor in order to span multiple by-expiry-timestamp dirs + // #nosec G115 err = s.Put(ctx, []byte("a"), uint64(now.Add(-2*time.Second*expiryDivisor).Unix())) Require(t, err) + // #nosec G115 err = s.Put(ctx, []byte("a"), uint64(now.Add(-1*time.Second*expiryDivisor).Unix())) Require(t, err) + // #nosec G115 err = s.Put(ctx, []byte("a"), uint64(now.Add(time.Second*expiryDivisor).Unix())) Require(t, err) + // #nosec G115 err = s.Put(ctx, []byte("d"), uint64(now.Add(time.Second*expiryDivisor).Unix())) Require(t, err) + // #nosec G115 err = s.Put(ctx, []byte("e"), uint64(now.Add(2*time.Second*expiryDivisor).Unix())) Require(t, err) + // #nosec G115 err = s.Put(ctx, []byte("f"), uint64(now.Add(3*time.Second*expiryDivisor).Unix())) Require(t, err) // Put the same entry and expiry again, should have no effect + // #nosec G115 err = s.Put(ctx, []byte("f"), uint64(now.Add(3*time.Second*expiryDivisor).Unix())) Require(t, err) diff --git a/das/redis_storage_service_test.go b/das/redis_storage_service_test.go index 55f3ecd82c..77d3e8cd0f 100644 --- a/das/redis_storage_service_test.go +++ b/das/redis_storage_service_test.go @@ -16,6 +16,7 @@ import ( func TestRedisStorageService(t *testing.T) { ctx := context.Background() + // #nosec G115 timeout := uint64(time.Now().Add(time.Hour).Unix()) baseStorageService := NewMemoryBackedStorageService(ctx) server, err := miniredis.Run() diff --git a/das/redundant_storage_test.go b/das/redundant_storage_test.go index b56f62ee24..11d3b58264 100644 --- a/das/redundant_storage_test.go +++ b/das/redundant_storage_test.go @@ -17,6 +17,7 @@ const NumServices = 3 func TestRedundantStorageService(t *testing.T) { ctx := context.Background() + // #nosec G115 timeout := uint64(time.Now().Add(time.Hour).Unix()) services := []StorageService{} for i := 0; i < NumServices; i++ { diff --git a/das/restful_server_test.go b/das/restful_server_test.go index 1d3675749a..e6982f9db5 100644 --- a/das/restful_server_test.go +++ b/das/restful_server_test.go @@ -48,6 +48,7 @@ func TestRestfulClientServer(t *testing.T) { server, port, err := NewRestfulDasServerOnRandomPort(LocalServerAddressForTest, storage) Require(t, err) + // #nosec G115 err = storage.Put(ctx, data, uint64(time.Now().Add(time.Hour).Unix())) Require(t, err) diff --git a/das/rpc_aggregator.go b/das/rpc_aggregator.go index 24a470be5b..1b3e2b8f44 100644 --- a/das/rpc_aggregator.go +++ b/das/rpc_aggregator.go @@ -119,7 +119,7 @@ func ParseServices(config AggregatorConfig, signer signature.DataSignerFunc) ([] return nil, err } - d, err := NewServiceDetails(service, *pubKey, 1< uint64(countsNum) && sectionNum > 1 { break } diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 457dae0910..7a78cee309 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -671,7 +671,7 @@ func l2MessageBatchDataFromTxes(txes types.Transactions) ([]byte, error) { if err != nil { return nil, err } - binary.BigEndian.PutUint64(sizeBuf, uint64(len(txBytes)+1)) + binary.BigEndian.PutUint64(sizeBuf, uint64(len(txBytes))+1) l2Message = append(l2Message, sizeBuf...) l2Message = append(l2Message, arbos.L2MessageKind_SignedTx) l2Message = append(l2Message, txBytes...) diff --git a/system_tests/estimation_test.go b/system_tests/estimation_test.go index 284c709fad..6285702342 100644 --- a/system_tests/estimation_test.go +++ b/system_tests/estimation_test.go @@ -214,7 +214,7 @@ func TestComponentEstimate(t *testing.T) { userBalance := big.NewInt(1e16) maxPriorityFeePerGas := big.NewInt(0) - maxFeePerGas := arbmath.BigMulByUfrac(l2BaseFee, 3, 2) + maxFeePerGas := arbmath.BigMulByUFrac(l2BaseFee, 3, 2) builder.L2Info.GenerateAccount("User") builder.L2.TransferBalance(t, "Owner", "User", userBalance, builder.L2Info) diff --git a/system_tests/nodeinterface_test.go b/system_tests/nodeinterface_test.go index 17bfb18892..927dc1b630 100644 --- a/system_tests/nodeinterface_test.go +++ b/system_tests/nodeinterface_test.go @@ -163,6 +163,7 @@ func TestGetL1Confirmations(t *testing.T) { numTransactions := 200 + // #nosec G115 if l1Confs >= uint64(numTransactions) { t.Fatalf("L1Confirmations for latest block %v is already %v (over %v)", genesisBlock.Number(), l1Confs, numTransactions) } @@ -175,6 +176,7 @@ func TestGetL1Confirmations(t *testing.T) { Require(t, err) // Allow a gap of 10 for asynchronicity, just in case + // #nosec G115 if l1Confs+10 < uint64(numTransactions) { t.Fatalf("L1Confirmations for latest block %v is only %v (did not hit expected %v)", genesisBlock.Number(), l1Confs, numTransactions) } diff --git a/system_tests/outbox_test.go b/system_tests/outbox_test.go index c68df6ea22..25c52396f9 100644 --- a/system_tests/outbox_test.go +++ b/system_tests/outbox_test.go @@ -175,6 +175,7 @@ func TestOutboxProofs(t *testing.T) { sibling := place ^ which position := merkletree.LevelAndLeaf{ + // #nosec G115 Level: uint64(level), Leaf: sibling, } @@ -201,6 +202,7 @@ func TestOutboxProofs(t *testing.T) { leaf := total - 1 // preceding it. We subtract 1 since we count from 0 partial := merkletree.LevelAndLeaf{ + // #nosec G115 Level: uint64(level), Leaf: leaf, } @@ -289,6 +291,7 @@ func TestOutboxProofs(t *testing.T) { step.Leaf += 1 << step.Level // we start on the min partial's zero-hash sibling known[step] = zero + // #nosec G115 for step.Level < uint64(treeLevels) { curr, ok := known[step] diff --git a/system_tests/recreatestate_rpc_test.go b/system_tests/recreatestate_rpc_test.go index 7b09552e28..4833d35536 100644 --- a/system_tests/recreatestate_rpc_test.go +++ b/system_tests/recreatestate_rpc_test.go @@ -362,12 +362,14 @@ func testSkippingSavingStateAndRecreatingAfterRestart(t *testing.T, cacheConfig Require(t, err) l2info.GenerateAccount("User2") + // #nosec G115 for i := genesis; i < uint64(txCount)+genesis; i++ { tx := l2info.PrepareTx("Owner", "User2", l2info.TransferGas, common.Big1, nil) err := client.SendTransaction(ctx, tx) Require(t, err) receipt, err := EnsureTxSucceeded(ctx, client, tx) Require(t, err) + // #nosec G115 if have, want := receipt.BlockNumber.Uint64(), uint64(i)+1; have != want { Fatal(t, "internal test error - tx got included in unexpected block number, have:", have, "want:", want) } @@ -378,6 +380,7 @@ func testSkippingSavingStateAndRecreatingAfterRestart(t *testing.T, cacheConfig Fatal(t, "missing current block") } lastBlock := currentHeader.Number.Uint64() + // #nosec G115 if want := genesis + uint64(txCount); lastBlock < want { Fatal(t, "internal test error - not enough blocks produced during preparation, want:", want, "have:", lastBlock) } @@ -391,6 +394,7 @@ func testSkippingSavingStateAndRecreatingAfterRestart(t *testing.T, cacheConfig bc = builder.L2.ExecNode.Backend.ArbInterface().BlockChain() gas := skipGas blocks := skipBlocks + // #nosec G115 for i := genesis; i <= genesis+uint64(txCount); i++ { block := bc.GetBlockByNumber(i) if block == nil { @@ -423,6 +427,7 @@ func testSkippingSavingStateAndRecreatingAfterRestart(t *testing.T, cacheConfig } } } + // #nosec G115 for i := genesis + 1; i <= genesis+uint64(txCount); i += i % 10 { _, err = client.BalanceAt(ctx, GetTestAddressForAccountName(t, "User2"), new(big.Int).SetUint64(i)) if err != nil { diff --git a/system_tests/seqfeed_test.go b/system_tests/seqfeed_test.go index 5e70fdf098..21f0755225 100644 --- a/system_tests/seqfeed_test.go +++ b/system_tests/seqfeed_test.go @@ -164,12 +164,12 @@ func compareAllMsgResultsFromConsensusAndExecution( } var lastResult *execution.MessageResult - for msgCount := 1; arbutil.MessageIndex(msgCount) <= consensusMsgCount; msgCount++ { + for msgCount := arbutil.MessageIndex(1); msgCount <= consensusMsgCount; msgCount++ { pos := msgCount - 1 resultExec, err := testClient.ExecNode.ResultAtPos(arbutil.MessageIndex(pos)) Require(t, err) - resultConsensus, err := testClient.ConsensusNode.TxStreamer.ResultAtCount(arbutil.MessageIndex(msgCount)) + resultConsensus, err := testClient.ConsensusNode.TxStreamer.ResultAtCount(msgCount) Require(t, err) if !reflect.DeepEqual(resultExec, resultConsensus) { diff --git a/system_tests/seqinbox_test.go b/system_tests/seqinbox_test.go index 6babe5833f..a9f66b0e2f 100644 --- a/system_tests/seqinbox_test.go +++ b/system_tests/seqinbox_test.go @@ -265,6 +265,7 @@ func testSequencerInboxReaderImpl(t *testing.T, validator bool) { for j := 0; j < numMessages; j++ { sourceNum := rand.Int() % len(state.accounts) source := state.accounts[sourceNum] + // #nosec G115 amount := new(big.Int).SetUint64(uint64(rand.Int()) % state.balances[source].Uint64()) reserveAmount := new(big.Int).SetUint64(l2pricing.InitialBaseFeeWei * 100000000) if state.balances[source].Cmp(new(big.Int).Add(amount, reserveAmount)) < 0 { @@ -314,6 +315,7 @@ func testSequencerInboxReaderImpl(t *testing.T, validator bool) { for j := 0; ; j++ { haveNonce, err := builder.L1.Client.PendingNonceAt(ctx, seqOpts.From) Require(t, err) + // #nosec G115 if haveNonce == uint64(seqNonce) { break } @@ -380,6 +382,7 @@ func testSequencerInboxReaderImpl(t *testing.T, validator bool) { t.Errorf("Transaction: %v was not refunded, balance diff: %v, cost: %v", tx.Hash(), diff, txCost) } + // #nosec G115 state.l2BlockNumber += uint64(numMessages) state.l1BlockNumber = txRes.BlockNumber.Uint64() blockStates = append(blockStates, state) diff --git a/system_tests/stylus_trace_test.go b/system_tests/stylus_trace_test.go index cb303874d6..5c4463d9f7 100644 --- a/system_tests/stylus_trace_test.go +++ b/system_tests/stylus_trace_test.go @@ -76,6 +76,7 @@ func sendAndTraceTransaction( } func intToBytes(v int) []byte { + // #nosec G115 return binary.BigEndian.AppendUint64(nil, uint64(v)) } diff --git a/util/arbmath/bips.go b/util/arbmath/bips.go index 646dad3a92..39b014f3ac 100644 --- a/util/arbmath/bips.go +++ b/util/arbmath/bips.go @@ -27,24 +27,35 @@ func BigMulByBips(value *big.Int, bips Bips) *big.Int { return BigMulByFrac(value, int64(bips), int64(OneInBips)) } +func BigMulByUBips(value *big.Int, bips UBips) *big.Int { + return BigMulByUFrac(value, uint64(bips), uint64(OneInUBips)) +} + func IntMulByBips(value int64, bips Bips) int64 { return value * int64(bips) / int64(OneInBips) } +// UintMulByBips multiplies a uint value by a bips value +// bips must be positive and not cause an overflow func UintMulByBips(value uint64, bips Bips) uint64 { + // #nosec G115 return value * uint64(bips) / uint64(OneInBips) } -func SaturatingCastToBips(value uint64) Bips { - return Bips(SaturatingCast[int64](value)) -} - -func (bips UBips) Uint64() uint64 { - return uint64(bips) +// UintSaturatingMulByBips multiplies a uint value by a bips value, +// saturating at the maximum bips value (not the maximum uint64 result), +// then rounding down and returning a uint64. +// Returns 0 if bips is less than or equal to zero +func UintSaturatingMulByBips(value uint64, bips Bips) uint64 { + if bips <= 0 { + return 0 + } + // #nosec G115 + return SaturatingUMul(value, uint64(bips)) / uint64(OneInBips) } -func (bips Bips) Uint64() uint64 { - return uint64(bips) +func SaturatingCastToBips(value uint64) Bips { + return Bips(SaturatingCast[int64](value)) } // BigDivToBips returns dividend/divisor as bips, saturating if out of bounds diff --git a/util/arbmath/math.go b/util/arbmath/math.go index e5bed67f6d..07a9941b65 100644 --- a/util/arbmath/math.go +++ b/util/arbmath/math.go @@ -29,6 +29,7 @@ func NextOrCurrentPowerOf2(value uint64) uint64 { // Log2ceil the log2 of the int, rounded up func Log2ceil(value uint64) uint64 { + // #nosec G115 return uint64(64 - bits.LeadingZeros64(value)) } @@ -228,8 +229,8 @@ func BigMulByFrac(value *big.Int, numerator, denominator int64) *big.Int { return value } -// BigMulByUfrac multiply a huge by a rational whose components are non-negative -func BigMulByUfrac(value *big.Int, numerator, denominator uint64) *big.Int { +// BigMulByUFrac multiply a huge by a rational whose components are non-negative +func BigMulByUFrac(value *big.Int, numerator, denominator uint64) *big.Int { value = new(big.Int).Set(value) value.Mul(value, new(big.Int).SetUint64(numerator)) value.Div(value, new(big.Int).SetUint64(denominator)) @@ -407,6 +408,8 @@ func ApproxExpBasisPoints(value Bips, accuracy uint64) Bips { if negative { input = -value } + // This cast is safe because input is always positive + // #nosec G115 x := uint64(input) bips := uint64(OneInBips) diff --git a/util/arbmath/math_test.go b/util/arbmath/math_test.go index 528666dc19..3660f3657e 100644 --- a/util/arbmath/math_test.go +++ b/util/arbmath/math_test.go @@ -44,6 +44,7 @@ func TestMath(t *testing.T) { // try the first million sqrts for i := 0; i < 1000000; i++ { + // #nosec G115 input := uint64(i) approx := ApproxSquareRoot(input) correct := math.Sqrt(float64(input)) diff --git a/util/headerreader/blob_client.go b/util/headerreader/blob_client.go index 2b47a940c3..160323cf60 100644 --- a/util/headerreader/blob_client.go +++ b/util/headerreader/blob_client.go @@ -191,6 +191,7 @@ func (b *BlobClient) blobSidecars(ctx context.Context, slot uint64, versionedHas rawData, err := beaconRequest[json.RawMessage](b, ctx, fmt.Sprintf("/eth/v1/beacon/blob_sidecars/%d", slot)) if err != nil || len(rawData) == 0 { // blobs are pruned after 4096 epochs (1 epoch = 32 slots), we determine if the requested slot were to be pruned by a non-archive endpoint + // #nosec G115 roughAgeOfSlot := uint64(time.Now().Unix()) - (b.genesisTime + slot*b.secondsPerSlot) if roughAgeOfSlot > b.secondsPerSlot*32*4096 { return nil, fmt.Errorf("beacon client in blobSidecars got error or empty response fetching older blobs in slot: %d, an archive endpoint is required, please refer to https://docs.arbitrum.io/run-arbitrum-node/l1-ethereum-beacon-chain-rpc-providers, err: %w", slot, err) diff --git a/util/merkletree/merkleEventProof_test.go b/util/merkletree/merkleEventProof_test.go index b64cc88c2a..6af8479190 100644 --- a/util/merkletree/merkleEventProof_test.go +++ b/util/merkletree/merkleEventProof_test.go @@ -22,6 +22,7 @@ func initializedMerkleAccumulatorForTesting() *merkleAccumulator.MerkleAccumulat func TestProofForNext(t *testing.T) { leaves := make([]common.Hash, 13) for i := range leaves { + // #nosec G115 leaves[i] = pseudorandomForTesting(uint64(i)) } diff --git a/validator/server_arb/machine_cache.go b/validator/server_arb/machine_cache.go index 55ef61cf11..35f3406236 100644 --- a/validator/server_arb/machine_cache.go +++ b/validator/server_arb/machine_cache.go @@ -31,7 +31,7 @@ type MachineCache struct { } type MachineCacheConfig struct { - CachedChallengeMachines int `koanf:"cached-challenge-machines"` + CachedChallengeMachines uint64 `koanf:"cached-challenge-machines"` InitialSteps uint64 `koanf:"initial-steps"` } @@ -42,7 +42,7 @@ var DefaultMachineCacheConfig = MachineCacheConfig{ func MachineCacheConfigConfigAddOptions(prefix string, f *flag.FlagSet) { f.Uint64(prefix+".initial-steps", DefaultMachineCacheConfig.InitialSteps, "initial steps between machines") - f.Int(prefix+".cached-challenge-machines", DefaultMachineCacheConfig.CachedChallengeMachines, "how many machines to store in cache while working on a challenge (should be even)") + f.Uint64(prefix+".cached-challenge-machines", DefaultMachineCacheConfig.CachedChallengeMachines, "how many machines to store in cache while working on a challenge (should be even)") } // `initialMachine` won't be mutated by this function. @@ -140,7 +140,7 @@ func (c *MachineCache) unlockBuild(err error) { } func (c *MachineCache) setRangeLocked(ctx context.Context, start uint64, end uint64) error { - newInterval := (end - start) / uint64(c.config.CachedChallengeMachines) + newInterval := (end - start) / c.config.CachedChallengeMachines if newInterval == 0 { newInterval = 2 } @@ -150,7 +150,7 @@ func (c *MachineCache) setRangeLocked(ctx context.Context, start uint64, end uin if end >= c.finalMachineStep { end = c.finalMachineStep - newInterval/2 } - newInterval = (end - start) / uint64(c.config.CachedChallengeMachines) + newInterval = (end - start) / c.config.CachedChallengeMachines if newInterval == 0 { newInterval = 1 } @@ -212,7 +212,7 @@ func (c *MachineCache) populateCache(ctx context.Context) error { if nextMachine.GetStepCount()+c.machineStepInterval >= c.finalMachineStep { break } - if len(c.machines) >= c.config.CachedChallengeMachines { + if uint64(len(c.machines)) >= c.config.CachedChallengeMachines { break } nextMachine = nextMachine.CloneMachineInterface() @@ -236,6 +236,7 @@ func (c *MachineCache) getClosestMachine(stepCount uint64) (int, MachineInterfac } stepsFromStart := stepCount - c.firstMachineStep var index int + // #nosec G115 if c.machineStepInterval == 0 || stepsFromStart > c.machineStepInterval*uint64(len(c.machines)-1) { index = len(c.machines) - 1 } else { diff --git a/validator/server_jit/jit_machine.go b/validator/server_jit/jit_machine.go index e7753748ab..06c451bda1 100644 --- a/validator/server_jit/jit_machine.go +++ b/validator/server_jit/jit_machine.go @@ -306,6 +306,7 @@ func (machine *JitMachine) prove( if err != nil { return state, fmt.Errorf("failed to read memory usage from Jit machine: %w", err) } + // #nosec G115 if memoryUsed > uint64(machine.wasmMemoryUsageLimit) { log.Warn("memory used by jit wasm exceeds the wasm memory usage limit", "limit", machine.wasmMemoryUsageLimit, "memoryUsed", memoryUsed) } diff --git a/wavmio/stub.go b/wavmio/stub.go index 1395fb4235..0c82506ff3 100644 --- a/wavmio/stub.go +++ b/wavmio/stub.go @@ -66,6 +66,7 @@ func parsePreimageBytes(path string) { if err != nil { panic(err) } + // #nosec G115 if uint64(read) != fieldSize { panic("missing bytes reading data") } @@ -77,18 +78,18 @@ func parsePreimageBytes(path string) { func StubInit() { preimages = make(map[common.Hash][]byte) var delayedMsgPath arrayFlags - seqMsgPosFlag := flag.Int("inbox-position", 0, "position for sequencer inbox message") - posWithinMsgFlag := flag.Int("position-within-message", 0, "position inside sequencer inbox message") - delayedPositionFlag := flag.Int("delayed-inbox-position", 0, "position for first delayed inbox message") + seqMsgPosFlag := flag.Uint64("inbox-position", 0, "position for sequencer inbox message") + posWithinMsgFlag := flag.Uint64("position-within-message", 0, "position inside sequencer inbox message") + delayedPositionFlag := flag.Uint64("delayed-inbox-position", 0, "position for first delayed inbox message") lastBlockFlag := flag.String("last-block-hash", "0000000000000000000000000000000000000000000000000000000000000000", "lastBlockHash") flag.Var(&delayedMsgPath, "delayed-inbox", "delayed inbox messages (multiple values)") inboxPath := flag.String("inbox", "", "file to load sequencer message") preimagesPath := flag.String("preimages", "", "file to load preimages from") flag.Parse() - seqMsgPos = uint64(*seqMsgPosFlag) - posWithinMsg = uint64(*posWithinMsgFlag) - delayedMsgFirstPos = uint64(*delayedPositionFlag) + seqMsgPos = *seqMsgPosFlag + posWithinMsg = *posWithinMsgFlag + delayedMsgFirstPos = *delayedPositionFlag lastBlockHash = common.HexToHash(*lastBlockFlag) for _, path := range delayedMsgPath { msg, err := os.ReadFile(path) From 35a22ac87ca793a0f3222ad3adeec365314afb1e Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Tue, 10 Sep 2024 16:27:10 +0530 Subject: [PATCH 155/489] address PR comments --- arbnode/dataposter/dataposter_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/arbnode/dataposter/dataposter_test.go b/arbnode/dataposter/dataposter_test.go index 769919474b..7bf0f86e6f 100644 --- a/arbnode/dataposter/dataposter_test.go +++ b/arbnode/dataposter/dataposter_test.go @@ -153,12 +153,12 @@ func TestMaxFeeCapFormulaCalculation(t *testing.T) { } } -type stubL1ClientInterface struct { +type stubL1ClientInner struct { senderNonce uint64 suggestedGasTipCap *big.Int } -func (c *stubL1ClientInterface) CallContext(ctx_in context.Context, result interface{}, method string, args ...interface{}) error { +func (c *stubL1ClientInner) CallContext(ctx_in context.Context, result interface{}, method string, args ...interface{}) error { switch method { case "eth_getTransactionCount": ptr, ok := result.(*hexutil.Uint64) @@ -176,13 +176,13 @@ func (c *stubL1ClientInterface) CallContext(ctx_in context.Context, result inter return nil } -func (c *stubL1ClientInterface) EthSubscribe(ctx context.Context, channel interface{}, args ...interface{}) (*rpc.ClientSubscription, error) { +func (c *stubL1ClientInner) EthSubscribe(ctx context.Context, channel interface{}, args ...interface{}) (*rpc.ClientSubscription, error) { return nil, nil } -func (c *stubL1ClientInterface) BatchCallContext(ctx context.Context, b []rpc.BatchElem) error { +func (c *stubL1ClientInner) BatchCallContext(ctx context.Context, b []rpc.BatchElem) error { return nil } -func (c *stubL1ClientInterface) Close() {} +func (c *stubL1ClientInner) Close() {} func TestFeeAndTipCaps_EnoughBalance_NoBacklog_NoUnconfirmed_BlobTx(t *testing.T) { conf := func() *DataPosterConfig { @@ -214,7 +214,7 @@ func TestFeeAndTipCaps_EnoughBalance_NoBacklog_NoUnconfirmed_BlobTx(t *testing.T extraBacklog: func() uint64 { return 0 }, balance: big.NewInt(0).Mul(big.NewInt(params.Ether), big.NewInt(10)), usingNoOpStorage: false, - client: ethclient.NewClient(&stubL1ClientInterface{ + client: ethclient.NewClient(&stubL1ClientInner{ senderNonce: 1, suggestedGasTipCap: big.NewInt(2 * params.GWei), }), @@ -345,7 +345,7 @@ func TestFeeAndTipCaps_RBF_RisingBlobFee_FallingBaseFee(t *testing.T) { extraBacklog: func() uint64 { return 0 }, balance: big.NewInt(0).Mul(big.NewInt(params.Ether), big.NewInt(10)), usingNoOpStorage: false, - client: ethclient.NewClient(&stubL1ClientInterface{ + client: ethclient.NewClient(&stubL1ClientInner{ senderNonce: 1, suggestedGasTipCap: big.NewInt(2 * params.GWei), }), From 6cc3a5dfe8ecf248c950264684612cdce958caf7 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 11 Sep 2024 10:12:38 -0600 Subject: [PATCH 156/489] block_validator: read batches once per batch --- staker/block_validator.go | 60 +++++++--- staker/stateless_block_validator.go | 171 ++++++++++++++++------------ 2 files changed, 145 insertions(+), 86 deletions(-) diff --git a/staker/block_validator.go b/staker/block_validator.go index 7a7efca846..18411c6163 100644 --- a/staker/block_validator.go +++ b/staker/block_validator.go @@ -56,12 +56,12 @@ type BlockValidator struct { chainCaughtUp bool // can only be accessed from creation thread or if holding reorg-write - nextCreateBatch []byte - nextCreateBatchBlockHash common.Hash - nextCreateBatchMsgCount arbutil.MessageIndex - nextCreateBatchReread bool - nextCreateStartGS validator.GoGlobalState - nextCreatePrevDelayed uint64 + nextCreateBatch *FullBatchInfo + nextCreateBatchReread bool + prevBatchCache map[uint64]*FullBatchInfo + + nextCreateStartGS validator.GoGlobalState + nextCreatePrevDelayed uint64 // can only be accessed from from validation thread or if holding reorg-write lastValidGS validator.GoGlobalState @@ -108,6 +108,7 @@ type BlockValidatorConfig struct { PrerecordedBlocks uint64 `koanf:"prerecorded-blocks" reload:"hot"` RecordingIterLimit uint64 `koanf:"recording-iter-limit"` ForwardBlocks uint64 `koanf:"forward-blocks" reload:"hot"` + BatchCacheLimit uint32 `koanf:"batch-cache-limit"` CurrentModuleRoot string `koanf:"current-module-root"` // TODO(magic) requires reinitialization on hot reload PendingUpgradeModuleRoot string `koanf:"pending-upgrade-module-root"` // TODO(magic) requires StatelessBlockValidator recreation on hot reload FailureIsFatal bool `koanf:"failure-is-fatal" reload:"hot"` @@ -174,6 +175,7 @@ func BlockValidatorConfigAddOptions(prefix string, f *pflag.FlagSet) { f.Duration(prefix+".validation-poll", DefaultBlockValidatorConfig.ValidationPoll, "poll time to check validations") f.Uint64(prefix+".forward-blocks", DefaultBlockValidatorConfig.ForwardBlocks, "prepare entries for up to that many blocks ahead of validation (small footprint)") f.Uint64(prefix+".prerecorded-blocks", DefaultBlockValidatorConfig.PrerecordedBlocks, "record that many blocks ahead of validation (larger footprint)") + f.Uint32(prefix+".batch-cache-limit", DefaultBlockValidatorConfig.BatchCacheLimit, "limit number of old batches to keep in block-validator") f.String(prefix+".current-module-root", DefaultBlockValidatorConfig.CurrentModuleRoot, "current wasm module root ('current' read from chain, 'latest' from machines/latest dir, or provide hash)") f.Uint64(prefix+".recording-iter-limit", DefaultBlockValidatorConfig.RecordingIterLimit, "limit on block recordings sent per iteration") f.String(prefix+".pending-upgrade-module-root", DefaultBlockValidatorConfig.PendingUpgradeModuleRoot, "pending upgrade wasm module root to additionally validate (hash, 'latest' or empty)") @@ -194,6 +196,7 @@ var DefaultBlockValidatorConfig = BlockValidatorConfig{ ValidationPoll: time.Second, ForwardBlocks: 1024, PrerecordedBlocks: uint64(2 * runtime.NumCPU()), + BatchCacheLimit: 20, CurrentModuleRoot: "current", PendingUpgradeModuleRoot: "latest", FailureIsFatal: true, @@ -209,6 +212,7 @@ var TestBlockValidatorConfig = BlockValidatorConfig{ RedisValidationClientConfig: redis.TestValidationClientConfig, ValidationPoll: 100 * time.Millisecond, ForwardBlocks: 128, + BatchCacheLimit: 20, PrerecordedBlocks: uint64(2 * runtime.NumCPU()), RecordingIterLimit: 20, CurrentModuleRoot: "latest", @@ -271,6 +275,7 @@ func NewBlockValidator( progressValidationsChan: make(chan struct{}, 1), config: config, fatalErr: fatalErr, + prevBatchCache: make(map[uint64]*FullBatchInfo), } if !config().Dangerous.ResetBlockValidation { validated, err := ret.ReadLastValidatedInfo() @@ -571,33 +576,58 @@ func (v *BlockValidator) createNextValidationEntry(ctx context.Context) (bool, e } if v.nextCreateStartGS.PosInBatch == 0 || v.nextCreateBatchReread { // new batch - found, batch, batchBlockHash, count, err := v.readBatch(ctx, v.nextCreateStartGS.Batch) + found, fullBatchInfo, err := v.readBatch(ctx, v.nextCreateStartGS.Batch) if !found { return false, err } - v.nextCreateBatch = batch - v.nextCreateBatchBlockHash = batchBlockHash - v.nextCreateBatchMsgCount = count + if v.nextCreateBatch != nil { + v.prevBatchCache[v.nextCreateBatch.Number] = v.nextCreateBatch + } + v.nextCreateBatch = fullBatchInfo // #nosec G115 - validatorMsgCountCurrentBatch.Update(int64(count)) + validatorMsgCountCurrentBatch.Update(int64(fullBatchInfo.MsgCount)) + batchCacheLimit := v.config().BatchCacheLimit + if len(v.prevBatchCache) > int(batchCacheLimit) { + for num := range v.prevBatchCache { + if num < v.nextCreateStartGS.Batch-uint64(batchCacheLimit) { + delete(v.prevBatchCache, num) + } + } + } v.nextCreateBatchReread = false } endGS := validator.GoGlobalState{ BlockHash: endRes.BlockHash, SendRoot: endRes.SendRoot, } - if pos+1 < v.nextCreateBatchMsgCount { + if pos+1 < v.nextCreateBatch.MsgCount { endGS.Batch = v.nextCreateStartGS.Batch endGS.PosInBatch = v.nextCreateStartGS.PosInBatch + 1 - } else if pos+1 == v.nextCreateBatchMsgCount { + } else if pos+1 == v.nextCreateBatch.MsgCount { endGS.Batch = v.nextCreateStartGS.Batch + 1 endGS.PosInBatch = 0 } else { - return false, fmt.Errorf("illegal batch msg count %d pos %d batch %d", v.nextCreateBatchMsgCount, pos, endGS.Batch) + return false, fmt.Errorf("illegal batch msg count %d pos %d batch %d", v.nextCreateBatch.MsgCount, pos, endGS.Batch) } chainConfig := v.streamer.ChainConfig() + batchReader := func(batchNum uint64) (*FullBatchInfo, error) { + if batchNum == v.nextCreateBatch.Number { + return v.nextCreateBatch, nil + } + if entry, found := v.prevBatchCache[batchNum]; found { + return entry, nil + } + found, entry, err := v.readBatch(ctx, batchNum) + if err != nil { + return nil, err + } + if !found { + return nil, fmt.Errorf("batch %d not found", batchNum) + } + return entry, nil + } entry, err := newValidationEntry( - pos, v.nextCreateStartGS, endGS, msg, v.nextCreateBatch, v.nextCreateBatchBlockHash, v.nextCreatePrevDelayed, chainConfig, + pos, v.nextCreateStartGS, endGS, msg, batchReader, v.nextCreatePrevDelayed, chainConfig, ) if err != nil { return false, err diff --git a/staker/stateless_block_validator.go b/staker/stateless_block_validator.go index f54ec8b58c..857bc255ad 100644 --- a/staker/stateless_block_validator.go +++ b/staker/stateless_block_validator.go @@ -114,6 +114,14 @@ const ( Ready ) +type FullBatchInfo struct { + Number uint64 + BlockHash common.Hash + PostedData []byte + MsgCount arbutil.MessageIndex + Preimages map[arbutil.PreimageType]map[common.Hash][]byte +} + type validationEntry struct { Stage ValidationEntryStage // Valid since ReadyforRecord: @@ -171,16 +179,27 @@ func newValidationEntry( start validator.GoGlobalState, end validator.GoGlobalState, msg *arbostypes.MessageWithMetadata, - batch []byte, - batchBlockHash common.Hash, + fullBatchFetcher func(uint64) (*FullBatchInfo, error), prevDelayed uint64, chainConfig *params.ChainConfig, ) (*validationEntry, error) { - batchInfo := validator.BatchInfo{ - Number: start.Batch, - BlockHash: batchBlockHash, - Data: batch, + preimages := make(map[arbutil.PreimageType]map[common.Hash][]byte) + valBatches := make([]validator.BatchInfo, 0) + + valBatchFetcher := func(batchNum uint64) ([]byte, error) { + fullBatchInfo, err := fullBatchFetcher(batchNum) + if err != nil { + return nil, err + } + valBatches = append(valBatches, validator.BatchInfo{ + Number: batchNum, + BlockHash: fullBatchInfo.BlockHash, + Data: fullBatchInfo.PostedData, + }) + clonePreimagesInto(preimages, fullBatchInfo.Preimages) + return fullBatchInfo.PostedData, nil } + hasDelayed := false var delayedNum uint64 if msg.DelayedMessagesRead == prevDelayed+1 { @@ -189,6 +208,15 @@ func newValidationEntry( } else if msg.DelayedMessagesRead != prevDelayed { return nil, fmt.Errorf("illegal validation entry delayedMessage %d, previous %d", msg.DelayedMessagesRead, prevDelayed) } + + if _, err := valBatchFetcher(start.Batch); err != nil { + return nil, err + } + msg.Message.BatchGasCost = nil + if err := msg.Message.FillInBatchGasCost(valBatchFetcher); err != nil { + return nil, err + } + return &validationEntry{ Stage: ReadyForRecord, Pos: pos, @@ -197,8 +225,9 @@ func newValidationEntry( HasDelayedMsg: hasDelayed, DelayedMsgNr: delayedNum, msg: msg, - BatchInfo: []validator.BatchInfo{batchInfo}, + BatchInfo: valBatches, ChainConfig: chainConfig, + Preimages: preimages, }, nil } @@ -246,30 +275,73 @@ func NewStatelessBlockValidator( }, nil } -func (v *StatelessBlockValidator) readBatch(ctx context.Context, batchNum uint64) (bool, []byte, common.Hash, arbutil.MessageIndex, error) { +func (v *StatelessBlockValidator) readBatch(ctx context.Context, batchNum uint64) (bool, *FullBatchInfo, error) { batchCount, err := v.inboxTracker.GetBatchCount() if err != nil { - return false, nil, common.Hash{}, 0, err + return false, nil, err } if batchCount <= batchNum { - return false, nil, common.Hash{}, 0, nil + return false, nil, nil } batchMsgCount, err := v.inboxTracker.GetBatchMessageCount(batchNum) if err != nil { - return false, nil, common.Hash{}, 0, err + return false, nil, err } - batch, batchBlockHash, err := v.inboxReader.GetSequencerMessageBytes(ctx, batchNum) + postedData, batchBlockHash, err := v.inboxReader.GetSequencerMessageBytes(ctx, batchNum) if err != nil { - return false, nil, common.Hash{}, 0, err + return false, nil, err + } + preimages := make(map[arbutil.PreimageType]map[common.Hash][]byte) + if len(postedData) > 40 { + foundDA := false + for _, dapReader := range v.dapReaders { + if dapReader != nil && dapReader.IsValidHeaderByte(postedData[40]) { + preimageRecorder := daprovider.RecordPreimagesTo(preimages) + _, err := dapReader.RecoverPayloadFromBatch(ctx, batchNum, batchBlockHash, postedData, preimageRecorder, true) + if err != nil { + // Matches the way keyset validation was done inside DAS readers i.e logging the error + // But other daproviders might just want to return the error + if errors.Is(err, daprovider.ErrSeqMsgValidation) && daprovider.IsDASMessageHeaderByte(postedData[40]) { + log.Error(err.Error()) + } else { + return false, nil, err + } + } + foundDA = true + break + } + } + if !foundDA { + if daprovider.IsDASMessageHeaderByte(postedData[40]) { + log.Error("No DAS Reader configured, but sequencer message found with DAS header") + } + } + } + fullInfo := FullBatchInfo{ + Number: batchNum, + BlockHash: batchBlockHash, + PostedData: postedData, + MsgCount: batchMsgCount, + Preimages: preimages, + } + return true, &fullInfo, nil +} + +func clonePreimagesInto(dest, source map[arbutil.PreimageType]map[common.Hash][]byte) { + for piType, piMap := range source { + if dest[piType] == nil { + dest[piType] = make(map[common.Hash][]byte, len(source[piType])) + } + for hash, preimage := range piMap { + dest[piType][hash] = preimage + } } - return true, batch, batchBlockHash, batchMsgCount, nil } func (v *StatelessBlockValidator) ValidationEntryRecord(ctx context.Context, e *validationEntry) error { if e.Stage != ReadyForRecord { return fmt.Errorf("validation entry should be ReadyForRecord, is: %v", e.Stage) } - e.Preimages = make(map[arbutil.PreimageType]map[common.Hash][]byte) if e.Pos != 0 { recording, err := v.recorder.RecordBlockCreation(ctx, e.Pos, e.msg) if err != nil { @@ -278,30 +350,10 @@ func (v *StatelessBlockValidator) ValidationEntryRecord(ctx context.Context, e * if recording.BlockHash != e.End.BlockHash { return fmt.Errorf("recording failed: pos %d, hash expected %v, got %v", e.Pos, e.End.BlockHash, recording.BlockHash) } - // record any additional batch fetching - batchFetcher := func(batchNum uint64) ([]byte, error) { - found, data, hash, _, err := v.readBatch(ctx, batchNum) - if err != nil { - return nil, err - } - if !found { - return nil, errors.New("batch not found") - } - e.BatchInfo = append(e.BatchInfo, validator.BatchInfo{ - Number: batchNum, - BlockHash: hash, - Data: data, - }) - return data, nil - } - e.msg.Message.BatchGasCost = nil - err = e.msg.Message.FillInBatchGasCost(batchFetcher) - if err != nil { - return err - } - if recording.Preimages != nil { - e.Preimages[arbutil.Keccak256PreimageType] = recording.Preimages + recordingPreimages := make(map[arbutil.PreimageType]map[common.Hash][]byte) + recordingPreimages[arbutil.Keccak256PreimageType] = recording.Preimages + clonePreimagesInto(e.Preimages, recordingPreimages) } e.UserWasms = recording.UserWasms } @@ -316,35 +368,6 @@ func (v *StatelessBlockValidator) ValidationEntryRecord(ctx context.Context, e * } e.DelayedMsg = delayedMsg } - for _, batch := range e.BatchInfo { - if len(batch.Data) <= 40 { - continue - } - foundDA := false - for _, dapReader := range v.dapReaders { - if dapReader != nil && dapReader.IsValidHeaderByte(batch.Data[40]) { - preimageRecorder := daprovider.RecordPreimagesTo(e.Preimages) - _, err := dapReader.RecoverPayloadFromBatch(ctx, batch.Number, batch.BlockHash, batch.Data, preimageRecorder, true) - if err != nil { - // Matches the way keyset validation was done inside DAS readers i.e logging the error - // But other daproviders might just want to return the error - if errors.Is(err, daprovider.ErrSeqMsgValidation) && daprovider.IsDASMessageHeaderByte(batch.Data[40]) { - log.Error(err.Error()) - } else { - return err - } - } - foundDA = true - break - } - } - if !foundDA { - if daprovider.IsDASMessageHeaderByte(batch.Data[40]) { - log.Error("No DAS Reader configured, but sequencer message found with DAS header") - } - } - } - e.msg = nil // no longer needed e.Stage = Ready return nil @@ -404,11 +427,17 @@ func (v *StatelessBlockValidator) CreateReadyValidationEntry(ctx context.Context } start := buildGlobalState(*prevResult, startPos) end := buildGlobalState(*result, endPos) - seqMsg, batchBlockHash, err := v.inboxReader.GetSequencerMessageBytes(ctx, startPos.BatchNumber) - if err != nil { - return nil, err + fullBatchReader := func(batchNum uint64) (*FullBatchInfo, error) { + found, fullBlockInfo, err := v.readBatch(ctx, batchNum) + if err != nil { + return nil, err + } + if !found { + return nil, fmt.Errorf("batch %d not found", startPos.BatchNumber) + } + return fullBlockInfo, nil } - entry, err := newValidationEntry(pos, start, end, msg, seqMsg, batchBlockHash, prevDelayed, v.streamer.ChainConfig()) + entry, err := newValidationEntry(pos, start, end, msg, fullBatchReader, prevDelayed, v.streamer.ChainConfig()) if err != nil { return nil, err } From d752445bfac0da673fcfe7ec7f3dbe209223ac71 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 11 Sep 2024 10:16:50 -0600 Subject: [PATCH 157/489] remove blockhash from validator.BatchInfo --- staker/stateless_block_validator.go | 5 ++--- validator/validation_entry.go | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/staker/stateless_block_validator.go b/staker/stateless_block_validator.go index 857bc255ad..3d92b9df59 100644 --- a/staker/stateless_block_validator.go +++ b/staker/stateless_block_validator.go @@ -192,9 +192,8 @@ func newValidationEntry( return nil, err } valBatches = append(valBatches, validator.BatchInfo{ - Number: batchNum, - BlockHash: fullBatchInfo.BlockHash, - Data: fullBatchInfo.PostedData, + Number: batchNum, + Data: fullBatchInfo.PostedData, }) clonePreimagesInto(preimages, fullBatchInfo.Preimages) return fullBatchInfo.PostedData, nil diff --git a/validator/validation_entry.go b/validator/validation_entry.go index d340993fa2..4ec6919d3b 100644 --- a/validator/validation_entry.go +++ b/validator/validation_entry.go @@ -7,9 +7,8 @@ import ( ) type BatchInfo struct { - Number uint64 - BlockHash common.Hash - Data []byte + Number uint64 + Data []byte } type ValidationInput struct { From 25b4afe0363f65a9df15ee27a20917151dddf3f8 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 11 Sep 2024 10:31:01 -0600 Subject: [PATCH 158/489] update default forward-blocks value --- staker/block_validator.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/staker/block_validator.go b/staker/block_validator.go index 18411c6163..847b6f6349 100644 --- a/staker/block_validator.go +++ b/staker/block_validator.go @@ -173,7 +173,7 @@ func BlockValidatorConfigAddOptions(prefix string, f *pflag.FlagSet) { redis.ValidationClientConfigAddOptions(prefix+".redis-validation-client-config", f) f.String(prefix+".validation-server-configs-list", DefaultBlockValidatorConfig.ValidationServerConfigsList, "array of execution rpc configs given as a json string. time duration should be supplied in number indicating nanoseconds") f.Duration(prefix+".validation-poll", DefaultBlockValidatorConfig.ValidationPoll, "poll time to check validations") - f.Uint64(prefix+".forward-blocks", DefaultBlockValidatorConfig.ForwardBlocks, "prepare entries for up to that many blocks ahead of validation (small footprint)") + f.Uint64(prefix+".forward-blocks", DefaultBlockValidatorConfig.ForwardBlocks, "prepare entries for up to that many blocks ahead of validation (stores batch-copy per block)") f.Uint64(prefix+".prerecorded-blocks", DefaultBlockValidatorConfig.PrerecordedBlocks, "record that many blocks ahead of validation (larger footprint)") f.Uint32(prefix+".batch-cache-limit", DefaultBlockValidatorConfig.BatchCacheLimit, "limit number of old batches to keep in block-validator") f.String(prefix+".current-module-root", DefaultBlockValidatorConfig.CurrentModuleRoot, "current wasm module root ('current' read from chain, 'latest' from machines/latest dir, or provide hash)") @@ -194,7 +194,7 @@ var DefaultBlockValidatorConfig = BlockValidatorConfig{ ValidationServer: rpcclient.DefaultClientConfig, RedisValidationClientConfig: redis.DefaultValidationClientConfig, ValidationPoll: time.Second, - ForwardBlocks: 1024, + ForwardBlocks: 128, PrerecordedBlocks: uint64(2 * runtime.NumCPU()), BatchCacheLimit: 20, CurrentModuleRoot: "current", From c98d652ce97e99063596096f5b0bd993c3cf22d4 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 11 Sep 2024 10:38:08 -0600 Subject: [PATCH 159/489] block_validator: prevBatchCache in reorgs --- staker/block_validator.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/staker/block_validator.go b/staker/block_validator.go index 847b6f6349..49dc59c4a2 100644 --- a/staker/block_validator.go +++ b/staker/block_validator.go @@ -1027,6 +1027,9 @@ func (v *BlockValidator) UpdateLatestStaked(count arbutil.MessageIndex, globalSt v.nextCreateStartGS = globalState v.nextCreatePrevDelayed = msg.DelayedMessagesRead v.nextCreateBatchReread = true + if v.nextCreateBatch != nil { + v.prevBatchCache[v.nextCreateBatch.Number] = v.nextCreateBatch + } v.createdA.Store(countUint64) } // under the reorg mutex we don't need atomic access @@ -1053,6 +1056,7 @@ func (v *BlockValidator) ReorgToBatchCount(count uint64) { defer v.reorgMutex.Unlock() if v.nextCreateStartGS.Batch >= count { v.nextCreateBatchReread = true + v.prevBatchCache = make(map[uint64]*FullBatchInfo) } } @@ -1093,6 +1097,7 @@ func (v *BlockValidator) Reorg(ctx context.Context, count arbutil.MessageIndex) v.nextCreateStartGS = buildGlobalState(*res, endPosition) v.nextCreatePrevDelayed = msg.DelayedMessagesRead v.nextCreateBatchReread = true + v.prevBatchCache = make(map[uint64]*FullBatchInfo) countUint64 := uint64(count) v.createdA.Store(countUint64) // under the reorg mutex we don't need atomic access From b71fd7b5befb34901b2fdfbe1ead03dd1b5eeb26 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 11 Sep 2024 11:03:59 -0600 Subject: [PATCH 160/489] block_recorder: separate config, add place to max blocks --- execution/gethexec/block_recorder.go | 32 +++++++++++++++++++++++++--- execution/gethexec/node.go | 30 +++++++++++++------------- 2 files changed, 44 insertions(+), 18 deletions(-) diff --git a/execution/gethexec/block_recorder.go b/execution/gethexec/block_recorder.go index 8879c90702..a31b6b3736 100644 --- a/execution/gethexec/block_recorder.go +++ b/execution/gethexec/block_recorder.go @@ -16,6 +16,7 @@ import ( "github.com/offchainlabs/nitro/arbos/arbostypes" "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/execution" + flag "github.com/spf13/pflag" ) // BlockRecorder uses a separate statedatabase from the blockchain. @@ -25,6 +26,8 @@ import ( // Most recent/advanced header we ever computed (lastHdr) // Hopefully - some recent valid block. For that we always keep one candidate block until it becomes validated. type BlockRecorder struct { + config *BlockRecorderConfig + recordingDatabase *arbitrum.RecordingDatabase execEngine *ExecutionEngine @@ -39,10 +42,33 @@ type BlockRecorder struct { preparedLock sync.Mutex } -func NewBlockRecorder(config *arbitrum.RecordingDatabaseConfig, execEngine *ExecutionEngine, ethDb ethdb.Database) *BlockRecorder { +type BlockRecorderConfig struct { + TrieDirtyCache int `koanf:"trie-dirty-cache"` + TrieCleanCache int `koanf:"trie-clean-cache"` + MaxPrepared int `koanf:"max-prepared"` +} + +var DefaultBlockRecorderConfig = BlockRecorderConfig{ + TrieDirtyCache: 1024, + TrieCleanCache: 16, + MaxPrepared: 1000, +} + +func BlockRecorderConfigAddOptions(prefix string, f *flag.FlagSet) { + f.Int(prefix+".trie-dirty-cache", DefaultBlockRecorderConfig.TrieDirtyCache, "like trie-dirty-cache for the separate, recording database (used for validation)") + f.Int(prefix+".trie-clean-cache", DefaultBlockRecorderConfig.TrieCleanCache, "like trie-clean-cache for the separate, recording database (used for validation)") + f.Int(prefix+".max-prepared", DefaultBlockRecorderConfig.MaxPrepared, "max references to store in the recording database") +} + +func NewBlockRecorder(config *BlockRecorderConfig, execEngine *ExecutionEngine, ethDb ethdb.Database) *BlockRecorder { + dbConfig := arbitrum.RecordingDatabaseConfig{ + TrieDirtyCache: config.TrieDirtyCache, + TrieCleanCache: config.TrieCleanCache, + } recorder := &BlockRecorder{ + config: config, execEngine: execEngine, - recordingDatabase: arbitrum.NewRecordingDatabase(config, ethDb, execEngine.bc), + recordingDatabase: arbitrum.NewRecordingDatabase(&dbConfig, ethDb, execEngine.bc), } execEngine.SetRecorder(recorder) return recorder @@ -303,7 +329,7 @@ func (r *BlockRecorder) PrepareForRecord(ctx context.Context, start, end arbutil r.updateLastHdr(header) hdrNum++ } - r.preparedAddTrim(references, 1000) + r.preparedAddTrim(references, r.config.MaxPrepared) return nil } diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index 21c2b4bece..5847f392b1 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -78,19 +78,19 @@ func StylusTargetConfigAddOptions(prefix string, f *flag.FlagSet) { } type Config struct { - ParentChainReader headerreader.Config `koanf:"parent-chain-reader" reload:"hot"` - Sequencer SequencerConfig `koanf:"sequencer" reload:"hot"` - RecordingDatabase arbitrum.RecordingDatabaseConfig `koanf:"recording-database"` - TxPreChecker TxPreCheckerConfig `koanf:"tx-pre-checker" reload:"hot"` - Forwarder ForwarderConfig `koanf:"forwarder"` - ForwardingTarget string `koanf:"forwarding-target"` - SecondaryForwardingTarget []string `koanf:"secondary-forwarding-target"` - Caching CachingConfig `koanf:"caching"` - RPC arbitrum.Config `koanf:"rpc"` - TxLookupLimit uint64 `koanf:"tx-lookup-limit"` - EnablePrefetchBlock bool `koanf:"enable-prefetch-block"` - SyncMonitor SyncMonitorConfig `koanf:"sync-monitor"` - StylusTarget StylusTargetConfig `koanf:"stylus-target"` + ParentChainReader headerreader.Config `koanf:"parent-chain-reader" reload:"hot"` + Sequencer SequencerConfig `koanf:"sequencer" reload:"hot"` + RecordingDatabase BlockRecorderConfig `koanf:"recording-database"` + TxPreChecker TxPreCheckerConfig `koanf:"tx-pre-checker" reload:"hot"` + Forwarder ForwarderConfig `koanf:"forwarder"` + ForwardingTarget string `koanf:"forwarding-target"` + SecondaryForwardingTarget []string `koanf:"secondary-forwarding-target"` + Caching CachingConfig `koanf:"caching"` + RPC arbitrum.Config `koanf:"rpc"` + TxLookupLimit uint64 `koanf:"tx-lookup-limit"` + EnablePrefetchBlock bool `koanf:"enable-prefetch-block"` + SyncMonitor SyncMonitorConfig `koanf:"sync-monitor"` + StylusTarget StylusTargetConfig `koanf:"stylus-target"` forwardingTarget string } @@ -123,7 +123,7 @@ func ConfigAddOptions(prefix string, f *flag.FlagSet) { arbitrum.ConfigAddOptions(prefix+".rpc", f) SequencerConfigAddOptions(prefix+".sequencer", f) headerreader.AddOptions(prefix+".parent-chain-reader", f) - arbitrum.RecordingDatabaseConfigAddOptions(prefix+".recording-database", f) + BlockRecorderConfigAddOptions(prefix+".recording-database", f) f.String(prefix+".forwarding-target", ConfigDefault.ForwardingTarget, "transaction forwarding target URL, or \"null\" to disable forwarding (iff not sequencer)") f.StringSlice(prefix+".secondary-forwarding-target", ConfigDefault.SecondaryForwardingTarget, "secondary transaction forwarding target URL") AddOptionsForNodeForwarderConfig(prefix+".forwarder", f) @@ -139,7 +139,7 @@ var ConfigDefault = Config{ RPC: arbitrum.DefaultConfig, Sequencer: DefaultSequencerConfig, ParentChainReader: headerreader.DefaultConfig, - RecordingDatabase: arbitrum.DefaultRecordingDatabaseConfig, + RecordingDatabase: DefaultBlockRecorderConfig, ForwardingTarget: "", SecondaryForwardingTarget: []string{}, TxPreChecker: DefaultTxPreCheckerConfig, From 5f5d9c8de93144874e47af3b0c56fb911e1f8de6 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Wed, 11 Sep 2024 15:21:36 -0300 Subject: [PATCH 161/489] Fix DAS test by setting the default config The batch-poster wasn't using the DAS because the max retention period wasn't being set correctly. --- system_tests/das_test.go | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/system_tests/das_test.go b/system_tests/das_test.go index 9f4d153b6f..593058eaac 100644 --- a/system_tests/das_test.go +++ b/system_tests/das_test.go @@ -44,18 +44,13 @@ func startLocalDASServer( pubkey, _, err := das.GenerateAndStoreKeys(keyDir) Require(t, err) - config := das.DataAvailabilityConfig{ - Enable: true, - Key: das.KeyConfig{ - KeyDir: keyDir, - }, - LocalFileStorage: das.LocalFileStorageConfig{ - Enable: true, - DataDir: dataDir, - }, - ParentChainNodeURL: "none", - RequestTimeout: 5 * time.Second, - } + config := das.DefaultDataAvailabilityConfig + config.Enable = true + config.Key = das.KeyConfig{KeyDir: keyDir} + config.ParentChainNodeURL = "none" + config.LocalFileStorage = das.DefaultLocalFileStorageConfig + config.LocalFileStorage.Enable = true + config.LocalFileStorage.DataDir = dataDir storageService, lifecycleManager, err := das.CreatePersistentStorageService(ctx, &config) defer lifecycleManager.StopAndWaitUntil(time.Second) From 747a0f37c0f72a1135ecfb3eb4d1eab6820b0ca1 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 11 Sep 2024 13:31:09 -0600 Subject: [PATCH 162/489] block_validator: delete batch from cache after used by report --- staker/block_validator.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/staker/block_validator.go b/staker/block_validator.go index 49dc59c4a2..4a5bd740a8 100644 --- a/staker/block_validator.go +++ b/staker/block_validator.go @@ -614,7 +614,9 @@ func (v *BlockValidator) createNextValidationEntry(ctx context.Context) (bool, e if batchNum == v.nextCreateBatch.Number { return v.nextCreateBatch, nil } + // only batch-posting-reports will get here, and there's only one per batch if entry, found := v.prevBatchCache[batchNum]; found { + delete(v.prevBatchCache, batchNum) return entry, nil } found, entry, err := v.readBatch(ctx, batchNum) From 919003894b09dfacdcaa0ba55b589217b8b74fdd Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Wed, 11 Sep 2024 17:53:30 -0300 Subject: [PATCH 163/489] Test manual batch-poster fallback for DAS This test checks whether the batch-poster manual fallback for DAS works correctly. Here are the steps of the test: * Setup a L2 chain using a DAS * Sends a batch using the DAS * Shutdown the DAS * Fail to send a batch because the fallback is disabled * Enable the fallback * Verify the batch was sent with the fallback --- system_tests/das_test.go | 77 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/system_tests/das_test.go b/system_tests/das_test.go index 593058eaac..4f9bc8ab9f 100644 --- a/system_tests/das_test.go +++ b/system_tests/das_test.go @@ -322,3 +322,80 @@ func initTest(t *testing.T) { enableLogging(logLvl) } } + +func TestDASBatchPosterFallback(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Setup L1 + builder := NewNodeBuilder(ctx).DefaultConfig(t, true) + builder.chainConfig = params.ArbitrumDevTestDASChainConfig() + builder.BuildL1(t) + l1client := builder.L1.Client + l1info := builder.L1Info + + // Setup DAS server + dasDataDir := t.TempDir() + dasRpcServer, pubkey, backendConfig, _, restServerUrl := startLocalDASServer( + t, ctx, dasDataDir, l1client, builder.addresses.SequencerInbox) + authorizeDASKeyset(t, ctx, pubkey, l1info, l1client) + + // Setup sequence/batch-poster L2 node + builder.nodeConfig.DataAvailability.Enable = true + builder.nodeConfig.DataAvailability.RPCAggregator = aggConfigForBackend(backendConfig) + builder.nodeConfig.DataAvailability.RestAggregator = das.DefaultRestfulClientAggregatorConfig + builder.nodeConfig.DataAvailability.RestAggregator.Enable = true + builder.nodeConfig.DataAvailability.RestAggregator.Urls = []string{restServerUrl} + builder.nodeConfig.DataAvailability.ParentChainNodeURL = "none" + builder.nodeConfig.BatchPoster.DisableDapFallbackStoreDataOnChain = true // Disable DAS fallback + builder.nodeConfig.BatchPoster.ErrorDelay = time.Millisecond * 250 // Increase error delay because we expect errors + builder.L2Info = NewArbTestInfo(t, builder.chainConfig.ChainID) + builder.L2Info.GenerateAccount("User2") + cleanup := builder.BuildL2OnL1(t) + defer cleanup() + l2client := builder.L2.Client + l2info := builder.L2Info + + // Setup secondary L2 node + nodeConfigB := arbnode.ConfigDefaultL1NonSequencerTest() + nodeConfigB.BlockValidator.Enable = false + nodeConfigB.DataAvailability.Enable = true + nodeConfigB.DataAvailability.RestAggregator = das.DefaultRestfulClientAggregatorConfig + nodeConfigB.DataAvailability.RestAggregator.Enable = true + nodeConfigB.DataAvailability.RestAggregator.Urls = []string{restServerUrl} + nodeConfigB.DataAvailability.ParentChainNodeURL = "none" + nodeBParams := SecondNodeParams{ + nodeConfig: nodeConfigB, + initData: &l2info.ArbInitData, + } + l2B, cleanupB := builder.Build2ndNode(t, &nodeBParams) + defer cleanupB() + + // Check batch posting using the DAS + checkBatchPosting(t, ctx, l1client, l2client, l1info, l2info, big.NewInt(1e12), l2B.Client) + + // Shutdown the DAS + err := dasRpcServer.Shutdown(ctx) + Require(t, err) + + // Send 2nd transaction and check it doesn't arrive on second node + tx, _ := TransferBalanceTo(t, "Owner", l2info.GetAddress("User2"), big.NewInt(1e12), l2info, l2client, ctx) + _, err = WaitForTx(ctx, l2B.Client, tx.Hash(), time.Second*3) + if err == nil { + Fatal(t, "expected error but got nil") + } + + // Enable the DAP fallback and check the transaction on the second node. + // (We don't need to restart the node because of the hot-reload.) + builder.nodeConfig.BatchPoster.DisableDapFallbackStoreDataOnChain = false + _, err = WaitForTx(ctx, l2B.Client, tx.Hash(), time.Second*3) + Require(t, err) + l2balance, err := l2B.Client.BalanceAt(ctx, l2info.GetAddress("User2"), nil) + Require(t, err) + if l2balance.Cmp(big.NewInt(2e12)) != 0 { + Fatal(t, "Unexpected balance:", l2balance) + } + + // Send another transaction with fallback on + checkBatchPosting(t, ctx, l1client, l2client, l1info, l2info, big.NewInt(3e12), l2B.Client) +} From a32334f70e0aa89ed1010bafae4718035f86d677 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 11 Sep 2024 17:55:39 -0600 Subject: [PATCH 164/489] move forwarder to prover build step --- Makefile | 31 +- arbitrator/prover/Cargo.toml | 3 + .../forward/src/main.rs => prover/build.rs} | 76 +- arbitrator/prover/src/binary.rs | 12 +- arbitrator/prover/src/lib.rs | 5 +- arbitrator/prover/src/machine.rs | 44 +- arbitrator/prover/src/main.rs | 3 + arbitrator/prover/src/test.rs | 2 +- arbitrator/prover/test-cases/forward-test.wat | 32 - .../prover/test-cases/forward/forward.wat | 8 - .../prover/test-cases/forward/target.wat | 27 - arbitrator/stylus/src/test/mod.rs | 1 + arbitrator/tools/module_roots/src/main.rs | 1 + arbitrator/wasm-libraries/Cargo.lock | 668 +++++++++++++++++- arbitrator/wasm-libraries/Cargo.toml | 1 - arbitrator/wasm-libraries/forward/.gitignore | 1 - arbitrator/wasm-libraries/forward/Cargo.toml | 8 - arbitrator/wasm-testsuite/src/main.rs | 2 + validator/server_arb/machine.go | 2 +- 19 files changed, 735 insertions(+), 192 deletions(-) rename arbitrator/{wasm-libraries/forward/src/main.rs => prover/build.rs} (78%) delete mode 100644 arbitrator/prover/test-cases/forward-test.wat delete mode 100644 arbitrator/prover/test-cases/forward/forward.wat delete mode 100644 arbitrator/prover/test-cases/forward/target.wat delete mode 100644 arbitrator/wasm-libraries/forward/.gitignore delete mode 100644 arbitrator/wasm-libraries/forward/Cargo.toml diff --git a/Makefile b/Makefile index 0a71d64f12..9ecba1ce2d 100644 --- a/Makefile +++ b/Makefile @@ -60,7 +60,7 @@ replay_wasm=$(output_latest)/replay.wasm arb_brotli_files = $(wildcard arbitrator/brotli/src/*.* arbitrator/brotli/src/*/*.* arbitrator/brotli/*.toml arbitrator/brotli/*.rs) .make/cbrotli-lib .make/cbrotli-wasm arbitrator_generated_header=$(output_root)/include/arbitrator.h -arbitrator_wasm_libs=$(patsubst %, $(output_root)/machines/latest/%.wasm, forward wasi_stub host_io soft-float arbcompress user_host program_exec) +arbitrator_wasm_libs=$(patsubst %, $(output_root)/machines/latest/%.wasm, wasi_stub host_io soft-float arbcompress user_host program_exec) arbitrator_stylus_lib=$(output_root)/lib/libstylus.a prover_bin=$(output_root)/bin/prover arbitrator_jit=$(output_root)/bin/jit @@ -75,16 +75,12 @@ arbitrator_test_wasms=$(patsubst %.wat,%.wasm, $(arbitrator_tests_wat)) $(patsub arbitrator_tests_link_info = $(shell cat $(arbitrator_cases)/link.txt | xargs) arbitrator_tests_link_deps = $(patsubst %,$(arbitrator_cases)/%.wasm, $(arbitrator_tests_link_info)) -arbitrator_tests_forward_wats = $(wildcard $(arbitrator_cases)/forward/*.wat) -arbitrator_tests_forward_deps = $(arbitrator_tests_forward_wats:wat=wasm) - WASI_SYSROOT?=/opt/wasi-sdk/wasi-sysroot arbitrator_wasm_lib_flags=$(patsubst %, -l %, $(arbitrator_wasm_libs)) rust_arbutil_files = $(wildcard arbitrator/arbutil/src/*.* arbitrator/arbutil/src/*/*.* arbitrator/arbutil/*.toml arbitrator/caller-env/src/*.* arbitrator/caller-env/src/*/*.* arbitrator/caller-env/*.toml) .make/cbrotli-lib -prover_direct_includes = $(patsubst %,$(output_latest)/%.wasm, forward forward_stub) prover_dir = arbitrator/prover/ rust_prover_files = $(wildcard $(prover_dir)/src/*.* $(prover_dir)/src/*/*.* $(prover_dir)/*.toml $(prover_dir)/*.rs) $(rust_arbutil_files) $(prover_direct_includes) $(arb_brotli_files) @@ -92,12 +88,9 @@ wasm_lib = arbitrator/wasm-libraries wasm_lib_cargo = $(wasm_lib)/.cargo/config.toml wasm_lib_deps = $(wildcard $(wasm_lib)/$(1)/*.toml $(wasm_lib)/$(1)/src/*.rs $(wasm_lib)/$(1)/*.rs) $(wasm_lib_cargo) $(rust_arbutil_files) $(arb_brotli_files) .make/machines wasm_lib_go_abi = $(call wasm_lib_deps,go-abi) -wasm_lib_forward = $(call wasm_lib_deps,forward) wasm_lib_user_host_trait = $(call wasm_lib_deps,user-host-trait) wasm_lib_user_host = $(call wasm_lib_deps,user-host) $(wasm_lib_user_host_trait) -forward_dir = $(wasm_lib)/forward - stylus_files = $(wildcard $(stylus_dir)/*.toml $(stylus_dir)/src/*.rs) $(wasm_lib_user_host_trait) $(rust_prover_files) jit_dir = arbitrator/jit @@ -281,7 +274,6 @@ clean: rm -f arbitrator/wasm-libraries/soft-float/*.o rm -f arbitrator/wasm-libraries/soft-float/SoftFloat/build/Wasm-Clang/*.o rm -f arbitrator/wasm-libraries/soft-float/SoftFloat/build/Wasm-Clang/*.a - rm -f arbitrator/wasm-libraries/forward/*.wat rm -rf arbitrator/stylus/tests/*/target/ arbitrator/stylus/tests/*/*.wasm @rm -rf contracts/build contracts/cache solgen/go/ @rm -f .make/* @@ -399,7 +391,7 @@ $(output_latest)/host_io.wasm: $(DEP_PREDICATE) $(call wasm_lib_deps,host-io) $( cargo build --manifest-path arbitrator/wasm-libraries/Cargo.toml --release --target wasm32-wasi --config $(wasm_lib_cargo) --package host-io install arbitrator/wasm-libraries/$(wasm32_wasi)/host_io.wasm $@ -$(output_latest)/user_host.wasm: $(DEP_PREDICATE) $(wasm_lib_user_host) $(rust_prover_files) $(output_latest)/forward_stub.wasm .make/machines +$(output_latest)/user_host.wasm: $(DEP_PREDICATE) $(wasm_lib_user_host) $(rust_prover_files) .make/machines cargo build --manifest-path arbitrator/wasm-libraries/Cargo.toml --release --target wasm32-wasi --config $(wasm_lib_cargo) --package user-host install arbitrator/wasm-libraries/$(wasm32_wasi)/user_host.wasm $@ @@ -415,17 +407,9 @@ $(output_latest)/arbcompress.wasm: $(DEP_PREDICATE) $(call wasm_lib_deps,brotli) cargo build --manifest-path arbitrator/wasm-libraries/Cargo.toml --release --target wasm32-wasi --config $(wasm_lib_cargo) --package arbcompress install arbitrator/wasm-libraries/$(wasm32_wasi)/arbcompress.wasm $@ -$(output_latest)/forward.wasm: $(DEP_PREDICATE) $(wasm_lib_forward) .make/machines - cargo run --manifest-path $(forward_dir)/Cargo.toml -- --path $(forward_dir)/forward.wat - wat2wasm $(wasm_lib)/forward/forward.wat -o $@ - -$(output_latest)/forward_stub.wasm: $(DEP_PREDICATE) $(wasm_lib_forward) .make/machines - cargo run --manifest-path $(forward_dir)/Cargo.toml -- --path $(forward_dir)/forward_stub.wat --stub - wat2wasm $(wasm_lib)/forward/forward_stub.wat -o $@ - $(output_latest)/machine.wavm.br: $(DEP_PREDICATE) $(prover_bin) $(arbitrator_wasm_libs) $(replay_wasm) - $(prover_bin) $(replay_wasm) --generate-binaries $(output_latest) \ - $(patsubst %,-l $(output_latest)/%.wasm, forward soft-float wasi_stub host_io user_host arbcompress program_exec) + $(prover_bin) $(replay_wasm) --generate-binaries $(output_latest) --with-forwarder \ + $(patsubst %,-l $(output_latest)/%.wasm, soft-float wasi_stub host_io user_host arbcompress program_exec) $(arbitrator_cases)/%.wasm: $(arbitrator_cases)/%.wat wat2wasm $< -o $@ @@ -492,10 +476,10 @@ target/testdata/preimages.bin: python3 scripts/create-test-preimages.py $@ contracts/test/prover/proofs/rust-%.json: $(arbitrator_cases)/rust/$(wasm32_wasi)/%.wasm $(prover_bin) $(arbitrator_wasm_libs) target/testdata/preimages.bin - $(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --allow-hostapi --require-success --inbox-add-stub-headers --inbox $(arbitrator_cases)/rust/data/msg0.bin --inbox $(arbitrator_cases)/rust/data/msg1.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg0.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg1.bin --preimages target/testdata/preimages.bin + $(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --allow-hostapi --require-success --inbox-add-stub-headers --inbox $(arbitrator_cases)/rust/data/msg0.bin --inbox $(arbitrator_cases)/rust/data/msg1.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg0.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg1.bin --preimages target/testdata/preimages.bin --with-forwarder contracts/test/prover/proofs/go.json: $(arbitrator_cases)/go/testcase.wasm $(prover_bin) $(arbitrator_wasm_libs) target/testdata/preimages.bin $(arbitrator_tests_link_deps) $(arbitrator_cases)/user.wasm - $(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --require-success --preimages target/testdata/preimages.bin --stylus-modules $(arbitrator_cases)/user.wasm + $(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --require-success --preimages target/testdata/preimages.bin --stylus-modules $(arbitrator_cases)/user.wasm --with-forwarder # avoid testing user.wasm in onestepproofs. It can only run as stylus program. contracts/test/prover/proofs/user.json: @@ -508,9 +492,6 @@ contracts/test/prover/proofs/read-inboxmsg-10.json: contracts/test/prover/proofs/global-state.json: echo "[]" > $@ -contracts/test/prover/proofs/forward-test.json: $(arbitrator_cases)/forward-test.wasm $(arbitrator_tests_forward_deps) $(prover_bin) - $(prover_bin) $< -o $@ --allow-hostapi $(patsubst %,-l %, $(arbitrator_tests_forward_deps)) - contracts/test/prover/proofs/link.json: $(arbitrator_cases)/link.wasm $(arbitrator_tests_link_deps) $(prover_bin) $(prover_bin) $< -o $@ --allow-hostapi --stylus-modules $(arbitrator_tests_link_deps) --require-success diff --git a/arbitrator/prover/Cargo.toml b/arbitrator/prover/Cargo.toml index 5475647765..a4a69fcb05 100644 --- a/arbitrator/prover/Cargo.toml +++ b/arbitrator/prover/Cargo.toml @@ -47,6 +47,9 @@ enum-iterator = "2.0.1" criterion = { version = "0.5.0", features = ["html_reports"] } rand = "0.8.4" +[build-dependencies] +wasmer = { path = "../tools/wasmer/lib/api" } + [[bench]] name = "merkle_bench" harness = false diff --git a/arbitrator/wasm-libraries/forward/src/main.rs b/arbitrator/prover/build.rs similarity index 78% rename from arbitrator/wasm-libraries/forward/src/main.rs rename to arbitrator/prover/build.rs index 05a949e8aa..8191771c79 100644 --- a/arbitrator/wasm-libraries/forward/src/main.rs +++ b/arbitrator/prover/build.rs @@ -1,12 +1,8 @@ -// Copyright 2022-2024, Offchain Labs, Inc. -// For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE - -use eyre::Result; -use std::{fs::File, io::Write, path::PathBuf}; -use structopt::StructOpt; +use core::str; +use std::{env, fmt::Write, fs, path::Path}; /// order matters! -const HOSTIOS: [[&str; 3]; 42] = [ +pub const HOSTIOS: [[&str; 3]; 42] = [ ["read_args", "i32", ""], ["write_result", "i32 i32", ""], ["exit_early", "i32", ""], @@ -51,44 +47,16 @@ const HOSTIOS: [[&str; 3]; 42] = [ ["pay_for_memory_grow", "i32", ""], ]; -#[derive(StructOpt)] -#[structopt(name = "arbitrator-prover")] -struct Opts { - #[structopt(long)] - path: PathBuf, - #[structopt(long)] - stub: bool, -} - -fn main() -> Result<()> { - let opts = Opts::from_args(); - let file = &mut File::options() - .create(true) - .write(true) - .truncate(true) - .open(opts.path)?; - - match opts.stub { - true => forward_stub(file), - false => forward(file), - } -} - -fn forward(file: &mut File) -> Result<()> { +pub fn gen_forwarder(out_path: &Path) { + let mut wat = String::new(); macro_rules! wln { ($($text:tt)*) => { - writeln!(file, $($text)*)?; + writeln!(wat, $($text)*).unwrap(); }; } let s = " "; - wln!( - ";; Copyright 2022-2023, Offchain Labs, Inc.\n\ - ;; For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE\n\ - ;; This file is auto-generated.\n\ - \n\ - (module" - ); + wln!("(module"); macro_rules! group { ($list:expr, $kind:expr) => { @@ -102,9 +70,7 @@ fn forward(file: &mut File) -> Result<()> { for [name, ins, outs] in HOSTIOS { let params = group!(ins, "param"); let result = group!(outs, "result"); - wln!( - r#"{s}(import "user_host" "arbitrator_forward__{name}" (func ${name}{params}{result}))"# - ); + wln!(r#"{s}(import "user_host" "{name}" (func $_{name}{params}{result}))"#); } wln!(); @@ -139,20 +105,25 @@ fn forward(file: &mut File) -> Result<()> { } wln!( - "{s}{s}call ${name}\n\ + "{s}{s}call $_{name}\n\ {s}{s}call $check\n\ {s})" ); } wln!(")"); - Ok(()) + + let wasm = wasmer::wat2wasm(wat.as_bytes()).unwrap(); + + fs::write(out_path, wasm.as_ref()).unwrap(); } -fn forward_stub(file: &mut File) -> Result<()> { +pub fn gen_forwarder_stub(out_path: &Path) { + let mut wat = String::new(); + macro_rules! wln { ($($text:tt)*) => { - writeln!(file, $($text)*)?; + writeln!(wat, $($text)*).unwrap(); }; } let s = " "; @@ -202,5 +173,16 @@ fn forward_stub(file: &mut File) -> Result<()> { } wln!(")"); - Ok(()) + + let wasm = wasmer::wat2wasm(wat.as_bytes()).unwrap(); + + fs::write(out_path, wasm.as_ref()).unwrap(); +} + +fn main() { + let out_dir = env::var("OUT_DIR").unwrap(); + let forwarder_path = Path::new(&out_dir).join("forwarder.wasm"); + let forwarder_stub_path = Path::new(&out_dir).join("forwarder_stub.wasm"); + gen_forwarder(&forwarder_path); + gen_forwarder_stub(&forwarder_stub_path); } diff --git a/arbitrator/prover/src/binary.rs b/arbitrator/prover/src/binary.rs index aa5537476c..b1d819deb2 100644 --- a/arbitrator/prover/src/binary.rs +++ b/arbitrator/prover/src/binary.rs @@ -471,10 +471,14 @@ pub fn parse<'a>(input: &'a [u8], path: &'_ Path) -> Result> { let export = export.rsplit("__").take(1); exports.extend(export); } - for import in &binary.imports { - let name = import.name; - if exports.contains(name) { - bail!("binary exports an import with the same name {}", name.red()); + // forwarder is allowed to re-export the same name + // TODO: is there a real problem if import_name == export_name but module names don't? + if path != Path::new("forwarder") { + for import in &binary.imports { + let name = import.name; + if exports.contains(name) { + bail!("binary exports an import with the same name {}", name.red()); + } } } diff --git a/arbitrator/prover/src/lib.rs b/arbitrator/prover/src/lib.rs index 0f537478eb..20472691c1 100644 --- a/arbitrator/prover/src/lib.rs +++ b/arbitrator/prover/src/lib.rs @@ -70,9 +70,10 @@ pub unsafe extern "C" fn arbitrator_load_machine( library_paths: *const *const c_char, library_paths_size: isize, debug_chain: usize, + with_forwarder: bool, ) -> *mut Machine { let debug_chain = debug_chain != 0; - match arbitrator_load_machine_impl(binary_path, library_paths, library_paths_size, debug_chain) + match arbitrator_load_machine_impl(binary_path, library_paths, library_paths_size, debug_chain, with_forwarder) { Ok(mach) => mach, Err(err) => { @@ -87,6 +88,7 @@ unsafe fn arbitrator_load_machine_impl( library_paths: *const *const c_char, library_paths_size: isize, debug_chain: bool, + with_forwarder: bool, ) -> Result<*mut Machine> { let binary_path = cstr_to_string(binary_path); let binary_path = Path::new(&binary_path); @@ -107,6 +109,7 @@ unsafe fn arbitrator_load_machine_impl( Default::default(), Default::default(), get_empty_preimage_resolver(), + with_forwarder, )?; Ok(Box::into_raw(Box::new(mach))) } diff --git a/arbitrator/prover/src/machine.rs b/arbitrator/prover/src/machine.rs index 358876bd25..da3a4e3a79 100644 --- a/arbitrator/prover/src/machine.rs +++ b/arbitrator/prover/src/machine.rs @@ -334,7 +334,7 @@ lazy_static! { static ref USER_IMPORTS: HashMap = { let mut imports = HashMap::default(); - let forward = include_bytes!("../../../target/machines/latest/forward_stub.wasm"); + let forward = include_bytes!(concat!(env!("OUT_DIR"), "/forwarder_stub.wasm")); let forward = binary::parse(forward, Path::new("forward")).unwrap(); for (name, &(export, kind)) in &forward.exports { @@ -352,13 +352,12 @@ lazy_static! { } impl Module { - const FORWARDING_PREFIX: &'static str = "arbitrator_forward__"; - fn from_binary( bin: &WasmBinary, available_imports: &HashMap, floating_point_impls: &FloatingPointImpls, allow_hostapi: bool, + is_forwarder: bool, debug_funcs: bool, stylus_data: Option, ) -> Result { @@ -371,16 +370,11 @@ impl Module { for import in &bin.imports { let module = import.module; let have_ty = &bin.types[import.offset as usize]; - let (forward, import_name) = match import.name.strip_prefix(Module::FORWARDING_PREFIX) { - Some(name) => (true, name), - None => (false, import.name), - }; - let mut qualified_name = format!("{module}__{import_name}"); - qualified_name = qualified_name.replace(&['/', '.', '-'] as &[char], "_"); + let qualified_name = format!("{module}__{}", import.name); let func = if let Some(import) = available_imports.get(&qualified_name) { - let call = match forward { + let call = match is_forwarder { true => Opcode::CrossModuleForward, false => Opcode::CrossModuleCall, }; @@ -393,18 +387,18 @@ impl Module { Instruction::simple(Opcode::Return), ]; Function::new_from_wavm(wavm, import.ty.clone(), vec![]) - } else if let Ok((hostio, debug)) = host::get_impl(import.module, import_name) { + } else if let Ok((hostio, debug)) = host::get_impl(import.module, import.name) { ensure!( (debug && debug_funcs) || (!debug && allow_hostapi), "Host func {} in {} not enabled debug_funcs={debug_funcs} hostapi={allow_hostapi} debug={debug}", - import_name.red(), + import.name.red(), import.module.red(), ); hostio } else { bail!( "No such import {} in {} for {}", - import_name.red(), + import.name.red(), import.module.red(), bin_name.red() ) @@ -412,12 +406,12 @@ impl Module { ensure!( &func.ty == have_ty, "Import {} for {} has different function signature than export.\nexpected {} in {}\nbut have {}", - import_name.red(), bin_name.red(), func.ty.red(), module.red(), have_ty.red(), + import.name.red(), bin_name.red(), func.ty.red(), module.red(), have_ty.red(), ); func_type_idxs.push(import.offset); code.push(func); - host_call_hooks.push(Some((import.module.into(), import_name.into()))); + host_call_hooks.push(Some((import.module.into(), import.name.into()))); } func_type_idxs.extend(bin.functions.iter()); @@ -616,6 +610,7 @@ impl Module { &USER_IMPORTS, &HashMap::default(), false, + false, debug_funcs, stylus_data, ) @@ -1227,6 +1222,7 @@ impl Machine { global_state: GlobalState, inbox_contents: HashMap<(InboxIdentifier, u64), Vec>, preimage_resolver: PreimageResolver, + with_forwarder: bool, ) -> Result { let bin_source = file_bytes(binary_path)?; let bin = parse(&bin_source, binary_path) @@ -1252,6 +1248,7 @@ impl Machine { inbox_contents, preimage_resolver, None, + with_forwarder, ) } @@ -1281,6 +1278,7 @@ impl Machine { HashMap::default(), Arc::new(|_, _, _| panic!("tried to read preimage")), Some(stylus_data), + false, )?; let footprint: u32 = stylus_data.footprint.into(); @@ -1328,6 +1326,7 @@ impl Machine { inbox_contents: HashMap<(InboxIdentifier, u64), Vec>, preimage_resolver: PreimageResolver, stylus_data: Option, + with_forwarder: bool, ) -> Result { use ArbValueType::*; @@ -1337,6 +1336,17 @@ impl Machine { let mut floating_point_impls = HashMap::default(); let main_module_index = u32::try_from(modules.len() + libraries.len())?; + let forwarder_wasm = include_bytes!(concat!(env!("OUT_DIR"), "/forwarder.wasm")); + let mut libs_vec = vec![]; + + let libraries = if with_forwarder { + libs_vec.push(parse(forwarder_wasm.as_ref(), Path::new("forwarder"))?); + libs_vec.extend_from_slice(libraries); + &libs_vec + } else { + libraries + }; + // make the main module's exports available to libraries for (name, &(export, kind)) in &bin.exports { if kind == ExportKind::Func { @@ -1367,12 +1377,13 @@ impl Machine { } } - for lib in libraries { + for (index, lib) in libraries.iter().enumerate() { let module = Module::from_binary( lib, &available_imports, &floating_point_impls, true, + with_forwarder && (index == 0), debug_funcs, None, )?; @@ -1408,6 +1419,7 @@ impl Machine { &available_imports, &floating_point_impls, allow_hostapi_from_main, + false, debug_funcs, stylus_data, )?); diff --git a/arbitrator/prover/src/main.rs b/arbitrator/prover/src/main.rs index dba32e0e72..442b35992d 100644 --- a/arbitrator/prover/src/main.rs +++ b/arbitrator/prover/src/main.rs @@ -86,6 +86,8 @@ struct Opts { skip_until_host_io: bool, #[structopt(long)] max_steps: Option, + #[structopt(short, long)] + with_forwarder: bool, } fn file_with_stub_header(path: &Path, headerlength: usize) -> Result> { @@ -211,6 +213,7 @@ fn main() -> Result<()> { global_state, inbox_contents, preimage_resolver, + opts.with_forwarder, )?; for path in &opts.stylus_modules { diff --git a/arbitrator/prover/src/test.rs b/arbitrator/prover/src/test.rs index 97170441ff..52bf394dd5 100644 --- a/arbitrator/prover/src/test.rs +++ b/arbitrator/prover/src/test.rs @@ -57,7 +57,7 @@ pub fn reject_ambiguous_imports() { #[test] pub fn test_compress() -> Result<()> { - let data = include_bytes!("../../../target/machines/latest/forward_stub.wasm"); + let data = include_bytes!(concat!(env!("OUT_DIR"), "/forwarder_stub.wasm")); let mut last = vec![]; for dict in [Dictionary::Empty, Dictionary::StylusProgram] { diff --git a/arbitrator/prover/test-cases/forward-test.wat b/arbitrator/prover/test-cases/forward-test.wat deleted file mode 100644 index b9beff0d82..0000000000 --- a/arbitrator/prover/test-cases/forward-test.wat +++ /dev/null @@ -1,32 +0,0 @@ - -(module - (import "forward" "add" (func $add (param i32 i32) (result i32))) - (import "forward" "sub" (func $sub (param i32 i32) (result i32))) - (import "forward" "mul" (func $mul (param i32 i32) (result i32))) - (func $start - ;; this address will update each time a forwarded call is made - i32.const 0xa4b - i32.const 805 - i32.store - - i32.const 11 - i32.const 5 - call $sub - - i32.const 3 - i32.const -2 - call $mul - - call $add - (if - (then (unreachable))) - - ;; check that the address has changed - i32.const 0xa4b - i32.load - i32.const 808 - i32.ne - (if - (then (unreachable)))) - (start $start) - (memory 1)) diff --git a/arbitrator/prover/test-cases/forward/forward.wat b/arbitrator/prover/test-cases/forward/forward.wat deleted file mode 100644 index ff55953e62..0000000000 --- a/arbitrator/prover/test-cases/forward/forward.wat +++ /dev/null @@ -1,8 +0,0 @@ - -(module - (import "target" "arbitrator_forward__add" (func $add (param i32 i32) (result i32))) - (import "target" "arbitrator_forward__sub" (func $sub (param i32 i32) (result i32))) - (import "target" "arbitrator_forward__mul" (func $mul (param i32 i32) (result i32))) - (export "forward__add" (func $add)) - (export "forward__sub" (func $sub)) - (export "forward__mul" (func $mul))) diff --git a/arbitrator/prover/test-cases/forward/target.wat b/arbitrator/prover/test-cases/forward/target.wat deleted file mode 100644 index 0779eb753d..0000000000 --- a/arbitrator/prover/test-cases/forward/target.wat +++ /dev/null @@ -1,27 +0,0 @@ - -(module - (import "env" "wavm_caller_load8" (func $load (param i32) (result i32))) - (import "env" "wavm_caller_store8" (func $store (param i32 i32))) - (func (export "target__add") (param i32 i32) (result i32) - call $write_caller - local.get 0 - local.get 1 - i32.add) - (func (export "target__sub") (param i32 i32) (result i32) - call $write_caller - local.get 0 - local.get 1 - i32.sub) - (func (export "target__mul") (param i32 i32) (result i32) - call $write_caller - local.get 0 - local.get 1 - i32.mul) - (func $write_caller (export "write_caller") - ;; increment the value at address 0xa4b in the caller - i32.const 0xa4b - i32.const 0xa4b - call $load - i32.const 1 - i32.add - call $store)) diff --git a/arbitrator/stylus/src/test/mod.rs b/arbitrator/stylus/src/test/mod.rs index 00c9c62ae4..42c9446564 100644 --- a/arbitrator/stylus/src/test/mod.rs +++ b/arbitrator/stylus/src/test/mod.rs @@ -164,6 +164,7 @@ fn new_test_machine(path: &str, compile: &CompileConfig) -> Result { HashMap::default(), Arc::new(|_, _, _| panic!("tried to read preimage")), Some(stylus_data), + false, )?; mach.set_ink(u64::MAX); mach.set_stack(u32::MAX); diff --git a/arbitrator/tools/module_roots/src/main.rs b/arbitrator/tools/module_roots/src/main.rs index 32e4764847..fc22bed5fa 100644 --- a/arbitrator/tools/module_roots/src/main.rs +++ b/arbitrator/tools/module_roots/src/main.rs @@ -39,6 +39,7 @@ fn main() -> Result<()> { GlobalState::default(), HashMap::default(), Arc::new(|_, _, _| panic!("tried to read preimage")), + false, )?; let mut stylus = vec![]; diff --git a/arbitrator/wasm-libraries/Cargo.lock b/arbitrator/wasm-libraries/Cargo.lock index 7620ff538b..b234424f69 100644 --- a/arbitrator/wasm-libraries/Cargo.lock +++ b/arbitrator/wasm-libraries/Cargo.lock @@ -2,6 +2,21 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" +dependencies = [ + "gimli 0.29.0", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "ahash" version = "0.7.8" @@ -91,6 +106,21 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "backtrace" +version = "0.3.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" +dependencies = [ + "addr2line", + "cc", + "cfg-if 1.0.0", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "bincode" version = "1.3.3" @@ -187,6 +217,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.7.0" @@ -203,6 +239,15 @@ dependencies = [ "rand_pcg", ] +[[package]] +name = "cc" +version = "1.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57b6a275aa2903740dc87da01c62040406b8812552e97129a63ea8850a17c6e6" +dependencies = [ + "shlex", +] + [[package]] name = "cfg-if" version = "0.1.10" @@ -236,6 +281,19 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +[[package]] +name = "corosensei" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80128832c58ea9cbd041d2a759ec449224487b2c1e400453d99d244eead87a8e" +dependencies = [ + "autocfg", + "cfg-if 1.0.0", + "libc", + "scopeguard", + "windows-sys 0.33.0", +] + [[package]] name = "cpufeatures" version = "0.2.12" @@ -245,6 +303,123 @@ dependencies = [ "libc", ] +[[package]] +name = "cranelift-bforest" +version = "0.91.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2ab4512dfd3a6f4be184403a195f76e81a8a9f9e6c898e19d2dc3ce20e0115" +dependencies = [ + "cranelift-entity", +] + +[[package]] +name = "cranelift-codegen" +version = "0.91.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98b022ed2a5913a38839dfbafe6cf135342661293b08049843362df4301261dc" +dependencies = [ + "arrayvec", + "bumpalo", + "cranelift-bforest", + "cranelift-codegen-meta", + "cranelift-codegen-shared", + "cranelift-egraph", + "cranelift-entity", + "cranelift-isle", + "gimli 0.26.2", + "log", + "regalloc2", + "smallvec", + "target-lexicon", +] + +[[package]] +name = "cranelift-codegen-meta" +version = "0.91.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "639307b45434ad112a98f8300c0f0ab085cbefcd767efcdef9ef19d4c0756e74" +dependencies = [ + "cranelift-codegen-shared", +] + +[[package]] +name = "cranelift-codegen-shared" +version = "0.91.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "278e52e29c53fcf32431ef08406c295699a70306d05a0715c5b1bf50e33a9ab7" + +[[package]] +name = "cranelift-egraph" +version = "0.91.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624b54323b06e675293939311943ba82d323bb340468ce1889be5da7932c8d73" +dependencies = [ + "cranelift-entity", + "fxhash", + "hashbrown 0.12.3", + "indexmap 1.9.3", + "log", + "smallvec", +] + +[[package]] +name = "cranelift-entity" +version = "0.91.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a59bcbca89c3f1b70b93ab3cbba5e5e0cbf3e63dadb23c7525cb142e21a9d4c" + +[[package]] +name = "cranelift-frontend" +version = "0.91.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d70abacb8cfef3dc8ff7e8836e9c1d70f7967dfdac824a4cd5e30223415aca6" +dependencies = [ + "cranelift-codegen", + "log", + "smallvec", + "target-lexicon", +] + +[[package]] +name = "cranelift-isle" +version = "0.91.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "393bc73c451830ff8dbb3a07f61843d6cb41a084f9996319917c0b291ed785bb" + +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + [[package]] name = "crunchy" version = "0.2.2" @@ -330,6 +505,19 @@ dependencies = [ "syn 2.0.72", ] +[[package]] +name = "dashmap" +version = "5.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" +dependencies = [ + "cfg-if 1.0.0", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "derivative" version = "2.2.0" @@ -456,26 +644,33 @@ dependencies = [ "once_cell", ] +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "forward" -version = "0.1.0" -dependencies = [ - "eyre", - "structopt", -] - [[package]] name = "funty" version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -497,6 +692,23 @@ dependencies = [ "wasi", ] +[[package]] +name = "gimli" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d" +dependencies = [ + "fallible-iterator", + "indexmap 1.9.3", + "stable_deref_trait", +] + +[[package]] +name = "gimli" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" + [[package]] name = "hashbrown" version = "0.12.3" @@ -595,6 +807,15 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +[[package]] +name = "js-sys" +version = "0.3.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "keccak" version = "0.1.5" @@ -632,6 +853,12 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + [[package]] name = "lru" version = "0.12.4" @@ -641,12 +868,57 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "mach" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +dependencies = [ + "libc", +] + +[[package]] +name = "mach2" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709" +dependencies = [ + "libc", +] + [[package]] name = "memchr" version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "memmap2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" +dependencies = [ + "libc", +] + +[[package]] +name = "memmap2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d28bba84adfe6646737845bc5ebbfa2c08424eb1c37e94a1fd2a82adb56a872" +dependencies = [ + "libc", +] + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + [[package]] name = "memory_units" version = "0.4.0" @@ -659,6 +931,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" +dependencies = [ + "adler", +] + [[package]] name = "more-asserts" version = "0.2.2" @@ -791,6 +1072,15 @@ dependencies = [ "syn 2.0.72", ] +[[package]] +name = "object" +version = "0.36.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.19.0" @@ -832,6 +1122,12 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pin-project-lite" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" + [[package]] name = "proc-macro-crate" version = "3.1.0" @@ -912,6 +1208,7 @@ dependencies = [ "smallvec", "static_assertions", "structopt", + "wasmer", "wasmer-types", "wasmparser", "wat", @@ -976,6 +1273,26 @@ dependencies = [ "rand_core", ] +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.3" @@ -985,6 +1302,30 @@ dependencies = [ "bitflags 2.6.0", ] +[[package]] +name = "regalloc2" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300d4fbfb40c1c66a78ba3ddd41c1110247cf52f97b87d0f2fc9209bd49b030c" +dependencies = [ + "fxhash", + "log", + "slice-group-by", + "smallvec", +] + +[[package]] +name = "region" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6b6ebd13bc009aef9cd476c1310d49ac354d36e240cf1bd753290f3dc7199a7" +dependencies = [ + "bitflags 1.3.2", + "libc", + "mach2", + "windows-sys 0.52.0", +] + [[package]] name = "rend" version = "0.4.2" @@ -1075,6 +1416,12 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" +[[package]] +name = "self_cell" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d369a96f978623eb3dc28807c4852d6cc617fed53da5d3c400feff1ef34a714a" + [[package]] name = "semver" version = "1.0.23" @@ -1090,6 +1437,17 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-wasm-bindgen" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b4c031cd0d9014307d82b8abf653c0290fbdaeb4c02d00c63cf52f728628bf" +dependencies = [ + "js-sys", + "serde", + "wasm-bindgen", +] + [[package]] name = "serde_derive" version = "1.0.204" @@ -1181,6 +1539,22 @@ dependencies = [ "keccak", ] +[[package]] +name = "shared-buffer" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6c99835bad52957e7aa241d3975ed17c1e5f8c92026377d117a606f36b84b16" +dependencies = [ + "bytes", + "memmap2 0.6.2", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "simdutf8" version = "0.1.4" @@ -1193,6 +1567,12 @@ version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" +[[package]] +name = "slice-group-by" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "826167069c09b99d56f31e9ae5c99049e932a98c9dc2dac47645b08dbbf76ba7" + [[package]] name = "smallvec" version = "1.13.2" @@ -1202,6 +1582,12 @@ dependencies = [ "serde", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "static_assertions" version = "1.1.0" @@ -1348,6 +1734,37 @@ dependencies = [ "winnow", ] +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + [[package]] name = "typenum" version = "1.17.0" @@ -1445,13 +1862,147 @@ dependencies = [ "wee_alloc", ] +[[package]] +name = "wasm-bindgen" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +dependencies = [ + "cfg-if 1.0.0", + "once_cell", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.72", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" + [[package]] name = "wasm-encoder" -version = "0.215.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb56df3e06b8e6b77e37d2969a50ba51281029a9aeb3855e76b7f49b6418847" +checksum = "1ba64e81215916eaeb48fee292f29401d69235d62d8b8fd92a7b2844ec5ae5f7" +dependencies = [ + "leb128", +] + +[[package]] +name = "wasmer" +version = "4.2.8" dependencies = [ + "bytes", + "cfg-if 1.0.0", + "derivative", + "indexmap 1.9.3", + "js-sys", + "more-asserts", + "rustc-demangle", + "serde", + "serde-wasm-bindgen", + "shared-buffer", + "target-lexicon", + "thiserror", + "tracing", + "wasm-bindgen", + "wasmer-compiler", + "wasmer-compiler-cranelift", + "wasmer-derive", + "wasmer-types", + "wasmer-vm", + "wat", + "winapi", +] + +[[package]] +name = "wasmer-compiler" +version = "4.2.8" +dependencies = [ + "backtrace", + "bytes", + "cfg-if 1.0.0", + "enum-iterator 0.7.0", + "enumset", + "lazy_static", "leb128", + "memmap2 0.5.10", + "more-asserts", + "region", + "rkyv", + "self_cell", + "shared-buffer", + "smallvec", + "thiserror", + "wasmer-types", + "wasmer-vm", + "wasmparser", + "winapi", +] + +[[package]] +name = "wasmer-compiler-cranelift" +version = "4.2.8" +dependencies = [ + "cranelift-codegen", + "cranelift-entity", + "cranelift-frontend", + "gimli 0.26.2", + "more-asserts", + "rayon", + "smallvec", + "target-lexicon", + "tracing", + "wasmer-compiler", + "wasmer-types", +] + +[[package]] +name = "wasmer-derive" +version = "4.2.8" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", ] [[package]] @@ -1468,6 +2019,32 @@ dependencies = [ "thiserror", ] +[[package]] +name = "wasmer-vm" +version = "4.2.8" +dependencies = [ + "backtrace", + "cc", + "cfg-if 1.0.0", + "corosensei", + "crossbeam-queue", + "dashmap", + "derivative", + "enum-iterator 0.7.0", + "fnv", + "indexmap 1.9.3", + "lazy_static", + "libc", + "mach", + "memoffset", + "more-asserts", + "region", + "scopeguard", + "thiserror", + "wasmer-types", + "winapi", +] + [[package]] name = "wasmparser" version = "0.121.2" @@ -1481,11 +2058,10 @@ dependencies = [ [[package]] name = "wast" -version = "215.0.0" +version = "64.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ff1d00d893593249e60720be04a7c1f42f1c4dc3806a2869f4e66ab61eb54cb" +checksum = "a259b226fd6910225aa7baeba82f9d9933b6d00f2ce1b49b80fa4214328237cc" dependencies = [ - "bumpalo", "leb128", "memchr", "unicode-width", @@ -1494,9 +2070,9 @@ dependencies = [ [[package]] name = "wat" -version = "1.215.0" +version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "670bf4d9c8cf76ae242d70ded47c546525b6dafaa6871f9bcb065344bf2b4e3d" +checksum = "53253d920ab413fca1c7dc2161d601c79b4fdf631d0ba51dd4343bf9b556c3f6" dependencies = [ "wast", ] @@ -1535,6 +2111,28 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-sys" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43dbb096663629518eb1dfa72d80243ca5a6aca764cae62a2df70af760a9be75" +dependencies = [ + "windows_aarch64_msvc 0.33.0", + "windows_i686_gnu 0.33.0", + "windows_i686_msvc 0.33.0", + "windows_x86_64_gnu 0.33.0", + "windows_x86_64_msvc 0.33.0", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -1542,13 +2140,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_x86_64_msvc 0.52.6", ] [[package]] @@ -1557,12 +2155,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_msvc" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd761fd3eb9ab8cc1ed81e56e567f02dd82c4c837e48ac3b2181b9ffc5060807" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_i686_gnu" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab0cf703a96bab2dc0c02c0fa748491294bf9b7feb27e1f4f96340f208ada0e" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -1575,12 +2185,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_msvc" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cfdbe89cc9ad7ce618ba34abc34bbb6c36d99e96cae2245b7943cd75ee773d0" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_x86_64_gnu" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4dd9b0c0e9ece7bb22e84d70d01b71c6d6248b81a3c60d11869451b4cb24784" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -1593,6 +2215,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_msvc" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff1e4aa646495048ec7f3ffddc411e1d829c026a2ec62b39da15c1055e406eaa" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" diff --git a/arbitrator/wasm-libraries/Cargo.toml b/arbitrator/wasm-libraries/Cargo.toml index 837df8f4da..2fd585ee7d 100644 --- a/arbitrator/wasm-libraries/Cargo.toml +++ b/arbitrator/wasm-libraries/Cargo.toml @@ -7,6 +7,5 @@ members = [ "user-host-trait", "user-test", "program-exec", - "forward", ] resolver = "2" diff --git a/arbitrator/wasm-libraries/forward/.gitignore b/arbitrator/wasm-libraries/forward/.gitignore deleted file mode 100644 index 40da2042b7..0000000000 --- a/arbitrator/wasm-libraries/forward/.gitignore +++ /dev/null @@ -1 +0,0 @@ -**.wat diff --git a/arbitrator/wasm-libraries/forward/Cargo.toml b/arbitrator/wasm-libraries/forward/Cargo.toml deleted file mode 100644 index 73ed9d8827..0000000000 --- a/arbitrator/wasm-libraries/forward/Cargo.toml +++ /dev/null @@ -1,8 +0,0 @@ -[package] -name = "forward" -version = "0.1.0" -edition = "2021" - -[dependencies] -eyre = "0.6.5" -structopt = "0.3.26" diff --git a/arbitrator/wasm-testsuite/src/main.rs b/arbitrator/wasm-testsuite/src/main.rs index 2144dcf992..cb861e3a35 100644 --- a/arbitrator/wasm-testsuite/src/main.rs +++ b/arbitrator/wasm-testsuite/src/main.rs @@ -343,6 +343,7 @@ fn main() -> eyre::Result<()> { GlobalState::default(), HashMap::default(), machine::get_empty_preimage_resolver(), + false, ); if let Err(error) = &mech { @@ -429,6 +430,7 @@ fn main() -> eyre::Result<()> { GlobalState::default(), HashMap::default(), machine::get_empty_preimage_resolver(), + false, ) .expect_err(&format!("failed to reject invalid module {}", filename)); } diff --git a/validator/server_arb/machine.go b/validator/server_arb/machine.go index adca9695e2..e77e071d70 100644 --- a/validator/server_arb/machine.go +++ b/validator/server_arb/machine.go @@ -92,7 +92,7 @@ func LoadSimpleMachine(wasm string, libraries []string, debugChain bool) (*Arbit cWasm := C.CString(wasm) cLibraries := CreateCStringList(libraries) debug := usize(arbmath.BoolToUint32(debugChain)) - mach := C.arbitrator_load_machine(cWasm, cLibraries, C.long(len(libraries)), debug) + mach := C.arbitrator_load_machine(cWasm, cLibraries, C.long(len(libraries)), debug, false) C.free(unsafe.Pointer(cWasm)) FreeCStringList(cLibraries, len(libraries)) if mach == nil { From ed18d13032542989318a664c3c85d1c33756eca7 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Thu, 5 Sep 2024 19:35:58 -0600 Subject: [PATCH 165/489] remove forwarder_stub --- arbitrator/arbutil/src/hostios.rs | 64 +++++++++++++++ arbitrator/arbutil/src/lib.rs | 1 + arbitrator/prover/Cargo.toml | 1 + arbitrator/prover/build.rs | 131 ++++-------------------------- arbitrator/prover/src/machine.rs | 118 ++++++++++++--------------- arbitrator/prover/src/test.rs | 2 +- arbitrator/prover/src/value.rs | 12 ++- 7 files changed, 145 insertions(+), 184 deletions(-) create mode 100644 arbitrator/arbutil/src/hostios.rs diff --git a/arbitrator/arbutil/src/hostios.rs b/arbitrator/arbutil/src/hostios.rs new file mode 100644 index 0000000000..05bcb94f73 --- /dev/null +++ b/arbitrator/arbutil/src/hostios.rs @@ -0,0 +1,64 @@ +use std::fmt::Display; + +pub enum ParamType { + I32, + I64, +} + +impl Display for ParamType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + use ParamType::*; + match self { + I32 => write!(f, "i32"), + I64 => write!(f, "i64"), + } + } +} + +use ParamType::*; + +/// order matters! +pub const HOSTIOS: [(&str, &[ParamType], &[ParamType]); 42] = [ + ("read_args", &[I32], &[]), + ("write_result", &[I32, I32], &[]), + ("exit_early", &[I32], &[]), + ("storage_load_bytes32", &[I32, I32], &[]), + ("storage_cache_bytes32", &[I32, I32], &[]), + ("storage_flush_cache", &[I32], &[]), + ("transient_load_bytes32", &[I32, I32], &[]), + ("transient_store_bytes32", &[I32, I32], &[]), + ("call_contract", &[I32, I32, I32, I32, I64, I32], &[I32]), + ("delegate_call_contract", &[I32, I32, I32, I64, I32], &[I32]), + ("static_call_contract", &[I32, I32, I32, I64, I32], &[I32]), + ("create1", &[I32, I32, I32, I32, I32], &[]), + ("create2", &[I32, I32, I32, I32, I32, I32], &[]), + ("read_return_data", &[I32, I32, I32], &[I32]), + ("return_data_size", &[], &[I32]), + ("emit_log", &[I32, I32, I32], &[]), + ("account_balance", &[I32, I32], &[]), + ("account_code", &[I32, I32, I32, I32], &[I32]), + ("account_code_size", &[I32], &[I32]), + ("account_codehash", &[I32, I32], &[]), + ("evm_gas_left", &[], &[I64]), + ("evm_ink_left", &[], &[I64]), + ("block_basefee", &[I32], &[]), + ("chainid", &[], &[I64]), + ("block_coinbase", &[I32], &[]), + ("block_gas_limit", &[], &[I64]), + ("block_number", &[], &[I64]), + ("block_timestamp", &[], &[I64]), + ("contract_address", &[I32], &[]), + ("math_div", &[I32, I32], &[]), + ("math_mod", &[I32, I32], &[]), + ("math_pow", &[I32, I32], &[]), + ("math_add_mod", &[I32, I32, I32], &[]), + ("math_mul_mod", &[I32, I32, I32], &[]), + ("msg_reentrant", &[], &[I32]), + ("msg_sender", &[I32], &[]), + ("msg_value", &[I32], &[]), + ("native_keccak256", &[I32, I32, I32], &[]), + ("tx_gas_price", &[I32], &[]), + ("tx_ink_price", &[], &[I32]), + ("tx_origin", &[I32], &[]), + ("pay_for_memory_grow", &[I32], &[]), +]; diff --git a/arbitrator/arbutil/src/lib.rs b/arbitrator/arbutil/src/lib.rs index 9c48a9fefc..8d8c1d0fca 100644 --- a/arbitrator/arbutil/src/lib.rs +++ b/arbitrator/arbutil/src/lib.rs @@ -6,6 +6,7 @@ pub mod color; pub mod crypto; pub mod evm; pub mod format; +pub mod hostios; pub mod math; pub mod operator; pub mod pricing; diff --git a/arbitrator/prover/Cargo.toml b/arbitrator/prover/Cargo.toml index a4a69fcb05..a634dba19e 100644 --- a/arbitrator/prover/Cargo.toml +++ b/arbitrator/prover/Cargo.toml @@ -49,6 +49,7 @@ rand = "0.8.4" [build-dependencies] wasmer = { path = "../tools/wasmer/lib/api" } +arbutil = { path = "../arbutil/" } [[bench]] name = "merkle_bench" diff --git a/arbitrator/prover/build.rs b/arbitrator/prover/build.rs index 8191771c79..db0560b2c5 100644 --- a/arbitrator/prover/build.rs +++ b/arbitrator/prover/build.rs @@ -1,52 +1,6 @@ -use core::str; +use arbutil::hostios::HOSTIOS; use std::{env, fmt::Write, fs, path::Path}; -/// order matters! -pub const HOSTIOS: [[&str; 3]; 42] = [ - ["read_args", "i32", ""], - ["write_result", "i32 i32", ""], - ["exit_early", "i32", ""], - ["storage_load_bytes32", "i32 i32", ""], - ["storage_cache_bytes32", "i32 i32", ""], - ["storage_flush_cache", "i32", ""], - ["transient_load_bytes32", "i32 i32", ""], - ["transient_store_bytes32", "i32 i32", ""], - ["call_contract", "i32 i32 i32 i32 i64 i32", "i32"], - ["delegate_call_contract", "i32 i32 i32 i64 i32", "i32"], - ["static_call_contract", "i32 i32 i32 i64 i32", "i32"], - ["create1", "i32 i32 i32 i32 i32", ""], - ["create2", "i32 i32 i32 i32 i32 i32", ""], - ["read_return_data", "i32 i32 i32", "i32"], - ["return_data_size", "", "i32"], - ["emit_log", "i32 i32 i32", ""], - ["account_balance", "i32 i32", ""], - ["account_code", "i32 i32 i32 i32", "i32"], - ["account_code_size", "i32", "i32"], - ["account_codehash", "i32 i32", ""], - ["evm_gas_left", "", "i64"], - ["evm_ink_left", "", "i64"], - ["block_basefee", "i32", ""], - ["chainid", "", "i64"], - ["block_coinbase", "i32", ""], - ["block_gas_limit", "", "i64"], - ["block_number", "", "i64"], - ["block_timestamp", "", "i64"], - ["contract_address", "i32", ""], - ["math_div", "i32 i32", ""], - ["math_mod", "i32 i32", ""], - ["math_pow", "i32 i32", ""], - ["math_add_mod", "i32 i32 i32", ""], - ["math_mul_mod", "i32 i32 i32", ""], - ["msg_reentrant", "", "i32"], - ["msg_sender", "i32", ""], - ["msg_value", "i32", ""], - ["native_keccak256", "i32 i32 i32", ""], - ["tx_gas_price", "i32", ""], - ["tx_ink_price", "", "i32"], - ["tx_origin", "i32", ""], - ["pay_for_memory_grow", "i32", ""], -]; - pub fn gen_forwarder(out_path: &Path) { let mut wat = String::new(); macro_rules! wln { @@ -61,13 +15,23 @@ pub fn gen_forwarder(out_path: &Path) { macro_rules! group { ($list:expr, $kind:expr) => { (!$list.is_empty()) - .then(|| format!(" ({} {})", $kind, $list)) + .then(|| { + format!( + " ({} {})", + $kind, + $list + .iter() + .map(|x| x.to_string()) + .collect::>() + .join(" ") + ) + }) .unwrap_or_default() }; } wln!("{s};; symbols to re-export"); - for [name, ins, outs] in HOSTIOS { + for (name, ins, outs) in HOSTIOS { let params = group!(ins, "param"); let result = group!(outs, "result"); wln!(r#"{s}(import "user_host" "{name}" (func $_{name}{params}{result}))"#); @@ -94,13 +58,12 @@ pub fn gen_forwarder(out_path: &Path) { ); wln!("{s};; user linkage"); - for [name, ins, outs] in HOSTIOS { + for (name, ins, outs) in HOSTIOS { let params = group!(ins, "param"); let result = group!(outs, "result"); wln!("{s}(func (export \"vm_hooks__{name}\"){params}{result}"); - let gets = (1 + ins.len()) / 4; - for i in 0..gets { + for i in 0..ins.len() { wln!("{s}{s}local.get {i}"); } @@ -112,67 +75,7 @@ pub fn gen_forwarder(out_path: &Path) { } wln!(")"); - - let wasm = wasmer::wat2wasm(wat.as_bytes()).unwrap(); - - fs::write(out_path, wasm.as_ref()).unwrap(); -} - -pub fn gen_forwarder_stub(out_path: &Path) { - let mut wat = String::new(); - - macro_rules! wln { - ($($text:tt)*) => { - writeln!(wat, $($text)*).unwrap(); - }; - } - let s = " "; - - wln!( - ";; Copyright 2022-2023, Offchain Labs, Inc.\n\ - ;; For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE\n\ - ;; This file is auto-generated.\n\ - \n\ - (module" - ); - - macro_rules! group { - ($list:expr, $kind:expr) => { - (!$list.is_empty()) - .then(|| format!(" ({} {})", $kind, $list)) - .unwrap_or_default() - }; - } - - wln!("{s};; stubs for the symbols we re-export"); - for [name, ins, outs] in HOSTIOS { - let params = group!(ins, "param"); - let result = group!(outs, "result"); - wln!("{s}(func ${name}{params}{result} unreachable)"); - } - wln!(); - - wln!("{s};; reserved offsets for future user_host imports"); - for i in HOSTIOS.len()..512 { - wln!("{s}(func $reserved_{i} unreachable)"); - } - wln!(); - - wln!( - "{s};; allows user_host to request a trap\n\ - {s}(global $trap (mut i32) (i32.const 0))\n\ - {s}(func $check unreachable)\n\ - {s}(func (export \"forward__set_trap\") unreachable)" - ); - - wln!("{s};; user linkage"); - for [name, ins, outs] in HOSTIOS { - let params = group!(ins, "param"); - let result = group!(outs, "result"); - wln!("{s}(func (export \"vm_hooks__{name}\"){params}{result} unreachable)"); - } - - wln!(")"); + eprintln!("{}", &wat); let wasm = wasmer::wat2wasm(wat.as_bytes()).unwrap(); @@ -182,7 +85,5 @@ pub fn gen_forwarder_stub(out_path: &Path) { fn main() { let out_dir = env::var("OUT_DIR").unwrap(); let forwarder_path = Path::new(&out_dir).join("forwarder.wasm"); - let forwarder_stub_path = Path::new(&out_dir).join("forwarder_stub.wasm"); gen_forwarder(&forwarder_path); - gen_forwarder_stub(&forwarder_stub_path); } diff --git a/arbitrator/prover/src/machine.rs b/arbitrator/prover/src/machine.rs index da3a4e3a79..c045fc72c9 100644 --- a/arbitrator/prover/src/machine.rs +++ b/arbitrator/prover/src/machine.rs @@ -19,7 +19,7 @@ use crate::{ IBinOpType, IRelOpType, IUnOpType, Instruction, Opcode, }, }; -use arbutil::{crypto, math, Bytes32, Color, DebugColor, PreimageType}; +use arbutil::{crypto, hostios::HOSTIOS, math, Bytes32, Color, DebugColor, PreimageType}; use brotli::Dictionary; #[cfg(feature = "native")] use c_kzg::BYTES_PER_BLOB; @@ -330,22 +330,17 @@ pub struct Module { pub(crate) extra_hash: Arc, } +#[cfg(feature = "native")] +static FORWARDER_WASM: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/forwarder.wasm")); + lazy_static! { static ref USER_IMPORTS: HashMap = { let mut imports = HashMap::default(); - let forward = include_bytes!(concat!(env!("OUT_DIR"), "/forwarder_stub.wasm")); - let forward = binary::parse(forward, Path::new("forward")).unwrap(); - - for (name, &(export, kind)) in &forward.exports { - if kind == ExportKind::Func { - let ty = match forward.get_function(FunctionIndex::from_u32(export)) { - Ok(ty) => ty, - Err(error) => panic!("failed to read export {name}: {error:?}"), - }; - let import = AvailableImport::new(ty, 1, export); - imports.insert(name.to_owned(), import); - } + // 0-513 are internal + for (index, (name, ins, outs)) in HOSTIOS.iter().enumerate() { + let import = AvailableImport::new(FunctionType::new(ins.iter().map(|x|x.into()).collect::>(), outs.iter().map(|x|x.into()).collect::>()), 1, (index + 514).try_into().unwrap()); + imports.insert(format!("vm_hooks__{name}"), import); } imports }; @@ -1334,36 +1329,19 @@ impl Machine { let mut modules = vec![Module::default()]; let mut available_imports = HashMap::default(); let mut floating_point_impls = HashMap::default(); - let main_module_index = u32::try_from(modules.len() + libraries.len())?; - let forwarder_wasm = include_bytes!(concat!(env!("OUT_DIR"), "/forwarder.wasm")); - let mut libs_vec = vec![]; + let forwarder = if with_forwarder { + #[cfg(not(feature = "native"))] + bail!("forwarder not supported without native"); - let libraries = if with_forwarder { - libs_vec.push(parse(forwarder_wasm.as_ref(), Path::new("forwarder"))?); - libs_vec.extend_from_slice(libraries); - &libs_vec + #[cfg(feature = "native")] + Some(parse(FORWARDER_WASM, Path::new("forwarder"))?) } else { - libraries + None }; - // make the main module's exports available to libraries - for (name, &(export, kind)) in &bin.exports { - if kind == ExportKind::Func { - let index: usize = export.try_into()?; - if let Some(index) = index.checked_sub(bin.imports.len()) { - let ty: usize = bin.functions[index].try_into()?; - let ty = bin.types[ty].clone(); - available_imports.insert( - format!("env__wavm_guest_call__{name}"), - AvailableImport::new(ty, main_module_index, export), - ); - } - } - } - // collect all the library exports in advance so they can use each other's - for (index, lib) in libraries.iter().enumerate() { + for (index, lib) in forwarder.iter().chain(libraries.iter()).enumerate() { let module = 1 + index as u32; // off by one due to the entry point for (name, &(export, kind)) in &lib.exports { if kind == ExportKind::Func { @@ -1371,49 +1349,55 @@ impl Machine { Ok(ty) => ty, Err(error) => bail!("failed to read export {name}: {error}"), }; - let import = AvailableImport::new(ty, module, export); + let import = AvailableImport::new(ty.clone(), module, export); available_imports.insert(name.to_owned(), import); + if let Ok(op) = name.parse::() { + let mut sig = op.signature(); + // wavm codegen takes care of effecting this type change at callsites + for ty in sig.inputs.iter_mut().chain(sig.outputs.iter_mut()) { + if *ty == F32 { + *ty = I32; + } else if *ty == F64 { + *ty = I64; + } + } + ensure!( + ty == sig, + "Wrong type for floating point impl {} expecting {} but got {}", + name.red(), + sig.red(), + ty.red() + ); + floating_point_impls.insert(op, (module, export)); + } } } } - for (index, lib) in libraries.iter().enumerate() { - let module = Module::from_binary( + if let Some(lib) = forwarder { + modules.push(Module::from_binary( + &lib, + &available_imports, + &floating_point_impls, + true, + true, + debug_funcs, + None, + )?); + } + + for lib in libraries.iter() { + modules.push(Module::from_binary( lib, &available_imports, &floating_point_impls, true, - with_forwarder && (index == 0), + false, debug_funcs, None, - )?; - for (name, &func) in &*module.func_exports { - let ty = module.func_types[func as usize].clone(); - if let Ok(op) = name.parse::() { - let mut sig = op.signature(); - // wavm codegen takes care of effecting this type change at callsites - for ty in sig.inputs.iter_mut().chain(sig.outputs.iter_mut()) { - if *ty == F32 { - *ty = I32; - } else if *ty == F64 { - *ty = I64; - } - } - ensure!( - ty == sig, - "Wrong type for floating point impl {} expecting {} but got {}", - name.red(), - sig.red(), - ty.red() - ); - floating_point_impls.insert(op, (modules.len() as u32, func)); - } - } - modules.push(module); + )?); } - // Shouldn't be necessary, but to be safe, don't allow the main binary to import its own guest calls - available_imports.retain(|_, i| i.module as usize != modules.len()); modules.push(Module::from_binary( &bin, &available_imports, diff --git a/arbitrator/prover/src/test.rs b/arbitrator/prover/src/test.rs index 52bf394dd5..2f32aff012 100644 --- a/arbitrator/prover/src/test.rs +++ b/arbitrator/prover/src/test.rs @@ -57,7 +57,7 @@ pub fn reject_ambiguous_imports() { #[test] pub fn test_compress() -> Result<()> { - let data = include_bytes!(concat!(env!("OUT_DIR"), "/forwarder_stub.wasm")); + let data = include_bytes!(concat!(env!("OUT_DIR"), "/forwarder.wasm")); let mut last = vec![]; for dict in [Dictionary::Empty, Dictionary::StylusProgram] { diff --git a/arbitrator/prover/src/value.rs b/arbitrator/prover/src/value.rs index 6afffdf7a0..f2f54404f0 100644 --- a/arbitrator/prover/src/value.rs +++ b/arbitrator/prover/src/value.rs @@ -2,7 +2,7 @@ // For license information, see https://github.com/nitro/blob/master/LICENSE use crate::binary::FloatType; -use arbutil::{Bytes32, Color}; +use arbutil::{hostios::ParamType, Bytes32, Color}; use digest::Digest; use eyre::{bail, ErrReport, Result}; use serde::{Deserialize, Serialize}; @@ -77,6 +77,16 @@ impl From for ValType { } } +impl From<&ParamType> for ArbValueType { + fn from(ty: &ParamType) -> Self { + use ParamType as V; + match ty { + V::I32 => Self::I32, + V::I64 => Self::I64, + } + } +} + #[cfg(feature = "native")] pub fn parser_type(ty: &wasmer::Type) -> wasmer::wasmparser::ValType { match ty { From 8b719d0d0e7a948849f6735e597eb7d04b699594 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 11 Sep 2024 18:05:27 -0600 Subject: [PATCH 166/489] Merge remote-tracking branch 'origin/master' into forwarder-refactor --- arbnode/inbox_test.go | 4 +- arbos/programs/native.go | 70 +++++++++++------ arbos/programs/wasmstorehelper.go | 3 +- cmd/nitro/init.go | 6 +- cmd/nitro/init_test.go | 10 +++ cmd/nitro/nitro.go | 2 +- execution/gethexec/executionengine.go | 40 ++++++---- execution/gethexec/node.go | 46 ++++++++++-- go-ethereum | 2 +- staker/block_validator.go | 10 ++- staker/challenge_manager.go | 3 +- staker/stateless_block_validator.go | 5 +- system_tests/common_test.go | 52 +++++++------ system_tests/forwarder_test.go | 4 +- system_tests/program_test.go | 92 +++++++++++++++++++++-- system_tests/recreatestate_rpc_test.go | 18 ++--- system_tests/retryable_test.go | 2 +- system_tests/validation_mock_test.go | 6 +- validator/client/redis/producer.go | 9 ++- validator/client/validation_client.go | 15 ++-- validator/interface.go | 4 +- validator/server_api/json.go | 8 +- validator/server_arb/validator_spawner.go | 5 +- validator/server_jit/spawner.go | 5 +- validator/validation_entry.go | 4 +- validator/valnode/redis/consumer.go | 11 ++- validator/valnode/validation_api.go | 4 +- 27 files changed, 310 insertions(+), 130 deletions(-) diff --git a/arbnode/inbox_test.go b/arbnode/inbox_test.go index 1c46c593b9..d579b7c278 100644 --- a/arbnode/inbox_test.go +++ b/arbnode/inbox_test.go @@ -72,7 +72,9 @@ func NewTransactionStreamerForTest(t *testing.T, ownerAddress common.Address) (* if err != nil { Fail(t, err) } - if err := execEngine.Initialize(gethexec.DefaultCachingConfig.StylusLRUCache, &gethexec.DefaultStylusTargetConfig); err != nil { + stylusTargetConfig := &gethexec.DefaultStylusTargetConfig + Require(t, stylusTargetConfig.Validate()) // pre-processes config (i.a. parses wasmTargets) + if err := execEngine.Initialize(gethexec.DefaultCachingConfig.StylusLRUCache, stylusTargetConfig); err != nil { Fail(t, err) } execSeq := &execClientWrapper{execEngine, t} diff --git a/arbos/programs/native.go b/arbos/programs/native.go index 9316e8f230..377e25a31e 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/offchainlabs/nitro/arbos/burn" "github.com/offchainlabs/nitro/arbos/util" @@ -71,7 +72,7 @@ func activateProgramInternal( version uint16, debug bool, gasLeft *uint64, -) (*activationInfo, map[rawdb.Target][]byte, error) { +) (*activationInfo, map[ethdb.WasmTarget][]byte, error) { output := &rustBytes{} moduleHash := &bytes32{} stylusData := &C.StylusData{} @@ -99,25 +100,50 @@ func activateProgramInternal( } return nil, nil, err } - target := rawdb.LocalTarget() - status_asm := C.stylus_compile( - goSlice(wasm), - u16(version), - cbool(debug), - goSlice([]byte(target)), - output, - ) - asm := output.intoBytes() - if status_asm != 0 { - return nil, nil, fmt.Errorf("%w: %s", ErrProgramActivation, string(asm)) + targets := db.Database().WasmTargets() + type result struct { + target ethdb.WasmTarget + asm []byte + err error } - asmMap := map[rawdb.Target][]byte{ - rawdb.TargetWavm: module, - target: asm, + results := make(chan result, len(targets)) + for _, target := range targets { + if target == rawdb.TargetWavm { + results <- result{target, module, nil} + } else { + target := target + go func() { + output := &rustBytes{} + status_asm := C.stylus_compile( + goSlice(wasm), + u16(version), + cbool(debug), + goSlice([]byte(target)), + output, + ) + asm := output.intoBytes() + if status_asm != 0 { + results <- result{target, nil, fmt.Errorf("%w: %s", ErrProgramActivation, string(asm))} + return + } + results <- result{target, asm, nil} + }() + } + } + asmMap := make(map[ethdb.WasmTarget][]byte, len(targets)) + for range targets { + res := <-results + if res.err != nil { + err = errors.Join(res.err, err) + } else { + asmMap[res.target] = res.asm + } + } + if err != nil { + return nil, nil, fmt.Errorf("compilation failed for one or more targets: %w", err) } hash := moduleHash.toHash() - info := &activationInfo{ moduleHash: hash, initGas: uint16(stylusData.init_cost), @@ -171,7 +197,7 @@ func getLocalAsm(statedb vm.StateDB, moduleHash common.Hash, addressForLogging c } asm, exists := asmMap[localTarget] if !exists { - var availableTargets []rawdb.Target + var availableTargets []ethdb.WasmTarget for target := range asmMap { availableTargets = append(availableTargets, target) } @@ -202,12 +228,8 @@ func callProgram( panic("missing asm") } - if db, ok := db.(*state.StateDB); ok { - targets := []rawdb.Target{ - rawdb.TargetWavm, - rawdb.LocalTarget(), - } - db.RecordProgram(targets, moduleHash) + if stateDb, ok := db.(*state.StateDB); ok { + stateDb.RecordProgram(db.Database().WasmTargets(), moduleHash) } evmApi := newApi(interpreter, tracingInfo, scope, memoryModel) @@ -291,7 +313,7 @@ func ResizeWasmLruCache(size uint32) { const DefaultTargetDescriptionArm = "arm64-linux-unknown+neon" const DefaultTargetDescriptionX86 = "x86_64-linux-unknown+sse4.2+lzcnt+bmi" -func SetTarget(name rawdb.Target, description string, native bool) error { +func SetTarget(name ethdb.WasmTarget, description string, native bool) error { output := &rustBytes{} status := userStatus(C.stylus_target_set( goSlice([]byte(name)), diff --git a/arbos/programs/wasmstorehelper.go b/arbos/programs/wasmstorehelper.go index 4f82d80282..434820dd9c 100644 --- a/arbos/programs/wasmstorehelper.go +++ b/arbos/programs/wasmstorehelper.go @@ -43,8 +43,9 @@ func (p Programs) SaveActiveProgramToWasmStore(statedb *state.StateDB, codeHash return err } + targets := statedb.Database().WasmTargets() // If already in wasm store then return early - _, err = statedb.TryGetActivatedAsmMap([]rawdb.Target{rawdb.TargetWavm, rawdb.LocalTarget()}, moduleHash) + _, err = statedb.TryGetActivatedAsmMap(targets, moduleHash) if err == nil { return nil } diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index 51d895c069..a8463a7d21 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -560,7 +560,7 @@ func rebuildLocalWasm(ctx context.Context, config *gethexec.Config, l2BlockChain return chainDb, l2BlockChain, nil } -func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeConfig, chainId *big.Int, cacheConfig *core.CacheConfig, persistentConfig *conf.PersistentConfig, l1Client arbutil.L1Interface, rollupAddrs chaininfo.RollupAddresses) (ethdb.Database, *core.BlockChain, error) { +func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeConfig, chainId *big.Int, cacheConfig *core.CacheConfig, targetConfig *gethexec.StylusTargetConfig, persistentConfig *conf.PersistentConfig, l1Client arbutil.L1Interface, rollupAddrs chaininfo.RollupAddresses) (ethdb.Database, *core.BlockChain, error) { if !config.Init.Force { if readOnlyDb, err := stack.OpenDatabaseWithFreezerWithExtraOptions("l2chaindata", 0, 0, config.Persistent.Ancient, "l2chaindata/", true, persistentConfig.Pebble.ExtraOptions("l2chaindata")); err == nil { if chainConfig := gethexec.TryReadStoredChainConfig(readOnlyDb); chainConfig != nil { @@ -585,7 +585,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo if err := dbutil.UnfinishedConversionCheck(wasmDb); err != nil { return nil, nil, fmt.Errorf("wasm unfinished database conversion check error: %w", err) } - chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmDb, 1) + chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmDb, 1, targetConfig.WasmTargets()) _, err = rawdb.ParseStateScheme(cacheConfig.StateScheme, chainDb) if err != nil { return nil, nil, err @@ -660,7 +660,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo if err := validateOrUpgradeWasmStoreSchemaVersion(wasmDb); err != nil { return nil, nil, err } - chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmDb, 1) + chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmDb, 1, targetConfig.WasmTargets()) _, err = rawdb.ParseStateScheme(cacheConfig.StateScheme, chainDb) if err != nil { return nil, nil, err diff --git a/cmd/nitro/init_test.go b/cmd/nitro/init_test.go index 7b9153a9cf..48d969f053 100644 --- a/cmd/nitro/init_test.go +++ b/cmd/nitro/init_test.go @@ -354,6 +354,12 @@ func TestEmptyDatabaseDir(t *testing.T) { } } +func defaultStylusTargetConfigForTest(t *testing.T) *gethexec.StylusTargetConfig { + targetConfig := gethexec.DefaultStylusTargetConfig + Require(t, targetConfig.Validate()) + return &targetConfig +} + func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { t.Parallel() @@ -381,6 +387,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), + defaultStylusTargetConfigForTest(t), &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, @@ -397,6 +404,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), + defaultStylusTargetConfigForTest(t), &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, @@ -414,6 +422,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), + defaultStylusTargetConfigForTest(t), &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, @@ -549,6 +558,7 @@ func TestOpenInitializeChainDbEmptyInit(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), + defaultStylusTargetConfigForTest(t), &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index d4b2a87a30..146a0049e7 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -495,7 +495,7 @@ func mainImpl() int { } } - chainDb, l2BlockChain, err := openInitializeChainDb(ctx, stack, nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), &nodeConfig.Persistent, l1Client, rollupAddrs) + chainDb, l2BlockChain, err := openInitializeChainDb(ctx, stack, nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), &nodeConfig.Execution.StylusTarget, &nodeConfig.Persistent, l1Client, rollupAddrs) if l2BlockChain != nil { deferFuncs = append(deferFuncs, func() { l2BlockChain.Stop() }) } diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index 991c94540e..8594d5867d 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -151,19 +151,33 @@ func (s *ExecutionEngine) MarkFeedStart(to arbutil.MessageIndex) { } func populateStylusTargetCache(targetConfig *StylusTargetConfig) error { - var effectiveStylusTarget string - target := rawdb.LocalTarget() - switch target { - case rawdb.TargetArm64: - effectiveStylusTarget = targetConfig.Arm64 - case rawdb.TargetAmd64: - effectiveStylusTarget = targetConfig.Amd64 - case rawdb.TargetHost: - effectiveStylusTarget = targetConfig.Host - } - err := programs.SetTarget(target, effectiveStylusTarget, true) - if err != nil { - return fmt.Errorf("Failed to set stylus target: %w", err) + localTarget := rawdb.LocalTarget() + targets := targetConfig.WasmTargets() + var nativeSet bool + for _, target := range targets { + var effectiveStylusTarget string + switch target { + case rawdb.TargetWavm: + // skip wavm target + continue + case rawdb.TargetArm64: + effectiveStylusTarget = targetConfig.Arm64 + case rawdb.TargetAmd64: + effectiveStylusTarget = targetConfig.Amd64 + case rawdb.TargetHost: + effectiveStylusTarget = targetConfig.Host + default: + return fmt.Errorf("unsupported stylus target: %v", target) + } + isNative := target == localTarget + err := programs.SetTarget(target, effectiveStylusTarget, isNative) + if err != nil { + return fmt.Errorf("failed to set stylus target: %w", err) + } + nativeSet = nativeSet || isNative + } + if !nativeSet { + return fmt.Errorf("local target %v missing in list of archs %v", localTarget, targets) } return nil } diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index b864332e83..21c2b4bece 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -11,6 +11,7 @@ import ( "github.com/ethereum/go-ethereum/arbitrum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/filters" @@ -29,21 +30,51 @@ import ( ) type StylusTargetConfig struct { - Arm64 string `koanf:"arm64"` - Amd64 string `koanf:"amd64"` - Host string `koanf:"host"` + Arm64 string `koanf:"arm64"` + Amd64 string `koanf:"amd64"` + Host string `koanf:"host"` + ExtraArchs []string `koanf:"extra-archs"` + + wasmTargets []ethdb.WasmTarget +} + +func (c *StylusTargetConfig) WasmTargets() []ethdb.WasmTarget { + return c.wasmTargets +} + +func (c *StylusTargetConfig) Validate() error { + targetsSet := make(map[ethdb.WasmTarget]bool, len(c.ExtraArchs)) + for _, arch := range c.ExtraArchs { + target := ethdb.WasmTarget(arch) + if !rawdb.IsSupportedWasmTarget(target) { + return fmt.Errorf("unsupported architecture: %v, possible values: %s, %s, %s, %s", arch, rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost) + } + targetsSet[target] = true + } + if !targetsSet[rawdb.TargetWavm] { + return fmt.Errorf("%s target not found in archs list, archs: %v", rawdb.TargetWavm, c.ExtraArchs) + } + targetsSet[rawdb.LocalTarget()] = true + targets := make([]ethdb.WasmTarget, 0, len(c.ExtraArchs)+1) + for target := range targetsSet { + targets = append(targets, target) + } + c.wasmTargets = targets + return nil } var DefaultStylusTargetConfig = StylusTargetConfig{ - Arm64: programs.DefaultTargetDescriptionArm, - Amd64: programs.DefaultTargetDescriptionX86, - Host: "", + Arm64: programs.DefaultTargetDescriptionArm, + Amd64: programs.DefaultTargetDescriptionX86, + Host: "", + ExtraArchs: []string{string(rawdb.TargetWavm)}, } func StylusTargetConfigAddOptions(prefix string, f *flag.FlagSet) { f.String(prefix+".arm64", DefaultStylusTargetConfig.Arm64, "stylus programs compilation target for arm64 linux") f.String(prefix+".amd64", DefaultStylusTargetConfig.Amd64, "stylus programs compilation target for amd64 linux") f.String(prefix+".host", DefaultStylusTargetConfig.Host, "stylus programs compilation target for system other than 64-bit ARM or 64-bit x86") + f.StringSlice(prefix+".extra-archs", DefaultStylusTargetConfig.ExtraArchs, fmt.Sprintf("Comma separated list of extra architectures to cross-compile stylus program to and cache in wasm store (additionally to local target). Currently must include at least %s. (supported targets: %s, %s, %s, %s)", rawdb.TargetWavm, rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost)) } type Config struct { @@ -82,6 +113,9 @@ func (c *Config) Validate() error { if c.forwardingTarget != "" && c.Sequencer.Enable { return errors.New("ForwardingTarget set and sequencer enabled") } + if err := c.StylusTarget.Validate(); err != nil { + return err + } return nil } diff --git a/go-ethereum b/go-ethereum index 575062fad7..81114dde8a 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 575062fad7ff4db9d7c235f49472f658be29e2fe +Subproject commit 81114dde8a26bae90c188605c4a36d5919a4a265 diff --git a/staker/block_validator.go b/staker/block_validator.go index 2239952b37..7a7efca846 100644 --- a/staker/block_validator.go +++ b/staker/block_validator.go @@ -106,6 +106,7 @@ type BlockValidatorConfig struct { ValidationServerConfigs []rpcclient.ClientConfig `koanf:"validation-server-configs"` ValidationPoll time.Duration `koanf:"validation-poll" reload:"hot"` PrerecordedBlocks uint64 `koanf:"prerecorded-blocks" reload:"hot"` + RecordingIterLimit uint64 `koanf:"recording-iter-limit"` ForwardBlocks uint64 `koanf:"forward-blocks" reload:"hot"` CurrentModuleRoot string `koanf:"current-module-root"` // TODO(magic) requires reinitialization on hot reload PendingUpgradeModuleRoot string `koanf:"pending-upgrade-module-root"` // TODO(magic) requires StatelessBlockValidator recreation on hot reload @@ -174,6 +175,7 @@ func BlockValidatorConfigAddOptions(prefix string, f *pflag.FlagSet) { f.Uint64(prefix+".forward-blocks", DefaultBlockValidatorConfig.ForwardBlocks, "prepare entries for up to that many blocks ahead of validation (small footprint)") f.Uint64(prefix+".prerecorded-blocks", DefaultBlockValidatorConfig.PrerecordedBlocks, "record that many blocks ahead of validation (larger footprint)") f.String(prefix+".current-module-root", DefaultBlockValidatorConfig.CurrentModuleRoot, "current wasm module root ('current' read from chain, 'latest' from machines/latest dir, or provide hash)") + f.Uint64(prefix+".recording-iter-limit", DefaultBlockValidatorConfig.RecordingIterLimit, "limit on block recordings sent per iteration") f.String(prefix+".pending-upgrade-module-root", DefaultBlockValidatorConfig.PendingUpgradeModuleRoot, "pending upgrade wasm module root to additionally validate (hash, 'latest' or empty)") f.Bool(prefix+".failure-is-fatal", DefaultBlockValidatorConfig.FailureIsFatal, "failing a validation is treated as a fatal error") BlockValidatorDangerousConfigAddOptions(prefix+".dangerous", f) @@ -197,6 +199,7 @@ var DefaultBlockValidatorConfig = BlockValidatorConfig{ FailureIsFatal: true, Dangerous: DefaultBlockValidatorDangerousConfig, MemoryFreeLimit: "default", + RecordingIterLimit: 20, } var TestBlockValidatorConfig = BlockValidatorConfig{ @@ -207,6 +210,7 @@ var TestBlockValidatorConfig = BlockValidatorConfig{ ValidationPoll: 100 * time.Millisecond, ForwardBlocks: 128, PrerecordedBlocks: uint64(2 * runtime.NumCPU()), + RecordingIterLimit: 20, CurrentModuleRoot: "latest", PendingUpgradeModuleRoot: "latest", FailureIsFatal: true, @@ -500,7 +504,7 @@ func (v *BlockValidator) sendRecord(s *validationStatus) error { //nolint:gosec func (v *BlockValidator) writeToFile(validationEntry *validationEntry, moduleRoot common.Hash) error { - input, err := validationEntry.ToInput([]rawdb.Target{rawdb.TargetWavm}) + input, err := validationEntry.ToInput([]ethdb.WasmTarget{rawdb.TargetWavm}) if err != nil { return err } @@ -652,6 +656,10 @@ func (v *BlockValidator) sendNextRecordRequests(ctx context.Context) (bool, erro if recordUntil < pos { return false, nil } + recordUntilLimit := pos + arbutil.MessageIndex(v.config().RecordingIterLimit) + if recordUntil > recordUntilLimit { + recordUntil = recordUntilLimit + } log.Trace("preparing to record", "pos", pos, "until", recordUntil) // prepare could take a long time so we do it without a lock err := v.recorder.PrepareForRecord(ctx, pos, recordUntil) diff --git a/staker/challenge_manager.go b/staker/challenge_manager.go index ef431d3c79..c488c8a96e 100644 --- a/staker/challenge_manager.go +++ b/staker/challenge_manager.go @@ -16,6 +16,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" "github.com/offchainlabs/nitro/arbutil" @@ -468,7 +469,7 @@ func (m *ChallengeManager) createExecutionBackend(ctx context.Context, step uint if err != nil { return fmt.Errorf("error creating validation entry for challenge %v msg %v for execution challenge: %w", m.challengeIndex, initialCount, err) } - input, err := entry.ToInput([]rawdb.Target{rawdb.TargetWavm}) + input, err := entry.ToInput([]ethdb.WasmTarget{rawdb.TargetWavm}) if err != nil { return fmt.Errorf("error getting validation entry input of challenge %v msg %v: %w", m.challengeIndex, initialCount, err) } diff --git a/staker/stateless_block_validator.go b/staker/stateless_block_validator.go index d5eeb8eb69..f54ec8b58c 100644 --- a/staker/stateless_block_validator.go +++ b/staker/stateless_block_validator.go @@ -12,7 +12,6 @@ import ( "github.com/offchainlabs/nitro/arbstate/daprovider" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" @@ -134,7 +133,7 @@ type validationEntry struct { DelayedMsg []byte } -func (e *validationEntry) ToInput(stylusArchs []rawdb.Target) (*validator.ValidationInput, error) { +func (e *validationEntry) ToInput(stylusArchs []ethdb.WasmTarget) (*validator.ValidationInput, error) { if e.Stage != Ready { return nil, errors.New("cannot create input from non-ready entry") } @@ -143,7 +142,7 @@ func (e *validationEntry) ToInput(stylusArchs []rawdb.Target) (*validator.Valida HasDelayedMsg: e.HasDelayedMsg, DelayedMsgNr: e.DelayedMsgNr, Preimages: e.Preimages, - UserWasms: make(map[rawdb.Target]map[common.Hash][]byte, len(e.UserWasms)), + UserWasms: make(map[ethdb.WasmTarget]map[common.Hash][]byte, len(e.UserWasms)), BatchInfo: e.BatchInfo, DelayedMsg: e.DelayedMsg, StartState: e.Start, diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 6e7375a921..457dae0910 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -197,7 +197,7 @@ var TestSequencerConfig = gethexec.SequencerConfig{ EnableProfiling: false, } -func ExecConfigDefaultNonSequencerTest() *gethexec.Config { +func ExecConfigDefaultNonSequencerTest(t *testing.T) *gethexec.Config { config := gethexec.ConfigDefault config.Caching = TestCachingConfig config.ParentChainReader = headerreader.TestConfig @@ -206,12 +206,12 @@ func ExecConfigDefaultNonSequencerTest() *gethexec.Config { config.ForwardingTarget = "null" config.TxPreChecker.Strictness = gethexec.TxPreCheckerStrictnessNone - _ = config.Validate() + Require(t, config.Validate()) return &config } -func ExecConfigDefaultTest() *gethexec.Config { +func ExecConfigDefaultTest(t *testing.T) *gethexec.Config { config := gethexec.ConfigDefault config.Caching = TestCachingConfig config.Sequencer = TestSequencerConfig @@ -219,7 +219,7 @@ func ExecConfigDefaultTest() *gethexec.Config { config.ForwardingTarget = "null" config.TxPreChecker.Strictness = gethexec.TxPreCheckerStrictnessNone - _ = config.Validate() + Require(t, config.Validate()) return &config } @@ -272,7 +272,7 @@ func (b *NodeBuilder) DefaultConfig(t *testing.T, withL1 bool) *NodeBuilder { b.l2StackConfig = testhelpers.CreateStackConfigForTest(b.dataDir) cp := valnode.TestValidationConfig b.valnodeConfig = &cp - b.execConfig = ExecConfigDefaultTest() + b.execConfig = ExecConfigDefaultTest(t) return b } @@ -293,6 +293,11 @@ func (b *NodeBuilder) WithWasmRootDir(wasmRootDir string) *NodeBuilder { return b } +func (b *NodeBuilder) WithExtraArchs(targets []string) *NodeBuilder { + b.execConfig.StylusTarget.ExtraArchs = targets + return b +} + func (b *NodeBuilder) Build(t *testing.T) func() { b.CheckConfig(t) if b.withL1 { @@ -310,7 +315,7 @@ func (b *NodeBuilder) CheckConfig(t *testing.T) { b.nodeConfig = arbnode.ConfigDefaultL1Test() } if b.execConfig == nil { - b.execConfig = ExecConfigDefaultTest() + b.execConfig = ExecConfigDefaultTest(t) } if b.L1Info == nil { b.L1Info = NewL1TestInfo(t) @@ -346,7 +351,7 @@ func (b *NodeBuilder) BuildL2OnL1(t *testing.T) func() { var l2arbDb ethdb.Database var l2blockchain *core.BlockChain _, b.L2.Stack, l2chainDb, l2arbDb, l2blockchain = createL2BlockChainWithStackConfig( - t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, &b.execConfig.Caching) + t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, b.execConfig) var sequencerTxOptsPtr *bind.TransactOpts var dataSigner signature.DataSignerFunc @@ -409,7 +414,7 @@ func (b *NodeBuilder) BuildL2(t *testing.T) func() { var arbDb ethdb.Database var blockchain *core.BlockChain b.L2Info, b.L2.Stack, chainDb, arbDb, blockchain = createL2BlockChain( - t, b.L2Info, b.dataDir, b.chainConfig, &b.execConfig.Caching) + t, b.L2Info, b.dataDir, b.chainConfig, b.execConfig) Require(t, b.execConfig.Validate()) execConfig := b.execConfig @@ -460,7 +465,7 @@ func (b *NodeBuilder) RestartL2Node(t *testing.T) { } b.L2.cleanup() - l2info, stack, chainDb, arbDb, blockchain := createL2BlockChainWithStackConfig(t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, &b.execConfig.Caching) + l2info, stack, chainDb, arbDb, blockchain := createL2BlockChainWithStackConfig(t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, b.execConfig) execConfigFetcher := func() *gethexec.Config { return b.execConfig } execNode, err := gethexec.CreateExecutionNode(b.ctx, stack, chainDb, blockchain, nil, execConfigFetcher) @@ -1047,30 +1052,32 @@ func DeployOnTestL1( } func createL2BlockChain( - t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, cacheConfig *gethexec.CachingConfig, + t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, execConfig *gethexec.Config, ) (*BlockchainTestInfo, *node.Node, ethdb.Database, ethdb.Database, *core.BlockChain) { - return createL2BlockChainWithStackConfig(t, l2info, dataDir, chainConfig, nil, nil, cacheConfig) + return createL2BlockChainWithStackConfig(t, l2info, dataDir, chainConfig, nil, nil, execConfig) } func createL2BlockChainWithStackConfig( - t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, initMessage *arbostypes.ParsedInitMessage, stackConfig *node.Config, cacheConfig *gethexec.CachingConfig, + t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, initMessage *arbostypes.ParsedInitMessage, stackConfig *node.Config, execConfig *gethexec.Config, ) (*BlockchainTestInfo, *node.Node, ethdb.Database, ethdb.Database, *core.BlockChain) { if l2info == nil { l2info = NewArbTestInfo(t, chainConfig.ChainID) } - var stack *node.Node - var err error if stackConfig == nil { stackConfig = testhelpers.CreateStackConfigForTest(dataDir) } - stack, err = node.New(stackConfig) + if execConfig == nil { + execConfig = ExecConfigDefaultTest(t) + } + + stack, err := node.New(stackConfig) Require(t, err) chainData, err := stack.OpenDatabaseWithExtraOptions("l2chaindata", 0, 0, "l2chaindata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("l2chaindata")) Require(t, err) wasmData, err := stack.OpenDatabaseWithExtraOptions("wasm", 0, 0, "wasm/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("wasm")) Require(t, err) - chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmData, 0) + chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmData, 0, execConfig.StylusTarget.WasmTargets()) arbDb, err := stack.OpenDatabaseWithExtraOptions("arbitrumdata", 0, 0, "arbitrumdata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("arbitrumdata")) Require(t, err) @@ -1085,11 +1092,8 @@ func createL2BlockChainWithStackConfig( SerializedChainConfig: serializedChainConfig, } } - var coreCacheConfig *core.CacheConfig - if cacheConfig != nil { - coreCacheConfig = gethexec.DefaultCacheConfigFor(stack, cacheConfig) - } - blockchain, err := gethexec.WriteOrTestBlockChain(chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest().TxLookupLimit, 0) + coreCacheConfig := gethexec.DefaultCacheConfigFor(stack, &execConfig.Caching) + blockchain, err := gethexec.WriteOrTestBlockChain(chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest(t).TxLookupLimit, 0) Require(t, err) return l2info, stack, chainDb, arbDb, blockchain @@ -1149,7 +1153,7 @@ func Create2ndNodeWithConfig( nodeConfig = arbnode.ConfigDefaultL1NonSequencerTest() } if execConfig == nil { - execConfig = ExecConfigDefaultNonSequencerTest() + execConfig = ExecConfigDefaultNonSequencerTest(t) } feedErrChan := make(chan error, 10) l1rpcClient := l1stack.Attach() @@ -1165,7 +1169,7 @@ func Create2ndNodeWithConfig( Require(t, err) wasmData, err := l2stack.OpenDatabaseWithExtraOptions("wasm", 0, 0, "wasm/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("wasm")) Require(t, err) - l2chainDb := rawdb.WrapDatabaseWithWasm(l2chainData, wasmData, 0) + l2chainDb := rawdb.WrapDatabaseWithWasm(l2chainData, wasmData, 0, execConfig.StylusTarget.WasmTargets()) l2arbDb, err := l2stack.OpenDatabaseWithExtraOptions("arbitrumdata", 0, 0, "arbitrumdata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("arbitrumdata")) Require(t, err) @@ -1179,7 +1183,7 @@ func Create2ndNodeWithConfig( chainConfig := firstExec.ArbInterface.BlockChain().Config() coreCacheConfig := gethexec.DefaultCacheConfigFor(l2stack, &execConfig.Caching) - l2blockchain, err := gethexec.WriteOrTestBlockChain(l2chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest().TxLookupLimit, 0) + l2blockchain, err := gethexec.WriteOrTestBlockChain(l2chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest(t).TxLookupLimit, 0) Require(t, err) AddValNodeIfNeeded(t, ctx, nodeConfig, true, "", valnodeConfig.Wasm.RootPath) diff --git a/system_tests/forwarder_test.go b/system_tests/forwarder_test.go index f87283432e..57381ca84e 100644 --- a/system_tests/forwarder_test.go +++ b/system_tests/forwarder_test.go @@ -38,7 +38,7 @@ func TestStaticForwarder(t *testing.T) { clientA := builder.L2.Client nodeConfigB := arbnode.ConfigDefaultL1Test() - execConfigB := ExecConfigDefaultTest() + execConfigB := ExecConfigDefaultTest(t) execConfigB.Sequencer.Enable = false nodeConfigB.Sequencer = false nodeConfigB.DelayedSequencer.Enable = false @@ -109,7 +109,7 @@ func createForwardingNode(t *testing.T, builder *NodeBuilder, ipcPath string, re nodeConfig.Sequencer = false nodeConfig.DelayedSequencer.Enable = false nodeConfig.BatchPoster.Enable = false - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.Sequencer.Enable = false execConfig.Forwarder.RedisUrl = redisUrl execConfig.ForwardingTarget = fallbackPath diff --git a/system_tests/program_test.go b/system_tests/program_test.go index c89a13e205..83c066fdb5 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -45,18 +45,31 @@ import ( var oneEth = arbmath.UintToBig(1e18) +var allWasmTargets = []string{string(rawdb.TargetWavm), string(rawdb.TargetArm64), string(rawdb.TargetAmd64), string(rawdb.TargetHost)} + func TestProgramKeccak(t *testing.T) { t.Parallel() - keccakTest(t, true) + t.Run("WithDefaultWasmTargets", func(t *testing.T) { + keccakTest(t, true) + }) + + t.Run("WithAllWasmTargets", func(t *testing.T) { + keccakTest(t, true, func(builder *NodeBuilder) { + builder.WithExtraArchs(allWasmTargets) + }) + }) } -func keccakTest(t *testing.T, jit bool) { - builder, auth, cleanup := setupProgramTest(t, jit) +func keccakTest(t *testing.T, jit bool, builderOpts ...func(*NodeBuilder)) { + builder, auth, cleanup := setupProgramTest(t, jit, builderOpts...) ctx := builder.ctx l2client := builder.L2.Client defer cleanup() programAddress := deployWasm(t, ctx, auth, l2client, rustFile("keccak")) + wasmDb := builder.L2.ExecNode.Backend.ArbInterface().BlockChain().StateCache().WasmStore() + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) + wasm, _ := readWasmFile(t, rustFile("keccak")) otherAddressSameCode := deployContract(t, ctx, auth, l2client, wasm) arbWasm, err := pgen.NewArbWasm(types.ArbWasmAddress, l2client) @@ -68,6 +81,7 @@ func keccakTest(t *testing.T, jit bool) { Fatal(t, "activate should have failed with ProgramUpToDate", err) } }) + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) if programAddress == otherAddressSameCode { Fatal(t, "expected to deploy at two separate program addresses") @@ -141,11 +155,18 @@ func keccakTest(t *testing.T, jit bool) { func TestProgramActivateTwice(t *testing.T) { t.Parallel() - testActivateTwice(t, true) + t.Run("WithDefaultWasmTargets", func(t *testing.T) { + testActivateTwice(t, true) + }) + t.Run("WithAllWasmTargets", func(t *testing.T) { + testActivateTwice(t, true, func(builder *NodeBuilder) { + builder.WithExtraArchs(allWasmTargets) + }) + }) } -func testActivateTwice(t *testing.T, jit bool) { - builder, auth, cleanup := setupProgramTest(t, jit) +func testActivateTwice(t *testing.T, jit bool, builderOpts ...func(*NodeBuilder)) { + builder, auth, cleanup := setupProgramTest(t, jit, builderOpts...) ctx := builder.ctx l2info := builder.L2Info l2client := builder.L2.Client @@ -171,6 +192,10 @@ func testActivateTwice(t *testing.T, jit bool) { colors.PrintBlue("keccak program B deployed to ", keccakB) multiAddr := deployWasm(t, ctx, auth, l2client, rustFile("multicall")) + + wasmDb := builder.L2.ExecNode.Backend.ArbInterface().BlockChain().StateCache().WasmStore() + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) + preimage := []byte("it's time to du-du-du-du d-d-d-d-d-d-d de-duplicate") keccakArgs := []byte{0x01} // keccak the preimage once @@ -194,6 +219,7 @@ func testActivateTwice(t *testing.T, jit bool) { // Calling the contract pre-activation should fail. checkReverts() + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) // mechanisms for creating calldata activateProgram, _ := util.NewCallParser(pgen.ArbWasmABI, "activateProgram") @@ -216,6 +242,7 @@ func testActivateTwice(t *testing.T, jit bool) { // Ensure the revert also reverted keccak's activation checkReverts() + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) // Activate keccak program A, then call into B, which should succeed due to being the same codehash args = argsForMulticall(vm.CALL, types.ArbWasmAddress, oneEth, pack(activateProgram(keccakA))) @@ -223,6 +250,7 @@ func testActivateTwice(t *testing.T, jit bool) { tx = l2info.PrepareTxTo("Owner", &multiAddr, 1e9, oneEth, args) ensure(tx, l2client.SendTransaction(ctx, tx)) + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 2) validateBlocks(t, 7, jit, builder) } @@ -1834,7 +1862,9 @@ func createMapFromDb(db ethdb.KeyValueStore) (map[string][]byte, error) { } func TestWasmStoreRebuilding(t *testing.T) { - builder, auth, cleanup := setupProgramTest(t, true) + builder, auth, cleanup := setupProgramTest(t, true, func(b *NodeBuilder) { + b.WithExtraArchs(allWasmTargets) + }) ctx := builder.ctx l2info := builder.L2Info l2client := builder.L2.Client @@ -1871,6 +1901,7 @@ func TestWasmStoreRebuilding(t *testing.T) { storeMap, err := createMapFromDb(wasmDb) Require(t, err) + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) // close nodeB cleanupB() @@ -1927,5 +1958,52 @@ func TestWasmStoreRebuilding(t *testing.T) { } } + checkWasmStoreContent(t, wasmDbAfterRebuild, builder.execConfig.StylusTarget.ExtraArchs, 1) cleanupB() } + +func readModuleHashes(t *testing.T, wasmDb ethdb.KeyValueStore) []common.Hash { + modulesSet := make(map[common.Hash]struct{}) + asmPrefix := []byte{0x00, 'w'} + it := wasmDb.NewIterator(asmPrefix, nil) + defer it.Release() + for it.Next() { + key := it.Key() + if len(key) != rawdb.WasmKeyLen { + t.Fatalf("unexpected activated module key length, len: %d, key: %v", len(key), key) + } + moduleHash := key[rawdb.WasmPrefixLen:] + if len(moduleHash) != common.HashLength { + t.Fatalf("Invalid moduleHash length in key: %v, moduleHash: %v", key, moduleHash) + } + modulesSet[common.BytesToHash(moduleHash)] = struct{}{} + } + modules := make([]common.Hash, 0, len(modulesSet)) + for module := range modulesSet { + modules = append(modules, module) + } + return modules +} + +func checkWasmStoreContent(t *testing.T, wasmDb ethdb.KeyValueStore, targets []string, numModules int) { + modules := readModuleHashes(t, wasmDb) + if len(modules) != numModules { + t.Fatalf("Unexpected number of module hashes found in wasm store, want: %d, have: %d", numModules, len(modules)) + } + for _, module := range modules { + for _, target := range targets { + wasmTarget := ethdb.WasmTarget(target) + if !rawdb.IsSupportedWasmTarget(wasmTarget) { + t.Fatalf("internal test error - unsupported target passed to checkWasmStoreContent: %v", target) + } + func() { + defer func() { + if r := recover(); r != nil { + t.Fatalf("Failed to read activated asm for target: %v, module: %v", target, module) + } + }() + _ = rawdb.ReadActivatedAsm(wasmDb, wasmTarget, module) + }() + } + } +} diff --git a/system_tests/recreatestate_rpc_test.go b/system_tests/recreatestate_rpc_test.go index cd3904ca06..7b09552e28 100644 --- a/system_tests/recreatestate_rpc_test.go +++ b/system_tests/recreatestate_rpc_test.go @@ -95,7 +95,7 @@ func removeStatesFromDb(t *testing.T, bc *core.BlockChain, db ethdb.Database, fr func TestRecreateStateForRPCNoDepthLimit(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = arbitrum.InfiniteMaxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -134,7 +134,7 @@ func TestRecreateStateForRPCBigEnoughDepthLimit(t *testing.T) { defer cancel() // #nosec G115 depthGasLimit := int64(256 * util.NormalizeL2GasForL1GasInitial(800_000, params.GWei)) - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = depthGasLimit execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -171,7 +171,7 @@ func TestRecreateStateForRPCBigEnoughDepthLimit(t *testing.T) { func TestRecreateStateForRPCDepthLimitExceeded(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = int64(200) execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -208,7 +208,7 @@ func TestRecreateStateForRPCMissingBlockParent(t *testing.T) { var headerCacheLimit uint64 = 512 ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = arbitrum.InfiniteMaxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -256,7 +256,7 @@ func TestRecreateStateForRPCBeyondGenesis(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = arbitrum.InfiniteMaxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -294,7 +294,7 @@ func TestRecreateStateForRPCBlockNotFoundWhileRecreating(t *testing.T) { var blockCacheLimit uint64 = 256 ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = arbitrum.InfiniteMaxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -342,7 +342,7 @@ func testSkippingSavingStateAndRecreatingAfterRestart(t *testing.T, cacheConfig ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = maxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -483,7 +483,7 @@ func TestSkippingSavingStateAndRecreatingAfterRestart(t *testing.T) { func TestGettingStateForRPCFullNode(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.Caching.SnapshotCache = 0 // disable snapshots execConfig.Caching.BlockAge = 0 // use only Caching.BlockCount to keep only last N blocks in dirties cache, no matter how new they are execConfig.Sequencer.MaxBlockSpeed = 0 @@ -527,7 +527,7 @@ func TestGettingStateForRPCFullNode(t *testing.T) { func TestGettingStateForRPCHybridArchiveNode(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.Caching.Archive = true // For now Archive node should use HashScheme execConfig.Caching.StateScheme = rawdb.HashScheme diff --git a/system_tests/retryable_test.go b/system_tests/retryable_test.go index 106dfc6d46..aa9fbfd72e 100644 --- a/system_tests/retryable_test.go +++ b/system_tests/retryable_test.go @@ -1042,7 +1042,7 @@ func elevateL2Basefee(t *testing.T, ctx context.Context, builder *NodeBuilder) { _, err = precompilesgen.NewArbosTest(common.HexToAddress("0x69"), builder.L2.Client) Require(t, err, "failed to deploy ArbosTest") - burnAmount := ExecConfigDefaultTest().RPC.RPCGasCap + burnAmount := ExecConfigDefaultTest(t).RPC.RPCGasCap burnTarget := uint64(5 * l2pricing.InitialSpeedLimitPerSecondV6 * l2pricing.InitialBacklogTolerance) for i := uint64(0); i < (burnTarget+burnAmount)/burnAmount; i++ { burnArbGas := arbosTestAbi.Methods["burnArbGas"] diff --git a/system_tests/validation_mock_test.go b/system_tests/validation_mock_test.go index 88421e4c4b..2739c7545e 100644 --- a/system_tests/validation_mock_test.go +++ b/system_tests/validation_mock_test.go @@ -8,9 +8,9 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rpc" "github.com/offchainlabs/nitro/arbnode" @@ -61,8 +61,8 @@ func (s *mockSpawner) WasmModuleRoots() ([]common.Hash, error) { return mockWasmModuleRoots, nil } -func (s *mockSpawner) StylusArchs() []rawdb.Target { - return []rawdb.Target{"mock"} +func (s *mockSpawner) StylusArchs() []ethdb.WasmTarget { + return []ethdb.WasmTarget{"mock"} } func (s *mockSpawner) Launch(entry *validator.ValidationInput, moduleRoot common.Hash) validator.ValidationRun { diff --git a/validator/client/redis/producer.go b/validator/client/redis/producer.go index f98c246d0e..adc2f34af5 100644 --- a/validator/client/redis/producer.go +++ b/validator/client/redis/producer.go @@ -7,6 +7,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/go-redis/redis/v8" "github.com/offchainlabs/nitro/pubsub" @@ -35,7 +36,7 @@ func (c ValidationClientConfig) Enabled() bool { func (c ValidationClientConfig) Validate() error { for _, arch := range c.StylusArchs { - if !rawdb.Target(arch).IsValid() { + if !rawdb.IsSupportedWasmTarget(ethdb.WasmTarget(arch)) { return fmt.Errorf("Invalid stylus arch: %v", arch) } } @@ -162,10 +163,10 @@ func (c *ValidationClient) Name() string { return c.config.Name } -func (c *ValidationClient) StylusArchs() []rawdb.Target { - stylusArchs := make([]rawdb.Target, 0, len(c.config.StylusArchs)) +func (c *ValidationClient) StylusArchs() []ethdb.WasmTarget { + stylusArchs := make([]ethdb.WasmTarget, 0, len(c.config.StylusArchs)) for _, arch := range c.config.StylusArchs { - stylusArchs = append(stylusArchs, rawdb.Target(arch)) + stylusArchs = append(stylusArchs, ethdb.WasmTarget(arch)) } return stylusArchs } diff --git a/validator/client/validation_client.go b/validator/client/validation_client.go index 00bd992f46..3b18ad1851 100644 --- a/validator/client/validation_client.go +++ b/validator/client/validation_client.go @@ -22,6 +22,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rpc" @@ -31,7 +32,7 @@ type ValidationClient struct { stopwaiter.StopWaiter client *rpcclient.RpcClient name string - stylusArchs []rawdb.Target + stylusArchs []ethdb.WasmTarget room atomic.Int32 wasmModuleRoots []common.Hash } @@ -40,7 +41,7 @@ func NewValidationClient(config rpcclient.ClientConfigFetcher, stack *node.Node) return &ValidationClient{ client: rpcclient.NewRpcClient(config, stack), name: "not started", - stylusArchs: []rawdb.Target{"not started"}, + stylusArchs: []ethdb.WasmTarget{"not started"}, } } @@ -67,20 +68,20 @@ func (c *ValidationClient) Start(ctx context.Context) error { if len(name) == 0 { return errors.New("couldn't read name from server") } - var stylusArchs []rawdb.Target + var stylusArchs []ethdb.WasmTarget if err := c.client.CallContext(ctx, &stylusArchs, server_api.Namespace+"_stylusArchs"); err != nil { var rpcError rpc.Error ok := errors.As(err, &rpcError) if !ok || rpcError.ErrorCode() != -32601 { return fmt.Errorf("could not read stylus arch from server: %w", err) } - stylusArchs = []rawdb.Target{rawdb.Target("pre-stylus")} // invalid, will fail if trying to validate block with stylus + stylusArchs = []ethdb.WasmTarget{ethdb.WasmTarget("pre-stylus")} // invalid, will fail if trying to validate block with stylus } else { if len(stylusArchs) == 0 { return fmt.Errorf("could not read stylus archs from validation server") } for _, stylusArch := range stylusArchs { - if stylusArch != rawdb.TargetWavm && stylusArch != rawdb.LocalTarget() && stylusArch != "mock" { + if !rawdb.IsSupportedWasmTarget(ethdb.WasmTarget(stylusArch)) && stylusArch != "mock" { return fmt.Errorf("unsupported stylus architecture: %v", stylusArch) } } @@ -118,11 +119,11 @@ func (c *ValidationClient) WasmModuleRoots() ([]common.Hash, error) { return nil, errors.New("not started") } -func (c *ValidationClient) StylusArchs() []rawdb.Target { +func (c *ValidationClient) StylusArchs() []ethdb.WasmTarget { if c.Started() { return c.stylusArchs } - return []rawdb.Target{"not started"} + return []ethdb.WasmTarget{"not started"} } func (c *ValidationClient) Stop() { diff --git a/validator/interface.go b/validator/interface.go index 81b40ae5cf..af08629137 100644 --- a/validator/interface.go +++ b/validator/interface.go @@ -4,7 +4,7 @@ import ( "context" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/util/containers" ) @@ -14,7 +14,7 @@ type ValidationSpawner interface { Start(context.Context) error Stop() Name() string - StylusArchs() []rawdb.Target + StylusArchs() []ethdb.WasmTarget Room() int } diff --git a/validator/server_api/json.go b/validator/server_api/json.go index dbe2bb1fee..6fe936e17d 100644 --- a/validator/server_api/json.go +++ b/validator/server_api/json.go @@ -11,7 +11,7 @@ import ( "os" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/arbcompress" "github.com/offchainlabs/nitro/arbutil" @@ -64,7 +64,7 @@ type InputJSON struct { BatchInfo []BatchInfoJson DelayedMsgB64 string StartState validator.GoGlobalState - UserWasms map[rawdb.Target]map[common.Hash]string + UserWasms map[ethdb.WasmTarget]map[common.Hash]string DebugChain bool } @@ -96,7 +96,7 @@ func ValidationInputToJson(entry *validator.ValidationInput) *InputJSON { DelayedMsgB64: base64.StdEncoding.EncodeToString(entry.DelayedMsg), StartState: entry.StartState, PreimagesB64: jsonPreimagesMap, - UserWasms: make(map[rawdb.Target]map[common.Hash]string), + UserWasms: make(map[ethdb.WasmTarget]map[common.Hash]string), DebugChain: entry.DebugChain, } for _, binfo := range entry.BatchInfo { @@ -128,7 +128,7 @@ func ValidationInputFromJson(entry *InputJSON) (*validator.ValidationInput, erro DelayedMsgNr: entry.DelayedMsgNr, StartState: entry.StartState, Preimages: preimages, - UserWasms: make(map[rawdb.Target]map[common.Hash][]byte), + UserWasms: make(map[ethdb.WasmTarget]map[common.Hash][]byte), DebugChain: entry.DebugChain, } delayed, err := base64.StdEncoding.DecodeString(entry.DelayedMsgB64) diff --git a/validator/server_arb/validator_spawner.go b/validator/server_arb/validator_spawner.go index eb53070303..6f0d0cee1d 100644 --- a/validator/server_arb/validator_spawner.go +++ b/validator/server_arb/validator_spawner.go @@ -13,6 +13,7 @@ import ( "github.com/spf13/pflag" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/util/containers" "github.com/offchainlabs/nitro/util/stopwaiter" @@ -89,8 +90,8 @@ func (s *ArbitratorSpawner) WasmModuleRoots() ([]common.Hash, error) { return s.locator.ModuleRoots(), nil } -func (s *ArbitratorSpawner) StylusArchs() []rawdb.Target { - return []rawdb.Target{rawdb.TargetWavm} +func (s *ArbitratorSpawner) StylusArchs() []ethdb.WasmTarget { + return []ethdb.WasmTarget{rawdb.TargetWavm} } func (s *ArbitratorSpawner) Name() string { diff --git a/validator/server_jit/spawner.go b/validator/server_jit/spawner.go index 92b50b17cb..d77317d218 100644 --- a/validator/server_jit/spawner.go +++ b/validator/server_jit/spawner.go @@ -10,6 +10,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/util/stopwaiter" "github.com/offchainlabs/nitro/validator" @@ -72,8 +73,8 @@ func (v *JitSpawner) WasmModuleRoots() ([]common.Hash, error) { return v.locator.ModuleRoots(), nil } -func (v *JitSpawner) StylusArchs() []rawdb.Target { - return []rawdb.Target{rawdb.LocalTarget()} +func (v *JitSpawner) StylusArchs() []ethdb.WasmTarget { + return []ethdb.WasmTarget{rawdb.LocalTarget()} } func (v *JitSpawner) execute( diff --git a/validator/validation_entry.go b/validator/validation_entry.go index 2c357659ad..d340993fa2 100644 --- a/validator/validation_entry.go +++ b/validator/validation_entry.go @@ -2,7 +2,7 @@ package validator import ( "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/arbutil" ) @@ -17,7 +17,7 @@ type ValidationInput struct { HasDelayedMsg bool DelayedMsgNr uint64 Preimages map[arbutil.PreimageType]map[common.Hash][]byte - UserWasms map[rawdb.Target]map[common.Hash][]byte + UserWasms map[ethdb.WasmTarget]map[common.Hash][]byte BatchInfo []BatchInfo DelayedMsg []byte StartState GoGlobalState diff --git a/validator/valnode/redis/consumer.go b/validator/valnode/redis/consumer.go index 7456466533..2b025600cc 100644 --- a/validator/valnode/redis/consumer.go +++ b/validator/valnode/redis/consumer.go @@ -111,10 +111,12 @@ func (s *ValidationServer) Start(ctx_in context.Context) { req, err := c.Consume(ctx) if err != nil { log.Error("Consuming request", "error", err) + requestTokenQueue <- struct{}{} return 0 } if req == nil { - // There's nothing in the queue. + // There's nothing in the queue + requestTokenQueue <- struct{}{} return time.Second } select { @@ -152,9 +154,10 @@ func (s *ValidationServer) Start(ctx_in context.Context) { res, err := valRun.Await(ctx) if err != nil { log.Error("Error validating", "request value", work.req.Value, "error", err) - } - if err := s.consumers[work.moduleRoot].SetResult(ctx, work.req.ID, res); err != nil { - log.Error("Error setting result for request", "id", work.req.ID, "result", res, "error", err) + } else { + if err := s.consumers[work.moduleRoot].SetResult(ctx, work.req.ID, res); err != nil { + log.Error("Error setting result for request", "id", work.req.ID, "result", res, "error", err) + } } select { case <-ctx.Done(): diff --git a/validator/valnode/validation_api.go b/validator/valnode/validation_api.go index a79ac7fa55..a10d931dfc 100644 --- a/validator/valnode/validation_api.go +++ b/validator/valnode/validation_api.go @@ -12,7 +12,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/util/stopwaiter" "github.com/offchainlabs/nitro/validator" @@ -45,7 +45,7 @@ func (a *ValidationServerAPI) WasmModuleRoots() ([]common.Hash, error) { return a.spawner.WasmModuleRoots() } -func (a *ValidationServerAPI) StylusArchs() ([]rawdb.Target, error) { +func (a *ValidationServerAPI) StylusArchs() ([]ethdb.WasmTarget, error) { return a.spawner.StylusArchs(), nil } From 925a623477d1ed3fa82cc42e6e90a947378d4d0b Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 11 Sep 2024 18:06:14 -0600 Subject: [PATCH 167/489] cargo_fmt --- arbitrator/prover/src/lib.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/arbitrator/prover/src/lib.rs b/arbitrator/prover/src/lib.rs index 20472691c1..e45d12c492 100644 --- a/arbitrator/prover/src/lib.rs +++ b/arbitrator/prover/src/lib.rs @@ -73,8 +73,13 @@ pub unsafe extern "C" fn arbitrator_load_machine( with_forwarder: bool, ) -> *mut Machine { let debug_chain = debug_chain != 0; - match arbitrator_load_machine_impl(binary_path, library_paths, library_paths_size, debug_chain, with_forwarder) - { + match arbitrator_load_machine_impl( + binary_path, + library_paths, + library_paths_size, + debug_chain, + with_forwarder, + ) { Ok(mach) => mach, Err(err) => { eprintln!("Error loading binary: {:?}", err); From c5f7181a9319f14dfd704c9e08e68508d3323788 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 11 Sep 2024 18:14:05 -0600 Subject: [PATCH 168/489] use import resolvers --- arbitrator/prover/src/host.rs | 9 +- arbitrator/prover/src/machine.rs | 148 ++++++++++++++++++++----------- 2 files changed, 102 insertions(+), 55 deletions(-) diff --git a/arbitrator/prover/src/host.rs b/arbitrator/prover/src/host.rs index 1d0fe658ec..dc0f0b74b2 100644 --- a/arbitrator/prover/src/host.rs +++ b/arbitrator/prover/src/host.rs @@ -405,7 +405,7 @@ impl Hostio { } } -pub fn get_impl(module: &str, name: &str) -> Result<(Function, bool)> { +pub fn get_impl(module: &str, name: &str) -> Result { let hostio: Hostio = format!("{module}__{name}").parse()?; let append = |code: &mut Vec| { @@ -414,8 +414,11 @@ pub fn get_impl(module: &str, name: &str) -> Result<(Function, bool)> { Ok(()) }; - let debug = module == "console" || module == "debug"; - Function::new(&[], append, hostio.ty(), &[]).map(|x| (x, debug)) + Function::new(&[], append, hostio.ty(), &[]) +} + +pub fn hostio_module_is_debug(module: &str) -> bool { + module == "console" || module == "debug" } /// Adds internal functions to a module. diff --git a/arbitrator/prover/src/machine.rs b/arbitrator/prover/src/machine.rs index c045fc72c9..9f2ba081dc 100644 --- a/arbitrator/prover/src/machine.rs +++ b/arbitrator/prover/src/machine.rs @@ -346,14 +346,95 @@ lazy_static! { }; } +trait ImportResolver: for<'a, 'b> Fn(&'a str, &'b str) -> Result + Sized { + fn compose(self, other: impl ImportResolver) -> impl ImportResolver { + move |module, name| { + self(module, name).or_else(|original_err| { + let mut result = other(module, name); + if result.is_err() { + for err_layer in original_err.chain().rev() { + result = result.wrap_err(format!("{err_layer}")); + } + } + result + }) + } + } + + fn condition(self, condition: bool) -> impl ImportResolver { + move |module, name| { + if condition { + self(module, name) + } else { + bail!("resolver disabled") + } + } + } +} + +impl Fn(&'a str, &'b str) -> Result> ImportResolver for F {} + impl Module { - fn from_binary( + fn make_imports_resolver(import_map: &HashMap) -> impl ImportResolver + '_ { + move |module, name| { + let qualified_name = format!("{module}__{name}"); + + let Some(import) = import_map.get(&qualified_name) else { + bail!("func not found {module} {name}") + }; + let wavm = vec![ + Instruction::simple(Opcode::InitFrame), + Instruction::with_data( + Opcode::CrossModuleCall, + pack_cross_module_call(import.module, import.func), + ), + Instruction::simple(Opcode::Return), + ]; + Ok(Function::new_from_wavm(wavm, import.ty.clone(), vec![])) + } + } + + fn make_forward_resolver(import_map: &HashMap) -> impl ImportResolver + '_ { + move |module, name| { + let qualified_name = format!("{module}__{name}"); + + let Some(import) = import_map.get(&qualified_name) else { + bail!("func not found {module} {name}") + }; + let wavm = vec![ + Instruction::simple(Opcode::InitFrame), + Instruction::with_data( + Opcode::CrossModuleForward, + pack_cross_module_call(import.module, import.func), + ), + Instruction::simple(Opcode::Return), + ]; + Ok(Function::new_from_wavm(wavm, import.ty.clone(), vec![])) + } + } + + fn hostio_resolver(module: &str, name: &str) -> Result { + if host::hostio_module_is_debug(module) { + return Self::notfound_resolver(module, name); + } + host::get_impl(module, name) + } + + fn debug_resolver(module: &str, name: &str) -> Result { + if !host::hostio_module_is_debug(module) { + return Self::notfound_resolver(module, name); + } + host::get_impl(module, name) + } + + fn notfound_resolver(module: &str, name: &str) -> Result { + bail!("import not found {module} {name}") + } + + fn from_binary( bin: &WasmBinary, - available_imports: &HashMap, + import_resolver: R, floating_point_impls: &FloatingPointImpls, - allow_hostapi: bool, - is_forwarder: bool, - debug_funcs: bool, stylus_data: Option, ) -> Result { let mut code = Vec::new(); @@ -366,38 +447,8 @@ impl Module { let module = import.module; let have_ty = &bin.types[import.offset as usize]; - let qualified_name = format!("{module}__{}", import.name); + let func = import_resolver(import.module, import.name)?; - let func = if let Some(import) = available_imports.get(&qualified_name) { - let call = match is_forwarder { - true => Opcode::CrossModuleForward, - false => Opcode::CrossModuleCall, - }; - let wavm = vec![ - Instruction::simple(Opcode::InitFrame), - Instruction::with_data( - call, - pack_cross_module_call(import.module, import.func), - ), - Instruction::simple(Opcode::Return), - ]; - Function::new_from_wavm(wavm, import.ty.clone(), vec![]) - } else if let Ok((hostio, debug)) = host::get_impl(import.module, import.name) { - ensure!( - (debug && debug_funcs) || (!debug && allow_hostapi), - "Host func {} in {} not enabled debug_funcs={debug_funcs} hostapi={allow_hostapi} debug={debug}", - import.name.red(), - import.module.red(), - ); - hostio - } else { - bail!( - "No such import {} in {} for {}", - import.name.red(), - import.module.red(), - bin_name.red() - ) - }; ensure!( &func.ty == have_ty, "Import {} for {} has different function signature than export.\nexpected {} in {}\nbut have {}", @@ -602,11 +653,9 @@ impl Module { ) -> Result { Self::from_binary( bin, - &USER_IMPORTS, + Module::make_imports_resolver(&USER_IMPORTS) + .compose(Module::debug_resolver.condition(debug_funcs)), &HashMap::default(), - false, - false, - debug_funcs, stylus_data, ) } @@ -1377,11 +1426,8 @@ impl Machine { if let Some(lib) = forwarder { modules.push(Module::from_binary( &lib, - &available_imports, + Module::make_forward_resolver(&available_imports), &floating_point_impls, - true, - true, - debug_funcs, None, )?); } @@ -1389,22 +1435,20 @@ impl Machine { for lib in libraries.iter() { modules.push(Module::from_binary( lib, - &available_imports, + Module::hostio_resolver + .compose(Module::debug_resolver.condition(debug_funcs)) + .compose(Module::make_imports_resolver(&available_imports)), &floating_point_impls, - true, - false, - debug_funcs, None, )?); } modules.push(Module::from_binary( &bin, - &available_imports, + Module::make_imports_resolver(&available_imports) + .compose(Module::hostio_resolver.condition(allow_hostapi_from_main)) + .compose(Module::debug_resolver.condition(debug_funcs)), &floating_point_impls, - allow_hostapi_from_main, - false, - debug_funcs, stylus_data, )?); From 06954ca91f186d8bcf58bd3a1bc4611f719ea6e5 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 11 Sep 2024 18:54:08 -0600 Subject: [PATCH 169/489] cargo fmt --- arbitrator/prover/src/machine.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/arbitrator/prover/src/machine.rs b/arbitrator/prover/src/machine.rs index 9f2ba081dc..2418d3cc2e 100644 --- a/arbitrator/prover/src/machine.rs +++ b/arbitrator/prover/src/machine.rs @@ -375,7 +375,9 @@ trait ImportResolver: for<'a, 'b> Fn(&'a str, &'b str) -> Result + Siz impl Fn(&'a str, &'b str) -> Result> ImportResolver for F {} impl Module { - fn make_imports_resolver(import_map: &HashMap) -> impl ImportResolver + '_ { + fn make_imports_resolver( + import_map: &HashMap, + ) -> impl ImportResolver + '_ { move |module, name| { let qualified_name = format!("{module}__{name}"); @@ -394,7 +396,9 @@ impl Module { } } - fn make_forward_resolver(import_map: &HashMap) -> impl ImportResolver + '_ { + fn make_forward_resolver( + import_map: &HashMap, + ) -> impl ImportResolver + '_ { move |module, name| { let qualified_name = format!("{module}__{name}"); From 6d0ee874def40855f2be93bf89625bc56854f48b Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 11 Sep 2024 19:18:36 -0600 Subject: [PATCH 170/489] block_validator: nits --- staker/stateless_block_validator.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/staker/stateless_block_validator.go b/staker/stateless_block_validator.go index 3d92b9df59..5c92fb4025 100644 --- a/staker/stateless_block_validator.go +++ b/staker/stateless_block_validator.go @@ -195,7 +195,7 @@ func newValidationEntry( Number: batchNum, Data: fullBatchInfo.PostedData, }) - clonePreimagesInto(preimages, fullBatchInfo.Preimages) + copyPreimagesInto(preimages, fullBatchInfo.Preimages) return fullBatchInfo.PostedData, nil } @@ -326,10 +326,10 @@ func (v *StatelessBlockValidator) readBatch(ctx context.Context, batchNum uint64 return true, &fullInfo, nil } -func clonePreimagesInto(dest, source map[arbutil.PreimageType]map[common.Hash][]byte) { +func copyPreimagesInto(dest, source map[arbutil.PreimageType]map[common.Hash][]byte) { for piType, piMap := range source { if dest[piType] == nil { - dest[piType] = make(map[common.Hash][]byte, len(source[piType])) + dest[piType] = make(map[common.Hash][]byte, len(piMap)) } for hash, preimage := range piMap { dest[piType][hash] = preimage @@ -350,9 +350,10 @@ func (v *StatelessBlockValidator) ValidationEntryRecord(ctx context.Context, e * return fmt.Errorf("recording failed: pos %d, hash expected %v, got %v", e.Pos, e.End.BlockHash, recording.BlockHash) } if recording.Preimages != nil { - recordingPreimages := make(map[arbutil.PreimageType]map[common.Hash][]byte) - recordingPreimages[arbutil.Keccak256PreimageType] = recording.Preimages - clonePreimagesInto(e.Preimages, recordingPreimages) + recordingPreimages := map[arbutil.PreimageType]map[common.Hash][]byte{ + arbutil.Keccak256PreimageType: recording.Preimages, + } + copyPreimagesInto(e.Preimages, recordingPreimages) } e.UserWasms = recording.UserWasms } From 5591e5a1c4e787c493e4957b3f73248a2ec704c1 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 11 Sep 2024 21:05:35 -0600 Subject: [PATCH 171/489] block_validator: separate reading full batches from just posted data --- arbos/arbostypes/incomingmessage.go | 11 ++++ staker/block_validator.go | 51 ++++++++++--------- staker/stateless_block_validator.go | 79 ++++++++++++++++++----------- 3 files changed, 86 insertions(+), 55 deletions(-) diff --git a/arbos/arbostypes/incomingmessage.go b/arbos/arbostypes/incomingmessage.go index 04ce8ebe2e..c4c2dc037b 100644 --- a/arbos/arbostypes/incomingmessage.go +++ b/arbos/arbostypes/incomingmessage.go @@ -182,6 +182,17 @@ func (msg *L1IncomingMessage) FillInBatchGasCost(batchFetcher FallibleBatchFetch return nil } +func (msg *L1IncomingMessage) PastBatchesRequired() ([]uint64, error) { + if msg.Header.Kind != L1MessageType_BatchPostingReport { + return nil, nil + } + _, _, _, batchNum, _, _, err := ParseBatchPostingReportMessageFields(bytes.NewReader(msg.L2msg)) + if err != nil { + return nil, fmt.Errorf("failed to parse batch posting report: %w", err) + } + return []uint64{batchNum}, nil +} + func ParseIncomingL1Message(rd io.Reader, batchFetcher FallibleBatchFetcher) (*L1IncomingMessage, error) { var kindBuf [1]byte _, err := rd.Read(kindBuf[:]) diff --git a/staker/block_validator.go b/staker/block_validator.go index 4a5bd740a8..e1b2c75b84 100644 --- a/staker/block_validator.go +++ b/staker/block_validator.go @@ -58,7 +58,7 @@ type BlockValidator struct { // can only be accessed from creation thread or if holding reorg-write nextCreateBatch *FullBatchInfo nextCreateBatchReread bool - prevBatchCache map[uint64]*FullBatchInfo + prevBatchCache map[uint64][]byte nextCreateStartGS validator.GoGlobalState nextCreatePrevDelayed uint64 @@ -275,7 +275,7 @@ func NewBlockValidator( progressValidationsChan: make(chan struct{}, 1), config: config, fatalErr: fatalErr, - prevBatchCache: make(map[uint64]*FullBatchInfo), + prevBatchCache: make(map[uint64][]byte), } if !config().Dangerous.ResetBlockValidation { validated, err := ret.ReadLastValidatedInfo() @@ -576,12 +576,12 @@ func (v *BlockValidator) createNextValidationEntry(ctx context.Context) (bool, e } if v.nextCreateStartGS.PosInBatch == 0 || v.nextCreateBatchReread { // new batch - found, fullBatchInfo, err := v.readBatch(ctx, v.nextCreateStartGS.Batch) + found, fullBatchInfo, err := v.readFullBatch(ctx, v.nextCreateStartGS.Batch) if !found { return false, err } if v.nextCreateBatch != nil { - v.prevBatchCache[v.nextCreateBatch.Number] = v.nextCreateBatch + v.prevBatchCache[v.nextCreateBatch.Number] = v.nextCreateBatch.PostedData } v.nextCreateBatch = fullBatchInfo // #nosec G115 @@ -589,7 +589,7 @@ func (v *BlockValidator) createNextValidationEntry(ctx context.Context) (bool, e batchCacheLimit := v.config().BatchCacheLimit if len(v.prevBatchCache) > int(batchCacheLimit) { for num := range v.prevBatchCache { - if num < v.nextCreateStartGS.Batch-uint64(batchCacheLimit) { + if num+uint64(batchCacheLimit) < v.nextCreateStartGS.Batch { delete(v.prevBatchCache, num) } } @@ -610,26 +610,29 @@ func (v *BlockValidator) createNextValidationEntry(ctx context.Context) (bool, e return false, fmt.Errorf("illegal batch msg count %d pos %d batch %d", v.nextCreateBatch.MsgCount, pos, endGS.Batch) } chainConfig := v.streamer.ChainConfig() - batchReader := func(batchNum uint64) (*FullBatchInfo, error) { - if batchNum == v.nextCreateBatch.Number { - return v.nextCreateBatch, nil - } - // only batch-posting-reports will get here, and there's only one per batch - if entry, found := v.prevBatchCache[batchNum]; found { + prevBatchNums, err := msg.Message.PastBatchesRequired() + if err != nil { + return false, err + } + prevBatches := make([]validator.BatchInfo, 0, len(prevBatchNums)) + // prevBatchNums are only used for batch reports, each is only used once + for _, batchNum := range prevBatchNums { + data, found := v.prevBatchCache[batchNum] + if found { delete(v.prevBatchCache, batchNum) - return entry, nil - } - found, entry, err := v.readBatch(ctx, batchNum) - if err != nil { - return nil, err - } - if !found { - return nil, fmt.Errorf("batch %d not found", batchNum) + } else { + data, err = v.readPostedBatch(ctx, batchNum) + if err != nil { + return false, err + } } - return entry, nil + prevBatches = append(prevBatches, validator.BatchInfo{ + Number: batchNum, + Data: data, + }) } entry, err := newValidationEntry( - pos, v.nextCreateStartGS, endGS, msg, batchReader, v.nextCreatePrevDelayed, chainConfig, + pos, v.nextCreateStartGS, endGS, msg, v.nextCreateBatch, prevBatches, v.nextCreatePrevDelayed, chainConfig, ) if err != nil { return false, err @@ -1030,7 +1033,7 @@ func (v *BlockValidator) UpdateLatestStaked(count arbutil.MessageIndex, globalSt v.nextCreatePrevDelayed = msg.DelayedMessagesRead v.nextCreateBatchReread = true if v.nextCreateBatch != nil { - v.prevBatchCache[v.nextCreateBatch.Number] = v.nextCreateBatch + v.prevBatchCache[v.nextCreateBatch.Number] = v.nextCreateBatch.PostedData } v.createdA.Store(countUint64) } @@ -1058,7 +1061,7 @@ func (v *BlockValidator) ReorgToBatchCount(count uint64) { defer v.reorgMutex.Unlock() if v.nextCreateStartGS.Batch >= count { v.nextCreateBatchReread = true - v.prevBatchCache = make(map[uint64]*FullBatchInfo) + v.prevBatchCache = make(map[uint64][]byte) } } @@ -1099,7 +1102,7 @@ func (v *BlockValidator) Reorg(ctx context.Context, count arbutil.MessageIndex) v.nextCreateStartGS = buildGlobalState(*res, endPosition) v.nextCreatePrevDelayed = msg.DelayedMessagesRead v.nextCreateBatchReread = true - v.prevBatchCache = make(map[uint64]*FullBatchInfo) + v.prevBatchCache = make(map[uint64][]byte) countUint64 := uint64(count) v.createdA.Store(countUint64) // under the reorg mutex we don't need atomic access diff --git a/staker/stateless_block_validator.go b/staker/stateless_block_validator.go index 5c92fb4025..60306d712f 100644 --- a/staker/stateless_block_validator.go +++ b/staker/stateless_block_validator.go @@ -116,7 +116,6 @@ const ( type FullBatchInfo struct { Number uint64 - BlockHash common.Hash PostedData []byte MsgCount arbutil.MessageIndex Preimages map[arbutil.PreimageType]map[common.Hash][]byte @@ -179,25 +178,27 @@ func newValidationEntry( start validator.GoGlobalState, end validator.GoGlobalState, msg *arbostypes.MessageWithMetadata, - fullBatchFetcher func(uint64) (*FullBatchInfo, error), + fullBatchInfo *FullBatchInfo, + prevBatches []validator.BatchInfo, prevDelayed uint64, chainConfig *params.ChainConfig, ) (*validationEntry, error) { preimages := make(map[arbutil.PreimageType]map[common.Hash][]byte) - valBatches := make([]validator.BatchInfo, 0) - - valBatchFetcher := func(batchNum uint64) ([]byte, error) { - fullBatchInfo, err := fullBatchFetcher(batchNum) - if err != nil { - return nil, err - } - valBatches = append(valBatches, validator.BatchInfo{ - Number: batchNum, + if fullBatchInfo == nil { + return nil, fmt.Errorf("fullbatchInfo cannot be nil") + } + if fullBatchInfo.Number != start.Batch { + return nil, fmt.Errorf("got wrong batch expected: %d got: %d", start.Batch, fullBatchInfo.Number) + } + valBatches := []validator.BatchInfo{ + { + Number: fullBatchInfo.Number, Data: fullBatchInfo.PostedData, - }) - copyPreimagesInto(preimages, fullBatchInfo.Preimages) - return fullBatchInfo.PostedData, nil + }, } + valBatches = append(valBatches, prevBatches...) + + copyPreimagesInto(preimages, fullBatchInfo.Preimages) hasDelayed := false var delayedNum uint64 @@ -208,14 +209,6 @@ func newValidationEntry( return nil, fmt.Errorf("illegal validation entry delayedMessage %d, previous %d", msg.DelayedMessagesRead, prevDelayed) } - if _, err := valBatchFetcher(start.Batch); err != nil { - return nil, err - } - msg.Message.BatchGasCost = nil - if err := msg.Message.FillInBatchGasCost(valBatchFetcher); err != nil { - return nil, err - } - return &validationEntry{ Stage: ReadyForRecord, Pos: pos, @@ -274,7 +267,19 @@ func NewStatelessBlockValidator( }, nil } -func (v *StatelessBlockValidator) readBatch(ctx context.Context, batchNum uint64) (bool, *FullBatchInfo, error) { +func (v *StatelessBlockValidator) readPostedBatch(ctx context.Context, batchNum uint64) ([]byte, error) { + batchCount, err := v.inboxTracker.GetBatchCount() + if err != nil { + return nil, err + } + if batchCount <= batchNum { + return nil, fmt.Errorf("batch not found: %d", batchNum) + } + postedData, _, err := v.inboxReader.GetSequencerMessageBytes(ctx, batchNum) + return postedData, err +} + +func (v *StatelessBlockValidator) readFullBatch(ctx context.Context, batchNum uint64) (bool, *FullBatchInfo, error) { batchCount, err := v.inboxTracker.GetBatchCount() if err != nil { return false, nil, err @@ -318,7 +323,6 @@ func (v *StatelessBlockValidator) readBatch(ctx context.Context, batchNum uint64 } fullInfo := FullBatchInfo{ Number: batchNum, - BlockHash: batchBlockHash, PostedData: postedData, MsgCount: batchMsgCount, Preimages: preimages, @@ -427,17 +431,30 @@ func (v *StatelessBlockValidator) CreateReadyValidationEntry(ctx context.Context } start := buildGlobalState(*prevResult, startPos) end := buildGlobalState(*result, endPos) - fullBatchReader := func(batchNum uint64) (*FullBatchInfo, error) { - found, fullBlockInfo, err := v.readBatch(ctx, batchNum) + found, fullBatchInfo, err := v.readFullBatch(ctx, start.Batch) + if err != nil { + return nil, err + } + if !found { + return nil, fmt.Errorf("batch %d not found", startPos.BatchNumber) + } + + prevBatchNums, err := msg.Message.PastBatchesRequired() + if err != nil { + return nil, err + } + prevBatches := make([]validator.BatchInfo, 0, len(prevBatchNums)) + for _, batchNum := range prevBatchNums { + data, err := v.readPostedBatch(ctx, batchNum) if err != nil { return nil, err } - if !found { - return nil, fmt.Errorf("batch %d not found", startPos.BatchNumber) - } - return fullBlockInfo, nil + prevBatches = append(prevBatches, validator.BatchInfo{ + Number: batchNum, + Data: data, + }) } - entry, err := newValidationEntry(pos, start, end, msg, fullBatchReader, prevDelayed, v.streamer.ChainConfig()) + entry, err := newValidationEntry(pos, start, end, msg, fullBatchInfo, prevBatches, prevDelayed, v.streamer.ChainConfig()) if err != nil { return nil, err } From 7553cec93b4d29c6ee68d0192ab586ad3fcac9d6 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 11 Sep 2024 21:19:48 -0600 Subject: [PATCH 172/489] go-ethereum: remove koanf from recordingdb --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 81114dde8a..c7539d8f7d 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 81114dde8a26bae90c188605c4a36d5919a4a265 +Subproject commit c7539d8f7dfa3070a050220588cbe3fa20e2ce08 From 0b518804ca523207cdb4cb0e96541578033aa0db Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Wed, 11 Sep 2024 23:01:27 -0500 Subject: [PATCH 173/489] Geth: pull in change to bubble up snapshot db errors --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 81114dde8a..4002f12d88 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 81114dde8a26bae90c188605c4a36d5919a4a265 +Subproject commit 4002f12d8816a2851413ec1c0fcda387090c216b From 3342f18c7bf1bb8e3cd23adb8d458d66e12e6f0e Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Wed, 11 Sep 2024 23:15:17 -0500 Subject: [PATCH 174/489] Only warn on potentially incompatible WASM module roots --- cmd/nitro/nitro.go | 110 +++++++++++++++++++++++---------------------- 1 file changed, 56 insertions(+), 54 deletions(-) diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index e66d99b56e..a13a92fdc1 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -437,61 +437,9 @@ func mainImpl() int { // Check that node is compatible with on-chain WASM module root on startup and before any ArbOS upgrades take effect to prevent divergences if nodeConfig.Node.ParentChainReader.Enable && nodeConfig.Validation.Wasm.EnableWasmrootsCheck { - // Fetch current on-chain WASM module root - rollupUserLogic, err := rollupgen.NewRollupUserLogic(rollupAddrs.Rollup, l1Client) + err := checkWasmModuleRootCompatibility(ctx, nodeConfig.Validation.Wasm, l1Client, rollupAddrs) if err != nil { - log.Error("failed to create rollupUserLogic", "err", err) - return 1 - } - moduleRoot, err := rollupUserLogic.WasmModuleRoot(&bind.CallOpts{Context: ctx}) - if err != nil { - log.Error("failed to get on-chain WASM module root", "err", err) - return 1 - } - if (moduleRoot == common.Hash{}) { - log.Error("on-chain WASM module root is zero") - return 1 - } - // Check if the on-chain WASM module root belongs to the set of allowed module roots - allowedWasmModuleRoots := nodeConfig.Validation.Wasm.AllowedWasmModuleRoots - if len(allowedWasmModuleRoots) > 0 { - moduleRootMatched := false - for _, root := range allowedWasmModuleRoots { - bytes, err := hex.DecodeString(strings.TrimPrefix(root, "0x")) - if err == nil { - if common.HexToHash(root) == common.BytesToHash(bytes) { - moduleRootMatched = true - break - } - continue - } - locator, locatorErr := server_common.NewMachineLocator(root) - if locatorErr != nil { - log.Warn("allowed-wasm-module-roots: value not a hex nor valid path:", "value", root, "locatorErr", locatorErr, "decodeErr", err) - continue - } - path := locator.GetMachinePath(moduleRoot) - if _, err := os.Stat(path); err == nil { - moduleRootMatched = true - break - } - } - if !moduleRootMatched { - log.Error("on-chain WASM module root did not match with any of the allowed WASM module roots") - return 1 - } - } else { - // If no allowed module roots were provided in config, check if we have a validator machine directory for the on-chain WASM module root - locator, err := server_common.NewMachineLocator(nodeConfig.Validation.Wasm.RootPath) - if err != nil { - log.Warn("failed to create machine locator. Skipping the check for compatibility with on-chain WASM module root", "err", err) - } else { - path := locator.GetMachinePath(moduleRoot) - if _, err := os.Stat(path); err != nil { - log.Error("unable to find validator machine directory for the on-chain WASM module root", "err", err) - return 1 - } - } + log.Warn("failed to check if node is compatible with on-chain WASM module root", "err", err) } } @@ -1078,3 +1026,57 @@ type NodeConfigFetcher struct { func (f *NodeConfigFetcher) Get() *arbnode.Config { return &f.LiveConfig.Get().Node } + +func checkWasmModuleRootCompatibility(ctx context.Context, wasmConfig valnode.WasmConfig, l1Client *ethclient.Client, rollupAddrs chaininfo.RollupAddresses) error { + // Fetch current on-chain WASM module root + rollupUserLogic, err := rollupgen.NewRollupUserLogic(rollupAddrs.Rollup, l1Client) + if err != nil { + return fmt.Errorf("failed to create RollupUserLogic: %w", err) + } + moduleRoot, err := rollupUserLogic.WasmModuleRoot(&bind.CallOpts{Context: ctx}) + if err != nil { + return fmt.Errorf("failed to get on-chain WASM module root: %w", err) + } + if (moduleRoot == common.Hash{}) { + return errors.New("on-chain WASM module root is zero") + } + // Check if the on-chain WASM module root belongs to the set of allowed module roots + allowedWasmModuleRoots := wasmConfig.AllowedWasmModuleRoots + if len(allowedWasmModuleRoots) > 0 { + moduleRootMatched := false + for _, root := range allowedWasmModuleRoots { + bytes, err := hex.DecodeString(strings.TrimPrefix(root, "0x")) + if err == nil { + if common.HexToHash(root) == common.BytesToHash(bytes) { + moduleRootMatched = true + break + } + continue + } + locator, locatorErr := server_common.NewMachineLocator(root) + if locatorErr != nil { + log.Warn("allowed-wasm-module-roots: value not a hex nor valid path:", "value", root, "locatorErr", locatorErr, "decodeErr", err) + continue + } + path := locator.GetMachinePath(moduleRoot) + if _, err := os.Stat(path); err == nil { + moduleRootMatched = true + break + } + } + if !moduleRootMatched { + return errors.New("on-chain WASM module root did not match with any of the allowed WASM module roots") + } + } else { + // If no allowed module roots were provided in config, check if we have a validator machine directory for the on-chain WASM module root + locator, err := server_common.NewMachineLocator(wasmConfig.RootPath) + if err != nil { + return fmt.Errorf("failed to create machine locator: %w", err) + } + path := locator.GetMachinePath(moduleRoot) + if _, err := os.Stat(path); err != nil { + return fmt.Errorf("unable to find validator machine directory for the on-chain WASM module root: %w", err) + } + } + return nil +} From 5d7a330c86b1f238d30106a3d00d140e582247b7 Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Thu, 12 Sep 2024 13:45:43 +0800 Subject: [PATCH 175/489] fix: deploy both erc20 based and eth based templates --- cmd/deploy/deploy.go | 2 -- deploy/deploy.go | 21 +++++++++++++-------- system_tests/common_test.go | 1 - 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/cmd/deploy/deploy.go b/cmd/deploy/deploy.go index d8c0aeeac4..e0388e3d4b 100644 --- a/cmd/deploy/deploy.go +++ b/cmd/deploy/deploy.go @@ -61,7 +61,6 @@ func main() { authorizevalidators := flag.Uint64("authorizevalidators", 0, "Number of validators to preemptively authorize") txTimeout := flag.Duration("txtimeout", 10*time.Minute, "Timeout when waiting for a transaction to be included in a block") prod := flag.Bool("prod", false, "Whether to configure the rollup for production or testing") - isUsingFeeToken := flag.Bool("isUsingFeeToken", false, "true if the chain uses custom fee token") flag.Parse() l1ChainId := new(big.Int).SetUint64(*l1ChainIdUint) maxDataSize := new(big.Int).SetUint64(*maxDataSizeUint) @@ -190,7 +189,6 @@ func main() { arbnode.GenerateRollupConfig(*prod, moduleRoot, ownerAddress, &chainConfig, chainConfigJson, loserEscrowAddress), nativeToken, maxDataSize, - *isUsingFeeToken, ) if err != nil { flag.Usage() diff --git a/deploy/deploy.go b/deploy/deploy.go index 5e7755cae3..f7ae6d0b59 100644 --- a/deploy/deploy.go +++ b/deploy/deploy.go @@ -31,7 +31,7 @@ func andTxSucceeded(ctx context.Context, l1Reader *headerreader.HeaderReader, tx return nil } -func deployBridgeCreator(ctx context.Context, l1Reader *headerreader.HeaderReader, auth *bind.TransactOpts, maxDataSize *big.Int, isUsingFeeToken bool) (common.Address, error) { +func deployBridgeCreator(ctx context.Context, l1Reader *headerreader.HeaderReader, auth *bind.TransactOpts, maxDataSize *big.Int) (common.Address, error) { client := l1Reader.Client() /// deploy eth based templates @@ -46,7 +46,12 @@ func deployBridgeCreator(ctx context.Context, l1Reader *headerreader.HeaderReade if err != nil { return common.Address{}, fmt.Errorf("blob basefee reader deploy error: %w", err) } - seqInboxTemplate, tx, _, err := bridgegen.DeploySequencerInbox(auth, client, maxDataSize, reader4844, isUsingFeeToken) + seqInboxTemplateEthBased, tx, _, err := bridgegen.DeploySequencerInbox(auth, client, maxDataSize, reader4844, false) + err = andTxSucceeded(ctx, l1Reader, tx, err) + if err != nil { + return common.Address{}, fmt.Errorf("sequencer inbox deploy error: %w", err) + } + seqInboxTemplateERC20Based, tx, _, err := bridgegen.DeploySequencerInbox(auth, client, maxDataSize, reader4844, true) err = andTxSucceeded(ctx, l1Reader, tx, err) if err != nil { return common.Address{}, fmt.Errorf("sequencer inbox deploy error: %w", err) @@ -72,7 +77,7 @@ func deployBridgeCreator(ctx context.Context, l1Reader *headerreader.HeaderReade ethBasedTemplates := rollupgen.BridgeCreatorBridgeContracts{ Bridge: bridgeTemplate, - SequencerInbox: seqInboxTemplate, + SequencerInbox: seqInboxTemplateEthBased, Inbox: inboxTemplate, RollupEventInbox: rollupEventBridgeTemplate, Outbox: outboxTemplate, @@ -105,7 +110,7 @@ func deployBridgeCreator(ctx context.Context, l1Reader *headerreader.HeaderReade erc20BasedTemplates := rollupgen.BridgeCreatorBridgeContracts{ Bridge: erc20BridgeTemplate, - SequencerInbox: seqInboxTemplate, + SequencerInbox: seqInboxTemplateERC20Based, Inbox: erc20InboxTemplate, RollupEventInbox: erc20RollupEventBridgeTemplate, Outbox: erc20OutboxTemplate, @@ -161,8 +166,8 @@ func deployChallengeFactory(ctx context.Context, l1Reader *headerreader.HeaderRe return ospEntryAddr, challengeManagerAddr, nil } -func deployRollupCreator(ctx context.Context, l1Reader *headerreader.HeaderReader, auth *bind.TransactOpts, maxDataSize *big.Int, isUsingFeeToken bool) (*rollupgen.RollupCreator, common.Address, common.Address, common.Address, error) { - bridgeCreator, err := deployBridgeCreator(ctx, l1Reader, auth, maxDataSize, isUsingFeeToken) +func deployRollupCreator(ctx context.Context, l1Reader *headerreader.HeaderReader, auth *bind.TransactOpts, maxDataSize *big.Int) (*rollupgen.RollupCreator, common.Address, common.Address, common.Address, error) { + bridgeCreator, err := deployBridgeCreator(ctx, l1Reader, auth, maxDataSize) if err != nil { return nil, common.Address{}, common.Address{}, common.Address{}, fmt.Errorf("bridge creator deploy error: %w", err) } @@ -234,12 +239,12 @@ func deployRollupCreator(ctx context.Context, l1Reader *headerreader.HeaderReade return rollupCreator, rollupCreatorAddress, validatorUtils, validatorWalletCreator, nil } -func DeployOnL1(ctx context.Context, parentChainReader *headerreader.HeaderReader, deployAuth *bind.TransactOpts, batchPosters []common.Address, batchPosterManager common.Address, authorizeValidators uint64, config rollupgen.Config, nativeToken common.Address, maxDataSize *big.Int, isUsingFeeToken bool) (*chaininfo.RollupAddresses, error) { +func DeployOnL1(ctx context.Context, parentChainReader *headerreader.HeaderReader, deployAuth *bind.TransactOpts, batchPosters []common.Address, batchPosterManager common.Address, authorizeValidators uint64, config rollupgen.Config, nativeToken common.Address, maxDataSize *big.Int) (*chaininfo.RollupAddresses, error) { if config.WasmModuleRoot == (common.Hash{}) { return nil, errors.New("no machine specified") } - rollupCreator, _, validatorUtils, validatorWalletCreator, err := deployRollupCreator(ctx, parentChainReader, deployAuth, maxDataSize, isUsingFeeToken) + rollupCreator, _, validatorUtils, validatorWalletCreator, err := deployRollupCreator(ctx, parentChainReader, deployAuth, maxDataSize) if err != nil { return nil, fmt.Errorf("error deploying rollup creator: %w", err) } diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 7a78cee309..6975990be0 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -1040,7 +1040,6 @@ func DeployOnTestL1( arbnode.GenerateRollupConfig(prodConfirmPeriodBlocks, wasmModuleRoot, l1info.GetAddress("RollupOwner"), chainConfig, serializedChainConfig, common.Address{}), nativeToken, maxDataSize, - false, ) Require(t, err) l1info.SetContract("Bridge", addresses.Bridge) From e92300cb2c652846295af6de7da692a6b901cf3e Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Thu, 12 Sep 2024 13:47:04 +0800 Subject: [PATCH 176/489] chore: error message --- deploy/deploy.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/deploy.go b/deploy/deploy.go index f7ae6d0b59..f2099f976a 100644 --- a/deploy/deploy.go +++ b/deploy/deploy.go @@ -49,12 +49,12 @@ func deployBridgeCreator(ctx context.Context, l1Reader *headerreader.HeaderReade seqInboxTemplateEthBased, tx, _, err := bridgegen.DeploySequencerInbox(auth, client, maxDataSize, reader4844, false) err = andTxSucceeded(ctx, l1Reader, tx, err) if err != nil { - return common.Address{}, fmt.Errorf("sequencer inbox deploy error: %w", err) + return common.Address{}, fmt.Errorf("sequencer inbox eth based deploy error: %w", err) } seqInboxTemplateERC20Based, tx, _, err := bridgegen.DeploySequencerInbox(auth, client, maxDataSize, reader4844, true) err = andTxSucceeded(ctx, l1Reader, tx, err) if err != nil { - return common.Address{}, fmt.Errorf("sequencer inbox deploy error: %w", err) + return common.Address{}, fmt.Errorf("sequencer inbox erc20 based deploy error: %w", err) } inboxTemplate, tx, _, err := bridgegen.DeployInbox(auth, client, maxDataSize) From beca92d21bfa6d1666f64cadd31334244e8fd98a Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Thu, 12 Sep 2024 11:43:15 -0300 Subject: [PATCH 177/489] Check for context-deadline exceeded error --- system_tests/das_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/system_tests/das_test.go b/system_tests/das_test.go index 4f9bc8ab9f..d59fca3e47 100644 --- a/system_tests/das_test.go +++ b/system_tests/das_test.go @@ -6,6 +6,7 @@ package arbtest import ( "context" "encoding/base64" + "errors" "io" "math/big" "net" @@ -381,8 +382,8 @@ func TestDASBatchPosterFallback(t *testing.T) { // Send 2nd transaction and check it doesn't arrive on second node tx, _ := TransferBalanceTo(t, "Owner", l2info.GetAddress("User2"), big.NewInt(1e12), l2info, l2client, ctx) _, err = WaitForTx(ctx, l2B.Client, tx.Hash(), time.Second*3) - if err == nil { - Fatal(t, "expected error but got nil") + if err == nil || !errors.Is(err, context.DeadlineExceeded) { + Fatal(t, "expected context-deadline exceeded error, but got:", err) } // Enable the DAP fallback and check the transaction on the second node. From 4c0c9b36973c026db464918eaa1034324b2ce39e Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Thu, 12 Sep 2024 10:34:28 -0500 Subject: [PATCH 178/489] Bump geth pin --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 4002f12d88..7e7d4ff4cd 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 4002f12d8816a2851413ec1c0fcda387090c216b +Subproject commit 7e7d4ff4cd440d5e2e294f65cfe60e1a37ba1bb8 From 374fa8ca43369eac3d0a947b68d177b2d4acd9ef Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Thu, 12 Sep 2024 11:17:16 -0500 Subject: [PATCH 179/489] Bump geth pin --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 7e7d4ff4cd..ad31b75098 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 7e7d4ff4cd440d5e2e294f65cfe60e1a37ba1bb8 +Subproject commit ad31b75098f874c7124e7b29b47db8662d47b9ff From a9b455ca5328cf31a4a243220588ed513ebc8dbf Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Thu, 12 Sep 2024 12:19:08 -0600 Subject: [PATCH 180/489] makefile lint --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9ecba1ce2d..b736b04906 100644 --- a/Makefile +++ b/Makefile @@ -479,7 +479,7 @@ contracts/test/prover/proofs/rust-%.json: $(arbitrator_cases)/rust/$(wasm32_wasi $(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --allow-hostapi --require-success --inbox-add-stub-headers --inbox $(arbitrator_cases)/rust/data/msg0.bin --inbox $(arbitrator_cases)/rust/data/msg1.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg0.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg1.bin --preimages target/testdata/preimages.bin --with-forwarder contracts/test/prover/proofs/go.json: $(arbitrator_cases)/go/testcase.wasm $(prover_bin) $(arbitrator_wasm_libs) target/testdata/preimages.bin $(arbitrator_tests_link_deps) $(arbitrator_cases)/user.wasm - $(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --require-success --preimages target/testdata/preimages.bin --stylus-modules $(arbitrator_cases)/user.wasm --with-forwarder + $(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --require-success --preimages target/testdata/preimages.bin --stylus-modules $(arbitrator_cases)/user.wasm --with-forwarder # avoid testing user.wasm in onestepproofs. It can only run as stylus program. contracts/test/prover/proofs/user.json: From d156b8081ebc5ebad47b6634317033fe21f7ad12 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Thu, 12 Sep 2024 12:40:39 -0600 Subject: [PATCH 181/489] add traces to workers --- pubsub/consumer.go | 6 ++++++ validator/valnode/redis/consumer.go | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/pubsub/consumer.go b/pubsub/consumer.go index df3695606d..1fccae56ad 100644 --- a/pubsub/consumer.go +++ b/pubsub/consumer.go @@ -77,6 +77,10 @@ func (c *Consumer[Request, Response]) Start(ctx context.Context) { ) } +func (c *Consumer[Request, Response]) Id() string { + return c.id +} + func (c *Consumer[Request, Response]) StopAndWait() { c.StopWaiter.StopAndWait() c.deleteHeartBeat(c.GetParentContext()) @@ -164,10 +168,12 @@ func (c *Consumer[Request, Response]) SetResult(ctx context.Context, messageID s if err != nil { return fmt.Errorf("marshaling result: %w", err) } + log.Trace("consumer: setting result", "cid", c.id, "messageId", messageID) acquired, err := c.client.SetNX(ctx, messageID, resp, c.cfg.ResponseEntryTimeout).Result() if err != nil || !acquired { return fmt.Errorf("setting result for message: %v, error: %w", messageID, err) } + log.Trace("consumer: xack", "cid", c.id, "messageId", messageID) if _, err := c.client.XAck(ctx, c.redisStream, c.redisGroup, messageID).Result(); err != nil { return fmt.Errorf("acking message: %v, error: %w", messageID, err) } diff --git a/validator/valnode/redis/consumer.go b/validator/valnode/redis/consumer.go index 2b025600cc..90bffbb704 100644 --- a/validator/valnode/redis/consumer.go +++ b/validator/valnode/redis/consumer.go @@ -103,11 +103,13 @@ func (s *ValidationServer) Start(ctx_in context.Context) { case <-ready: // Wait until the stream exists and start consuming iteratively. } s.StopWaiter.CallIteratively(func(ctx context.Context) time.Duration { + log.Trace("waiting for request token", "cid", c.Id()) select { case <-ctx.Done(): return 0 case <-requestTokenQueue: } + log.Trace("got request token", "cid", c.Id()) req, err := c.Consume(ctx) if err != nil { log.Error("Consuming request", "error", err) @@ -115,10 +117,12 @@ func (s *ValidationServer) Start(ctx_in context.Context) { return 0 } if req == nil { + log.Trace("consumed nil", "cid", c.Id()) // There's nothing in the queue requestTokenQueue <- struct{}{} return time.Second } + log.Trace("forwarding work", "cid", c.Id(), "workid", req.ID) select { case <-ctx.Done(): case workQueue <- workUnit{req, moduleRoot}: @@ -144,20 +148,24 @@ func (s *ValidationServer) Start(ctx_in context.Context) { for i := 0; i < workers; i++ { s.StopWaiter.LaunchThread(func(ctx context.Context) { for { + log.Trace("waiting for work", "thread", i) var work workUnit select { case <-ctx.Done(): return case work = <-workQueue: } + log.Trace("got work", "thread", i, "workid", work.req.ID) valRun := s.spawner.Launch(work.req.Value, work.moduleRoot) res, err := valRun.Await(ctx) if err != nil { log.Error("Error validating", "request value", work.req.Value, "error", err) } else { + log.Trace("done work", "thread", i, "workid", work.req.ID) if err := s.consumers[work.moduleRoot].SetResult(ctx, work.req.ID, res); err != nil { log.Error("Error setting result for request", "id", work.req.ID, "result", res, "error", err) } + log.Trace("set result", "thread", i, "workid", work.req.ID) } select { case <-ctx.Done(): From 1ab23ac4fcb8bf088244254a2db8b56dd2b4dc90 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Thu, 12 Sep 2024 12:43:13 -0600 Subject: [PATCH 182/489] redis consumer logs to Debug --- pubsub/consumer.go | 4 ++-- validator/valnode/redis/consumer.go | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pubsub/consumer.go b/pubsub/consumer.go index 1fccae56ad..bd73e729e7 100644 --- a/pubsub/consumer.go +++ b/pubsub/consumer.go @@ -168,12 +168,12 @@ func (c *Consumer[Request, Response]) SetResult(ctx context.Context, messageID s if err != nil { return fmt.Errorf("marshaling result: %w", err) } - log.Trace("consumer: setting result", "cid", c.id, "messageId", messageID) + log.Debug("consumer: setting result", "cid", c.id, "messageId", messageID) acquired, err := c.client.SetNX(ctx, messageID, resp, c.cfg.ResponseEntryTimeout).Result() if err != nil || !acquired { return fmt.Errorf("setting result for message: %v, error: %w", messageID, err) } - log.Trace("consumer: xack", "cid", c.id, "messageId", messageID) + log.Debug("consumer: xack", "cid", c.id, "messageId", messageID) if _, err := c.client.XAck(ctx, c.redisStream, c.redisGroup, messageID).Result(); err != nil { return fmt.Errorf("acking message: %v, error: %w", messageID, err) } diff --git a/validator/valnode/redis/consumer.go b/validator/valnode/redis/consumer.go index 90bffbb704..49e72477f5 100644 --- a/validator/valnode/redis/consumer.go +++ b/validator/valnode/redis/consumer.go @@ -103,13 +103,13 @@ func (s *ValidationServer) Start(ctx_in context.Context) { case <-ready: // Wait until the stream exists and start consuming iteratively. } s.StopWaiter.CallIteratively(func(ctx context.Context) time.Duration { - log.Trace("waiting for request token", "cid", c.Id()) + log.Debug("waiting for request token", "cid", c.Id()) select { case <-ctx.Done(): return 0 case <-requestTokenQueue: } - log.Trace("got request token", "cid", c.Id()) + log.Debug("got request token", "cid", c.Id()) req, err := c.Consume(ctx) if err != nil { log.Error("Consuming request", "error", err) @@ -117,12 +117,12 @@ func (s *ValidationServer) Start(ctx_in context.Context) { return 0 } if req == nil { - log.Trace("consumed nil", "cid", c.Id()) + log.Debug("consumed nil", "cid", c.Id()) // There's nothing in the queue requestTokenQueue <- struct{}{} return time.Second } - log.Trace("forwarding work", "cid", c.Id(), "workid", req.ID) + log.Debug("forwarding work", "cid", c.Id(), "workid", req.ID) select { case <-ctx.Done(): case workQueue <- workUnit{req, moduleRoot}: @@ -135,7 +135,7 @@ func (s *ValidationServer) Start(ctx_in context.Context) { for { select { case <-readyStreams: - log.Trace("At least one stream is ready") + log.Debug("At least one stream is ready") return // Don't block Start if at least one of the stream is ready. case <-time.After(s.config.StreamTimeout): log.Error("Waiting for redis streams timed out") @@ -148,24 +148,24 @@ func (s *ValidationServer) Start(ctx_in context.Context) { for i := 0; i < workers; i++ { s.StopWaiter.LaunchThread(func(ctx context.Context) { for { - log.Trace("waiting for work", "thread", i) + log.Debug("waiting for work", "thread", i) var work workUnit select { case <-ctx.Done(): return case work = <-workQueue: } - log.Trace("got work", "thread", i, "workid", work.req.ID) + log.Debug("got work", "thread", i, "workid", work.req.ID) valRun := s.spawner.Launch(work.req.Value, work.moduleRoot) res, err := valRun.Await(ctx) if err != nil { log.Error("Error validating", "request value", work.req.Value, "error", err) } else { - log.Trace("done work", "thread", i, "workid", work.req.ID) + log.Debug("done work", "thread", i, "workid", work.req.ID) if err := s.consumers[work.moduleRoot].SetResult(ctx, work.req.ID, res); err != nil { log.Error("Error setting result for request", "id", work.req.ID, "result", res, "error", err) } - log.Trace("set result", "thread", i, "workid", work.req.ID) + log.Debug("set result", "thread", i, "workid", work.req.ID) } select { case <-ctx.Done(): From 370c6f1494fb5c24a1edf25f300fec9b12b505a1 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Thu, 12 Sep 2024 13:16:43 -0600 Subject: [PATCH 183/489] node config: always add targets in the same order This prevents an errored change in StylusTargetConfig from a change in the order of targets --- execution/gethexec/node.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index 21c2b4bece..355ed7a647 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "reflect" + "sort" "sync/atomic" "testing" @@ -59,6 +60,11 @@ func (c *StylusTargetConfig) Validate() error { for target := range targetsSet { targets = append(targets, target) } + sort.Slice( + targets, + func(i, j int) bool { + return targets[i] < targets[j] + }) c.wasmTargets = targets return nil } From 44ff879590ca80beb6a3e4b64d826690acab660f Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Thu, 12 Sep 2024 14:37:54 -0500 Subject: [PATCH 184/489] Fix setupFastConfirmation not getting called --- staker/fast_confirm.go | 4 ++++ staker/staker.go | 16 +++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/staker/fast_confirm.go b/staker/fast_confirm.go index 88f457f528..5dc7f01205 100644 --- a/staker/fast_confirm.go +++ b/staker/fast_confirm.go @@ -121,10 +121,12 @@ func (f *FastConfirmSafe) tryFastConfirmation(ctx context.Context, blockHash com return err } if alreadyApproved.Cmp(common.Big1) == 0 { + log.Info("Already approved Safe tx hash for fast confirmation, checking if we can execute the Safe tx", "safeHash", safeTxHash, "nodeHash", nodeHash) _, err = f.checkApprovedHashAndExecTransaction(ctx, fastConfirmCallData, safeTxHash) return err } + log.Info("Approving Safe tx hash to fast confirm", "safeHash", safeTxHash, "nodeHash", nodeHash) auth, err := f.builder.Auth(ctx) if err != nil { return err @@ -231,6 +233,7 @@ func (f *FastConfirmSafe) checkApprovedHashAndExecTransaction(ctx context.Contex if err != nil { return false, err } + log.Info("Executing Safe tx to fast confirm", "safeHash", safeTxHash) _, err = f.safe.ExecTransaction( auth, f.wallet.RollupAddress(), @@ -249,5 +252,6 @@ func (f *FastConfirmSafe) checkApprovedHashAndExecTransaction(ctx context.Contex } return true, nil } + log.Info("Not enough Safe tx approvals yet to fast confirm", "safeHash", safeTxHash) return false, nil } diff --git a/staker/staker.go b/staker/staker.go index 6e93d27311..77ca93e02c 100644 --- a/staker/staker.go +++ b/staker/staker.go @@ -268,7 +268,6 @@ type Staker struct { inboxReader InboxReaderInterface statelessBlockValidator *StatelessBlockValidator fatalErr chan<- error - enableFastConfirmation bool fastConfirmSafe *FastConfirmSafe } @@ -363,7 +362,10 @@ func (s *Staker) Initialize(ctx context.Context) error { return err } - return s.blockValidator.InitAssumeValid(stakedInfo.AfterState().GlobalState) + err = s.blockValidator.InitAssumeValid(stakedInfo.AfterState().GlobalState) + if err != nil { + return err + } } return s.setupFastConfirmation(ctx) } @@ -390,9 +392,9 @@ func (s *Staker) setupFastConfirmation(ctx context.Context) error { if err != nil { return fmt.Errorf("getting rollup fast confirmer address: %w", err) } + log.Info("Setting up fast confirmation", "wallet", walletAddress, "fastConfirmer", fastConfirmer) if fastConfirmer == walletAddress { // We can directly fast confirm nodes - s.enableFastConfirmation = true return nil } else if fastConfirmer == (common.Address{}) { // No fast confirmer enabled @@ -419,13 +421,12 @@ func (s *Staker) setupFastConfirmation(ctx context.Context) error { if !isOwner { return fmt.Errorf("staker wallet address %v is not an owner of the fast confirm safe %v", walletAddress, fastConfirmer) } - s.enableFastConfirmation = true s.fastConfirmSafe = fastConfirmSafe return nil } func (s *Staker) tryFastConfirmationNodeNumber(ctx context.Context, number uint64, hash common.Hash) error { - if !s.enableFastConfirmation { + if !s.config().EnableFastConfirmation { return nil } nodeInfo, err := s.rollup.LookupNode(ctx, number) @@ -436,7 +437,7 @@ func (s *Staker) tryFastConfirmationNodeNumber(ctx context.Context, number uint6 } func (s *Staker) tryFastConfirmation(ctx context.Context, blockHash common.Hash, sendRoot common.Hash, nodeHash common.Hash) error { - if !s.enableFastConfirmation { + if !s.config().EnableFastConfirmation { return nil } if s.fastConfirmSafe != nil { @@ -446,6 +447,7 @@ func (s *Staker) tryFastConfirmation(ctx context.Context, blockHash common.Hash, if err != nil { return err } + log.Info("Fast confirming node with wallet", "wallet", auth.From, "nodeHash", nodeHash) _, err = s.rollup.FastConfirmNextNode(auth, blockHash, sendRoot, nodeHash) return err } @@ -802,13 +804,13 @@ func (s *Staker) Act(ctx context.Context) (*types.Transaction, error) { confirmedCorrect = stakedOnNode } if confirmedCorrect { + log.Info("trying to fast confirm previous node", "node", firstUnresolvedNode, "nodeHash", nodeInfo.NodeHash) err = s.tryFastConfirmationNodeNumber(ctx, firstUnresolvedNode, nodeInfo.NodeHash) if err != nil { return nil, err } if s.builder.BuildingTransactionCount() > 0 { // Try to fast confirm previous nodes before working on new ones - log.Info("fast confirming previous node", "node", firstUnresolvedNode) return s.wallet.ExecuteTransactions(ctx, s.builder, cfg.gasRefunder) } } From ae1ceaecea9bedc48a06e43966a639bf7e069d5f Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Thu, 12 Sep 2024 18:11:30 -0300 Subject: [PATCH 185/489] Enable external-signer for batch-poster with DA When running a batch-poster with anytrust, Nitro needs to load the batch-poster parent-chain wallet config to sign the DA messages even if an external signer is also being used. --- cmd/nitro/nitro.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index e66d99b56e..61f914f14d 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -249,7 +249,7 @@ func mainImpl() int { // If sequencer and signing is enabled or batchposter is enabled without // external signing sequencer will need a key. sequencerNeedsKey := (nodeConfig.Node.Sequencer && !nodeConfig.Node.Feed.Output.DisableSigning) || - (nodeConfig.Node.BatchPoster.Enable && nodeConfig.Node.BatchPoster.DataPoster.ExternalSigner.URL == "") + (nodeConfig.Node.BatchPoster.Enable && (nodeConfig.Node.BatchPoster.DataPoster.ExternalSigner.URL == "" || nodeConfig.Node.DataAvailability.Enable)) validatorNeedsKey := nodeConfig.Node.Staker.OnlyCreateWalletContract || (nodeConfig.Node.Staker.Enable && !strings.EqualFold(nodeConfig.Node.Staker.Strategy, "watchtower") && nodeConfig.Node.Staker.DataPoster.ExternalSigner.URL == "") From 7c3b4be042313c36bf7c23553327aeb8c4bc53d6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Sep 2024 21:28:27 +0000 Subject: [PATCH 186/489] Bump rustix from 0.37.23 to 0.37.27 in /arbitrator/stylus/tests/erc20 Bumps [rustix](https://github.com/bytecodealliance/rustix) from 0.37.23 to 0.37.27. - [Release notes](https://github.com/bytecodealliance/rustix/releases) - [Changelog](https://github.com/bytecodealliance/rustix/blob/main/CHANGELOG.md) - [Commits](https://github.com/bytecodealliance/rustix/compare/v0.37.23...v0.37.27) --- updated-dependencies: - dependency-name: rustix dependency-type: indirect ... Signed-off-by: dependabot[bot] --- arbitrator/stylus/tests/erc20/Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arbitrator/stylus/tests/erc20/Cargo.lock b/arbitrator/stylus/tests/erc20/Cargo.lock index c3e215978d..f5e1e0b15e 100644 --- a/arbitrator/stylus/tests/erc20/Cargo.lock +++ b/arbitrator/stylus/tests/erc20/Cargo.lock @@ -575,9 +575,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.23" +version = "0.37.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06" +checksum = "fea8ca367a3a01fe35e6943c400addf443c0f57670e6ec51196f71a4b8762dd2" dependencies = [ "bitflags", "errno", From 7075c7f2e02c8404fadf0243da9282577a2406b6 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Thu, 12 Sep 2024 19:21:42 -0600 Subject: [PATCH 187/489] redis producer: delete entry after result exists --- pubsub/producer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pubsub/producer.go b/pubsub/producer.go index 2b1cdb5e3f..e7f2ac938c 100644 --- a/pubsub/producer.go +++ b/pubsub/producer.go @@ -234,6 +234,7 @@ func (p *Producer[Request, Response]) checkResponses(ctx context.Context) time.D promise.Produce(resp) responded++ } + p.client.Del(ctx, id) delete(p.promises, id) } var trimmed int64 From f5750e7ad22812cf193aabb959badf4a8b25d0cb Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Thu, 12 Sep 2024 19:23:52 -0600 Subject: [PATCH 188/489] producer: debug check responses loop --- pubsub/producer.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pubsub/producer.go b/pubsub/producer.go index e7f2ac938c..5eec3a4b52 100644 --- a/pubsub/producer.go +++ b/pubsub/producer.go @@ -205,30 +205,33 @@ func setMinIdInt(min *[2]uint64, id string) error { // checkResponses checks iteratively whether response for the promise is ready. func (p *Producer[Request, Response]) checkResponses(ctx context.Context) time.Duration { minIdInt := [2]uint64{math.MaxUint64, math.MaxUint64} + log.Debug("redis producer: check responses starting") p.promisesLock.Lock() defer p.promisesLock.Unlock() responded := 0 errored := 0 + checked := 0 for id, promise := range p.promises { if ctx.Err() != nil { return 0 } + checked++ res, err := p.client.Get(ctx, id).Result() if err != nil { errSetId := setMinIdInt(&minIdInt, id) if errSetId != nil { - log.Error("error setting minId", "err", err) + log.Error("redis producer: error setting minId", "err", err) return p.cfg.CheckResultInterval } if !errors.Is(err, redis.Nil) { - log.Error("Error reading value in redis", "key", id, "error", err) + log.Error("redis producer: Error reading value in redis", "key", id, "error", err) } continue } var resp Response if err := json.Unmarshal([]byte(res), &resp); err != nil { promise.ProduceError(fmt.Errorf("error unmarshalling: %w", err)) - log.Error("Error unmarshaling", "value", res, "error", err) + log.Error("redis producer: Error unmarshaling", "value", res, "error", err) errored++ } else { promise.Produce(resp) @@ -246,7 +249,7 @@ func (p *Producer[Request, Response]) checkResponses(ctx context.Context) time.D } else { trimmed, trimErr = p.client.XTrimMaxLen(ctx, p.redisStream, 0).Result() } - log.Trace("trimming", "id", minId, "trimmed", trimmed, "responded", responded, "errored", errored, "trim-err", trimErr) + log.Debug("trimming", "id", minId, "trimmed", trimmed, "responded", responded, "errored", errored, "trim-err", trimErr, "checked", checked) return p.cfg.CheckResultInterval } From 0975c44a39e207dffb5f1f480112e5ed6da5f012 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Thu, 12 Sep 2024 19:29:27 -0600 Subject: [PATCH 189/489] update geth pin --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index c7539d8f7d..9ca65b8610 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit c7539d8f7dfa3070a050220588cbe3fa20e2ce08 +Subproject commit 9ca65b86101b4c87df188923d6eeff4db992520f From b5fa17d9a447d39b08680fdf9807301623aa2d2e Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Thu, 12 Sep 2024 20:25:42 -0600 Subject: [PATCH 190/489] fix race in test --- validator/valnode/redis/consumer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/validator/valnode/redis/consumer.go b/validator/valnode/redis/consumer.go index 49e72477f5..e0d53ffb2e 100644 --- a/validator/valnode/redis/consumer.go +++ b/validator/valnode/redis/consumer.go @@ -146,6 +146,7 @@ func (s *ValidationServer) Start(ctx_in context.Context) { } }) for i := 0; i < workers; i++ { + i := i s.StopWaiter.LaunchThread(func(ctx context.Context) { for { log.Debug("waiting for work", "thread", i) From e84264be43d9c9a8c584548f2df3c17af18324b0 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Fri, 13 Sep 2024 00:00:31 -0500 Subject: [PATCH 191/489] Fix inbox reading window after a reorg --- arbnode/inbox_reader.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/arbnode/inbox_reader.go b/arbnode/inbox_reader.go index c596cfa9b0..835aef18b8 100644 --- a/arbnode/inbox_reader.go +++ b/arbnode/inbox_reader.go @@ -534,14 +534,6 @@ func (r *InboxReader) run(ctx context.Context, hadError bool) error { storeSeenBatchCount() } } - if reorgingDelayed || reorgingSequencer { - from, err = r.getPrevBlockForReorg(from) - if err != nil { - return err - } - } else { - from = arbmath.BigAddByUint(to, 1) - } // #nosec G115 haveMessages := uint64(len(delayedMessages) + len(sequencerBatches)) if haveMessages <= (config.TargetMessagesRead / 2) { @@ -555,6 +547,14 @@ func (r *InboxReader) run(ctx context.Context, hadError bool) error { } else if blocksToFetch > config.MaxBlocksToRead { blocksToFetch = config.MaxBlocksToRead } + if reorgingDelayed || reorgingSequencer { + from, err = r.getPrevBlockForReorg(from, blocksToFetch) + if err != nil { + return err + } + } else { + from = arbmath.BigAddByUint(to, 1) + } } if !readAnyBatches { @@ -578,11 +578,11 @@ func (r *InboxReader) addMessages(ctx context.Context, sequencerBatches []*Seque return false, nil } -func (r *InboxReader) getPrevBlockForReorg(from *big.Int) (*big.Int, error) { +func (r *InboxReader) getPrevBlockForReorg(from *big.Int, maxBlocksBackwards uint64) (*big.Int, error) { if from.Cmp(r.firstMessageBlock) <= 0 { return nil, errors.New("can't get older messages") } - newFrom := arbmath.BigSub(from, big.NewInt(10)) + newFrom := arbmath.BigSub(from, new(big.Int).SetUint64(maxBlocksBackwards)) if newFrom.Cmp(r.firstMessageBlock) < 0 { newFrom = new(big.Int).Set(r.firstMessageBlock) } From 8525ad4228f2f1bfda0397c18a1afa92e4efb36c Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Fri, 13 Sep 2024 15:17:43 +0530 Subject: [PATCH 192/489] address PR comments --- pubsub/consumer.go | 38 ++++++++++++++++++----------- pubsub/pubsub_test.go | 14 +++++------ validator/validation_entry.go | 2 +- validator/valnode/redis/consumer.go | 13 +++++----- 4 files changed, 37 insertions(+), 30 deletions(-) diff --git a/pubsub/consumer.go b/pubsub/consumer.go index 20bfccb6a8..0ec3f11eb7 100644 --- a/pubsub/consumer.go +++ b/pubsub/consumer.go @@ -51,8 +51,9 @@ type Consumer[Request any, Response any] struct { } type Message[Request any] struct { - ID string - Value Request + ID string + Value Request + AckNotifier chan struct{} } func NewConsumer[Request any, Response any](client redis.UniversalClient, streamName string, cfg *ConsumerConfig) (*Consumer[Request, Response], error) { @@ -103,7 +104,7 @@ func decrementMsgIdByOne(msgId string) string { // Consumer first checks it there exists pending message that is claimed by // unresponsive consumer, if not then reads from the stream. -func (c *Consumer[Request, Response]) Consume(ctx context.Context) (*Message[Request], chan struct{}, error) { +func (c *Consumer[Request, Response]) Consume(ctx context.Context) (*Message[Request], error) { // First try to XAUTOCLAIM, with start as a random messageID from PEL with MinIdle as IdletimeToAutoclaim // this prioritizes processing PEL messages that have been waiting for more than IdletimeToAutoclaim duration var messages []redis.XMessage @@ -133,7 +134,7 @@ func (c *Consumer[Request, Response]) Consume(ctx context.Context) (*Message[Req } } if len(messages) == 0 { - // Fallback to reading new messages + // If we fail to autoclaim then we do not retry but instead fallback to reading new messages res, err := c.client.XReadGroup(ctx, &redis.XReadGroupArgs{ Group: c.redisGroup, Consumer: c.id, @@ -144,13 +145,13 @@ func (c *Consumer[Request, Response]) Consume(ctx context.Context) (*Message[Req Block: time.Millisecond, // 0 seems to block the read instead of immediately returning }).Result() if errors.Is(err, redis.Nil) { - return nil, nil, nil + return nil, nil } if err != nil { - return nil, nil, fmt.Errorf("reading message for consumer: %q: %w", c.id, err) + return nil, fmt.Errorf("reading message for consumer: %q: %w", c.id, err) } if len(res) != 1 || len(res[0].Messages) != 1 { - return nil, nil, fmt.Errorf("redis returned entries: %+v, for querying single message", res) + return nil, fmt.Errorf("redis returned entries: %+v, for querying single message", res) } messages = res[0].Messages } @@ -160,11 +161,11 @@ func (c *Consumer[Request, Response]) Consume(ctx context.Context) (*Message[Req data, ok = (value).(string) ) if !ok { - return nil, nil, errors.New("error casting request to string") + return nil, errors.New("error casting request to string") } var req Request if err := json.Unmarshal([]byte(data), &req); err != nil { - return nil, nil, fmt.Errorf("unmarshaling value: %v, error: %w", value, err) + return nil, fmt.Errorf("unmarshaling value: %v, error: %w", value, err) } ackNotifier := make(chan struct{}) c.StopWaiter.LaunchThread(func(ctx context.Context) { @@ -179,14 +180,22 @@ func (c *Consumer[Request, Response]) Consume(ctx context.Context) (*Message[Req Messages: []string{messages[0].ID}, }).Result(); err != nil { log.Error("Error claiming message, it might be possible that other consumers might pick this request", "msgID", messages[0].ID) - } else if len(ids) != 1 { + } else if len(ids) == 0 { log.Warn("XClaimJustID returned empty response when indicating hearbeat", "msgID", messages[0].ID) + } else if len(ids) > 1 { + log.Error("XClaimJustID returned response with more than entry", "msgIDs", ids) } select { case <-ackNotifier: return case <-ctx.Done(): - log.Info("Context done while claiming message to indicate hearbeat", "error", ctx.Err().Error()) + log.Info("Context done while claiming message to indicate hearbeat", "messageID", messages[0].ID, "error", ctx.Err().Error()) + if c.StopWaiter.GetParentContext().Err() == nil { + // Proceeding to set the Idle time of message to IdletimeToAutoclaim to allow it to be picked by other consumers + if err := c.client.Do(c.StopWaiter.GetParentContext(), "XCLAIM", c.redisStream, c.redisGroup, c.id, 0, messages[0].ID, "IDLE", c.cfg.IdletimeToAutoclaim.Milliseconds()).Err(); err != nil { + log.Error("error when trying to set the idle time of currently worked on message to IdletimeToAutoclaim", "messageID", messages[0].ID, "err", err) + } + } return case <-time.After(c.cfg.IdletimeToAutoclaim / 10): } @@ -194,9 +203,10 @@ func (c *Consumer[Request, Response]) Consume(ctx context.Context) (*Message[Req }) log.Debug("Redis stream consuming", "consumer_id", c.id, "message_id", messages[0].ID) return &Message[Request]{ - ID: messages[0].ID, - Value: req, - }, ackNotifier, nil + ID: messages[0].ID, + Value: req, + AckNotifier: ackNotifier, + }, nil } func (c *Consumer[Request, Response]) SetResult(ctx context.Context, id string, messageID string, result Response) error { diff --git a/pubsub/pubsub_test.go b/pubsub/pubsub_test.go index c4e11b8a72..13258aea3d 100644 --- a/pubsub/pubsub_test.go +++ b/pubsub/pubsub_test.go @@ -138,16 +138,14 @@ func flatten(responses [][]string) []string { return ret } -func produceMessages(ctx context.Context, msgs []string, producer *Producer[testRequest, testResponse], useUniqueIdentifier, withInvalidEntries bool) ([]*containers.Promise[testResponse], error) { +func produceMessages(ctx context.Context, msgs []string, producer *Producer[testRequest, testResponse], withInvalidEntries bool) ([]*containers.Promise[testResponse], error) { var promises []*containers.Promise[testResponse] for i := 0; i < len(msgs); i++ { req := testRequest{Request: msgs[i]} if withInvalidEntries && i%50 == 0 { req.IsInvalid = true } - if useUniqueIdentifier { - req.SetSelfHash() - } + req.SetSelfHash() promise, err := producer.Produce(ctx, req.SelfHash, req) if err != nil { return nil, err @@ -187,7 +185,7 @@ func consume(ctx context.Context, t *testing.T, consumers []*Consumer[testReques func(ctx context.Context) { for { - res, ackNotifier, err := c.Consume(ctx) + res, err := c.Consume(ctx) if err != nil { if !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, context.Canceled) { t.Errorf("Consume() unexpected error: %v", err) @@ -206,7 +204,7 @@ func consume(ctx context.Context, t *testing.T, consumers []*Consumer[testReques } wantResponses[idx] = append(wantResponses[idx], resp) } - close(ackNotifier) + close(res.AckNotifier) } }) } @@ -291,7 +289,7 @@ func TestRedisProduceComplex(t *testing.T) { var promises [][]*containers.Promise[testResponse] for i := 0; i < tc.numProducers; i++ { - prs, err := produceMessages(ctx, entries[i], producers[i], tc.numProducers == 2, tc.withInvalidEntries) + prs, err := produceMessages(ctx, entries[i], producers[i], tc.withInvalidEntries) if err != nil { t.Fatalf("Error producing messages from producer%d: %v", i, err) } @@ -304,7 +302,7 @@ func TestRedisProduceComplex(t *testing.T) { // that other consumers will claim ownership on those messages. for i := 0; i < len(consumers); i += 3 { consumers[i].Start(ctx) - req, _, err := consumers[i].Consume(ctx) + req, err := consumers[i].Consume(ctx) if err != nil { t.Errorf("Error consuming message: %v", err) } diff --git a/validator/validation_entry.go b/validator/validation_entry.go index 59e2b53300..d51a9cbe55 100644 --- a/validator/validation_entry.go +++ b/validator/validation_entry.go @@ -34,7 +34,7 @@ type ValidationInput struct { // SetSelfHash should be only called once. In the context of redis streams- by the producer, before submitting a request func (v *ValidationInput) SetSelfHash() { if v.SelfHash != "" { - log.Error("SetSelfHash called more then once") + log.Warn("SetSelfHash called more then once") return // exiting early as hash has already been set } jsonData, err := json.Marshal(v) diff --git a/validator/valnode/redis/consumer.go b/validator/valnode/redis/consumer.go index 5255feb522..558cc6ba12 100644 --- a/validator/valnode/redis/consumer.go +++ b/validator/valnode/redis/consumer.go @@ -57,9 +57,8 @@ func (s *ValidationServer) Start(ctx_in context.Context) { // Channel that all consumers use to indicate their readiness. readyStreams := make(chan struct{}, len(s.consumers)) type workUnit struct { - req *pubsub.Message[*validator.ValidationInput] - moduleRoot common.Hash - ackNotifier chan struct{} + req *pubsub.Message[*validator.ValidationInput] + moduleRoot common.Hash } workers := s.config.Workers if workers == 0 { @@ -109,7 +108,7 @@ func (s *ValidationServer) Start(ctx_in context.Context) { return 0 case <-requestTokenQueue: } - req, ackNotifier, err := c.Consume(ctx) + req, err := c.Consume(ctx) if err != nil { log.Error("Consuming request", "error", err) requestTokenQueue <- struct{}{} @@ -122,7 +121,7 @@ func (s *ValidationServer) Start(ctx_in context.Context) { } select { case <-ctx.Done(): - case workQueue <- workUnit{req, moduleRoot, ackNotifier}: + case workQueue <- workUnit{req, moduleRoot}: } return 0 }) @@ -155,11 +154,11 @@ func (s *ValidationServer) Start(ctx_in context.Context) { res, err := valRun.Await(ctx) if err != nil { log.Error("Error validating", "request value", work.req.Value, "error", err) - close(work.ackNotifier) + close(work.req.AckNotifier) } else { err := s.consumers[work.moduleRoot].SetResult(ctx, work.req.Value.SelfHash, work.req.ID, res) // Even in error we close ackNotifier as there's no retry mechanism here and closing it will alow other consumers to autoclaim - close(work.ackNotifier) + close(work.req.AckNotifier) if err != nil { log.Error("Error setting result for request", "id", work.req.ID, "result", res, "error", err) } From 6a967acf92c2786933c04d5016e8b9b2bc8458d6 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Fri, 13 Sep 2024 19:10:40 +0530 Subject: [PATCH 193/489] Fix Submodule CI Pin check --- .github/workflows/submodule-pin-check.sh | 26 ----------------------- .github/workflows/submodule-pin-check.yml | 23 +++++++++++++++++++- 2 files changed, 22 insertions(+), 27 deletions(-) delete mode 100755 .github/workflows/submodule-pin-check.sh diff --git a/.github/workflows/submodule-pin-check.sh b/.github/workflows/submodule-pin-check.sh deleted file mode 100755 index aecb287ce1..0000000000 --- a/.github/workflows/submodule-pin-check.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -declare -Ar exceptions=( - [contracts]=origin/develop - [nitro-testnode]=origin/master - - #TODO Rachel to check these are the intended branches. - [arbitrator/langs/c]=origin/vm-storage-cache - [arbitrator/tools/wasmer]=origin/adopt-v4.2.8 -) - -divergent=0 -for mod in `git submodule --quiet foreach 'echo $name'`; do - branch=origin/HEAD - if [[ -v exceptions[$mod] ]]; then - branch=${exceptions[$mod]} - fi - - if ! git -C $mod merge-base --is-ancestor HEAD $branch; then - echo $mod diverges from $branch - divergent=1 - fi -done - -exit $divergent - diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index f045f71f68..5e4be387d9 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -18,11 +18,32 @@ jobs: with: fetch-depth: 0 submodules: true + ref: "${{ github.event.pull_request.merge_commit_sha }}" - name: Check all submodules are ancestors of origin/HEAD or configured branch run: | status_state="pending" - if ${{ github.workspace }}/.github/workflows/submodule-pin-check.sh; then + declare -Ar exceptions=( + [contracts]=origin/develop + [nitro-testnode]=origin/master + + #TODO Rachel to check these are the intended branches. + [arbitrator/langs/c]=origin/vm-storage-cache + [arbitrator/tools/wasmer]=origin/adopt-v4.2.8 + ) + divergent=0 + for mod in `git submodule --quiet foreach 'echo $name'`; do + branch=origin/HEAD + if [[ -v exceptions[$mod] ]]; then + branch=${exceptions[$mod]} + fi + + if ! git -C $mod merge-base --is-ancestor HEAD $branch; then + echo $mod diverges from $branch + divergent=1 + fi + done + if $divergent then status_state="success" else resp="$(curl -sSL --fail-with-body \ From 37f1ec56f432bab835cbecb1e16fa365134583d7 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Fri, 13 Sep 2024 21:09:02 +0530 Subject: [PATCH 194/489] test the change --- .github/workflows/submodule-pin-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index 5e4be387d9..b2df5bff0c 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -1,7 +1,7 @@ name: Merge Checks on: - pull_request_target: + pull_request: branches: [ master ] types: [synchronize, opened, reopened] From 3321cc4f998d5184bf4edda8ef0619681efbe672 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Fri, 13 Sep 2024 21:09:30 +0530 Subject: [PATCH 195/489] revert test the change --- .github/workflows/submodule-pin-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index b2df5bff0c..5e4be387d9 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -1,7 +1,7 @@ name: Merge Checks on: - pull_request: + pull_request_target: branches: [ master ] types: [synchronize, opened, reopened] From 55109fc6b386ff1d69e236d04fb1c9f454d0727e Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Fri, 13 Sep 2024 21:14:08 +0530 Subject: [PATCH 196/489] fix bash --- .github/workflows/submodule-pin-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index 5e4be387d9..76c777baad 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -43,7 +43,7 @@ jobs: divergent=1 fi done - if $divergent then + if $divergent; then status_state="success" else resp="$(curl -sSL --fail-with-body \ From 2b95a6099b6d9ae39e76e3641870f7ebe2fc9ba5 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Fri, 13 Sep 2024 21:14:26 +0530 Subject: [PATCH 197/489] test the change --- .github/workflows/submodule-pin-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index 76c777baad..46926b6430 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -1,7 +1,7 @@ name: Merge Checks on: - pull_request_target: + pull_request: branches: [ master ] types: [synchronize, opened, reopened] From 421bca1baccc7ed2e87c164f6ef2616243bacc1f Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Fri, 13 Sep 2024 21:14:37 +0530 Subject: [PATCH 198/489] revert test the change --- .github/workflows/submodule-pin-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index 46926b6430..76c777baad 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -1,7 +1,7 @@ name: Merge Checks on: - pull_request: + pull_request_target: branches: [ master ] types: [synchronize, opened, reopened] From f8bbc3804e6e9f67850de1d6ba2c4f6b80a701e9 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Fri, 13 Sep 2024 11:15:07 -0500 Subject: [PATCH 199/489] Add a bunch of trace logging --- arbnode/inbox_reader.go | 42 +++++++++++++++++++++++++++++++++++++++- arbnode/inbox_tracker.go | 20 +++++++++++-------- 2 files changed, 53 insertions(+), 9 deletions(-) diff --git a/arbnode/inbox_reader.go b/arbnode/inbox_reader.go index 835aef18b8..295f0acf4c 100644 --- a/arbnode/inbox_reader.go +++ b/arbnode/inbox_reader.go @@ -333,6 +333,7 @@ func (r *InboxReader) run(ctx context.Context, hadError bool) error { return err } if ourLatestDelayedCount < checkingDelayedCount { + log.Trace("Expecting to find delayed messages", "checkingDelayedCount", checkingDelayedCount, "ourLatestDelayedCount", ourLatestDelayedCount, "currentHeight", currentHeight) checkingDelayedCount = ourLatestDelayedCount missingDelayed = true } else if ourLatestDelayedCount > checkingDelayedCount { @@ -353,6 +354,7 @@ func (r *InboxReader) run(ctx context.Context, hadError bool) error { return err } if dbDelayedAcc != l1DelayedAcc { + log.Trace("Latest delayed accumulator mismatch", "delayedSeqNum", checkingDelayedSeqNum, "dbDelayedAcc", dbDelayedAcc, "l1DelayedAcc", l1DelayedAcc) reorgingDelayed = true } } @@ -370,6 +372,7 @@ func (r *InboxReader) run(ctx context.Context, hadError bool) error { return err } if ourLatestBatchCount < checkingBatchCount { + log.Trace("Expecting to find sequencer batches", "checkingBatchCount", checkingBatchCount, "ourLatestBatchCount", ourLatestBatchCount, "currentHeight", currentHeight) checkingBatchCount = ourLatestBatchCount missingSequencer = true } else if ourLatestBatchCount > checkingBatchCount && config.HardReorg { @@ -389,6 +392,7 @@ func (r *InboxReader) run(ctx context.Context, hadError bool) error { return err } if dbBatchAcc != l1BatchAcc { + log.Trace("Latest sequencer batch accumulator mismatch", "batchSeqNum", checkingBatchSeqNum, "dbBatchAcc", dbBatchAcc, "l1BatchAcc", l1BatchAcc) reorgingSequencer = true } } @@ -431,6 +435,15 @@ func (r *InboxReader) run(ctx context.Context, hadError bool) error { if to.Cmp(currentHeight) > 0 { to.Set(currentHeight) } + log.Trace( + "Looking up messages", + "from", from.String(), + "to", to.String(), + "missingDelayed", missingDelayed, + "missingSequencer", missingSequencer, + "reorgingDelayed", reorgingDelayed, + "reorgingSequencer", reorgingSequencer, + ) sequencerBatches, err := r.sequencerInbox.LookupBatchesInRange(ctx, from, to) if err != nil { return err @@ -456,6 +469,7 @@ func (r *InboxReader) run(ctx context.Context, hadError bool) error { if len(sequencerBatches) > 0 { missingSequencer = false reorgingSequencer = false + var havePrevAcc common.Hash firstBatch := sequencerBatches[0] if firstBatch.SequenceNumber > 0 { haveAcc, err := r.tracker.GetBatchAcc(firstBatch.SequenceNumber - 1) @@ -466,7 +480,10 @@ func (r *InboxReader) run(ctx context.Context, hadError bool) error { } else if haveAcc != firstBatch.BeforeInboxAcc { reorgingSequencer = true } + havePrevAcc = haveAcc } + readLastAcc := sequencerBatches[len(sequencerBatches)-1].AfterInboxAcc + var duplicateBatches int if !reorgingSequencer { // Skip any batches we already have in the database for len(sequencerBatches) > 0 { @@ -481,6 +498,7 @@ func (r *InboxReader) run(ctx context.Context, hadError bool) error { } else if haveAcc == batch.AfterInboxAcc { // Skip this batch, as we already have it in the database sequencerBatches = sequencerBatches[1:] + duplicateBatches++ } else { // The first batch AfterInboxAcc matches, but this batch doesn't, // so we'll successfully reorg it when we hit the addMessages @@ -488,7 +506,18 @@ func (r *InboxReader) run(ctx context.Context, hadError bool) error { } } } + log.Trace( + "Found sequencer batches", + "newBatchesCount", len(sequencerBatches), + "duplicateBatches", duplicateBatches, + "firstSequenceNumber", firstBatch.SequenceNumber, + "reorgingSequencer", reorgingSequencer, + "readBeforeAcc", firstBatch.BeforeInboxAcc, + "haveBeforeAcc", havePrevAcc, + "readLastAcc", readLastAcc, + ) } else if missingSequencer && to.Cmp(currentHeight) >= 0 { + log.Trace("Didn't find expected sequencer batches", "from", from, "to", to, "currentHeight", currentHeight) // We were missing sequencer batches but didn't find any. // This must mean that the sequencer batches are in the past. reorgingSequencer = true @@ -503,6 +532,7 @@ func (r *InboxReader) run(ctx context.Context, hadError bool) error { if err != nil { return err } + var havePrevAcc common.Hash if beforeCount > 0 { haveAcc, err := r.tracker.GetDelayedAcc(beforeCount - 1) if errors.Is(err, AccumulatorNotFoundErr) { @@ -512,14 +542,24 @@ func (r *InboxReader) run(ctx context.Context, hadError bool) error { } else if haveAcc != beforeAcc { reorgingDelayed = true } + havePrevAcc = haveAcc } + log.Trace( + "Found delayed messages", + "count", len(delayedMessages), + "firstSequenceNumber", beforeCount, + "reorgingDelayed", reorgingDelayed, + "readBeforeAcc", beforeAcc, + "haveBeforeAcc", havePrevAcc, + "readLastAcc", delayedMessages[len(delayedMessages)-1].AfterInboxAcc, + ) } else if missingDelayed && to.Cmp(currentHeight) >= 0 { + log.Trace("Didn't find expected delayed messages", "from", from, "to", to, "currentHeight", currentHeight) // We were missing delayed messages but didn't find any. // This must mean that the delayed messages are in the past. reorgingDelayed = true } - log.Trace("looking up messages", "from", from.String(), "to", to.String(), "missingDelayed", missingDelayed, "missingSequencer", missingSequencer, "reorgingDelayed", reorgingDelayed, "reorgingSequencer", reorgingSequencer) if !reorgingDelayed && !reorgingSequencer && (len(delayedMessages) != 0 || len(sequencerBatches) != 0) { delayedMismatch, err := r.addMessages(ctx, sequencerBatches, delayedMessages) if err != nil { diff --git a/arbnode/inbox_tracker.go b/arbnode/inbox_tracker.go index fe4149c80e..862610100c 100644 --- a/arbnode/inbox_tracker.go +++ b/arbnode/inbox_tracker.go @@ -696,22 +696,26 @@ func (t *InboxTracker) AddSequencerBatches(ctx context.Context, client arbutil.L for _, batch := range batches { if batch.SequenceNumber != pos { - return errors.New("unexpected batch sequence number") + return fmt.Errorf("unexpected batch sequence number %v expected %v", batch.SequenceNumber, pos) } if nextAcc != batch.BeforeInboxAcc { - return errors.New("previous batch accumulator mismatch") + return fmt.Errorf("previous batch accumulator %v mismatch expected %v", batch.BeforeInboxAcc, nextAcc) } if batch.AfterDelayedCount > 0 { haveDelayedAcc, err := t.GetDelayedAcc(batch.AfterDelayedCount - 1) - if errors.Is(err, AccumulatorNotFoundErr) { - // We somehow missed a referenced delayed message; go back and look for it - return delayedMessagesMismatch - } - if err != nil { + notFound := errors.Is(err, AccumulatorNotFoundErr) + if err != nil && !notFound { return err } - if haveDelayedAcc != batch.AfterDelayedAcc { + if notFound || haveDelayedAcc != batch.AfterDelayedAcc { + log.Trace( + "Delayed message accumulator doesn't match sequencer batch", + "batch", batch.SequenceNumber, + "delayedPosition", batch.AfterDelayedCount-1, + "haveDelayedAcc", haveDelayedAcc, + "batchDelayedAcc", batch.AfterDelayedAcc, + ) // We somehow missed a delayed message reorg; go back and look for it return delayedMessagesMismatch } From ff4802e66cda9d5f88423aa1164401fe0375f753 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Fri, 13 Sep 2024 14:11:59 -0500 Subject: [PATCH 200/489] Fix delayed message readLastAcc logging --- arbnode/inbox_reader.go | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/arbnode/inbox_reader.go b/arbnode/inbox_reader.go index 295f0acf4c..42e99fad66 100644 --- a/arbnode/inbox_reader.go +++ b/arbnode/inbox_reader.go @@ -228,6 +228,26 @@ func (r *InboxReader) CaughtUp() chan struct{} { return r.caughtUpChan } +type lazyHashLogging struct { + f func() common.Hash +} + +func (l lazyHashLogging) String() string { + return l.f().String() +} + +func (l lazyHashLogging) TerminalString() string { + return l.f().TerminalString() +} + +func (l lazyHashLogging) MarshalText() ([]byte, error) { + return l.f().MarshalText() +} + +func (l lazyHashLogging) Format(s fmt.State, c rune) { + l.f().Format(s, c) +} + func (r *InboxReader) run(ctx context.Context, hadError bool) error { readMode := r.config().ReadMode from, err := r.getNextBlockToRead(ctx) @@ -508,9 +528,9 @@ func (r *InboxReader) run(ctx context.Context, hadError bool) error { } log.Trace( "Found sequencer batches", + "firstSequenceNumber", firstBatch.SequenceNumber, "newBatchesCount", len(sequencerBatches), "duplicateBatches", duplicateBatches, - "firstSequenceNumber", firstBatch.SequenceNumber, "reorgingSequencer", reorgingSequencer, "readBeforeAcc", firstBatch.BeforeInboxAcc, "haveBeforeAcc", havePrevAcc, @@ -546,12 +566,15 @@ func (r *InboxReader) run(ctx context.Context, hadError bool) error { } log.Trace( "Found delayed messages", - "count", len(delayedMessages), "firstSequenceNumber", beforeCount, + "count", len(delayedMessages), "reorgingDelayed", reorgingDelayed, "readBeforeAcc", beforeAcc, "haveBeforeAcc", havePrevAcc, - "readLastAcc", delayedMessages[len(delayedMessages)-1].AfterInboxAcc, + "readLastAcc", lazyHashLogging{func() common.Hash { + // Only compute this if we need to log it, as it's expensive + return delayedMessages[len(delayedMessages)-1].AfterInboxAcc() + }}, ) } else if missingDelayed && to.Cmp(currentHeight) >= 0 { log.Trace("Didn't find expected delayed messages", "from", from, "to", to, "currentHeight", currentHeight) From e64daf6d833f33699ac85f078cd280bfacf9338c Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Fri, 13 Sep 2024 14:12:07 -0500 Subject: [PATCH 201/489] Don't prune the last delayed message --- arbnode/message_pruner.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arbnode/message_pruner.go b/arbnode/message_pruner.go index e1bc72632b..b249bd886c 100644 --- a/arbnode/message_pruner.go +++ b/arbnode/message_pruner.go @@ -112,6 +112,10 @@ func (m *MessagePruner) prune(ctx context.Context, count arbutil.MessageIndex, g } msgCount := endBatchMetadata.MessageCount delayedCount := endBatchMetadata.DelayedMessageCount + if delayedCount > 0 { + // keep an extra delayed message for the inbox reader to use + delayedCount-- + } return m.deleteOldMessagesFromDB(ctx, msgCount, delayedCount) } From a8967d17e2ac92aadc4d294c67b1c059200cbf93 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Mon, 16 Sep 2024 12:03:53 +0530 Subject: [PATCH 202/489] remove separate id impl --- pubsub/consumer.go | 14 ++--- pubsub/producer.go | 89 +++++------------------------ pubsub/pubsub_test.go | 67 +++++++--------------- validator/client/redis/producer.go | 3 +- validator/validation_entry.go | 20 ------- validator/valnode/redis/consumer.go | 2 +- 6 files changed, 42 insertions(+), 153 deletions(-) diff --git a/pubsub/consumer.go b/pubsub/consumer.go index 0ec3f11eb7..bf8aac8b45 100644 --- a/pubsub/consumer.go +++ b/pubsub/consumer.go @@ -113,7 +113,7 @@ func (c *Consumer[Request, Response]) Consume(ctx context.Context) (*Message[Req Group: c.redisGroup, Start: "-", End: "+", - Count: math.MaxInt64, + Count: 50, Idle: c.cfg.IdletimeToAutoclaim, }).Result(); err != nil { if !errors.Is(err, redis.Nil) { @@ -130,7 +130,7 @@ func (c *Consumer[Request, Response]) Consume(ctx context.Context) (*Message[Req Count: 1, }).Result() if err != nil { - log.Error("error from xautoclaim", "err", err) + log.Info("error from xautoclaim", "err", err) } } if len(messages) == 0 { @@ -209,18 +209,14 @@ func (c *Consumer[Request, Response]) Consume(ctx context.Context) (*Message[Req }, nil } -func (c *Consumer[Request, Response]) SetResult(ctx context.Context, id string, messageID string, result Response) error { - if id == "" { - log.Info("Request doesn't have a unique identifier (SelfHash field is not set), defaulting to using redis stream messageId", "msgId", messageID) - id = messageID - } +func (c *Consumer[Request, Response]) SetResult(ctx context.Context, messageID string, result Response) error { resp, err := json.Marshal(result) if err != nil { return fmt.Errorf("marshaling result: %w", err) } - acquired, err := c.client.SetNX(ctx, MessageKeyFor(c.StreamName(), id), resp, c.cfg.ResponseEntryTimeout).Result() + acquired, err := c.client.SetNX(ctx, MessageKeyFor(c.StreamName(), messageID), resp, c.cfg.ResponseEntryTimeout).Result() if err != nil || !acquired { - return fmt.Errorf("setting result for message with message-id in stream: %v, unique request identifier: %v, error: %w", messageID, id, err) + return fmt.Errorf("setting result for message with message-id in stream: %v, error: %w", messageID, err) } if _, err := c.client.XAck(ctx, c.redisStream, c.redisGroup, messageID).Result(); err != nil { return fmt.Errorf("acking message: %v, error: %w", messageID, err) diff --git a/pubsub/producer.go b/pubsub/producer.go index 74023ad5b0..9e354a82ee 100644 --- a/pubsub/producer.go +++ b/pubsub/producer.go @@ -31,11 +31,6 @@ const ( defaultGroup = "default_consumer_group" ) -type MsgIdAndPromise[Response any] struct { - msgID string - promise *containers.Promise[Response] -} - type Producer[Request any, Response any] struct { stopwaiter.StopWaiter id string @@ -45,7 +40,7 @@ type Producer[Request any, Response any] struct { cfg *ProducerConfig promisesLock sync.RWMutex - promises map[string]*MsgIdAndPromise[Response] + promises map[string]*containers.Promise[Response] // Used for checking responses from consumers iteratively // For the first time when Produce is called. @@ -92,7 +87,7 @@ func NewProducer[Request any, Response any](client redis.UniversalClient, stream redisStream: streamName, redisGroup: streamName, // There is 1-1 mapping of redis stream and consumer group. cfg: cfg, - promises: make(map[string]*MsgIdAndPromise[Response]), + promises: make(map[string]*containers.Promise[Response]), }, nil } @@ -142,18 +137,11 @@ func (p *Producer[Request, Response]) checkResponses(ctx context.Context) time.D if err != nil { log.Error("error getting PEL data from xpending, xtrimming is disabled", "err", err) } - deletePromise := func(id string) { - // Try deleting UNIQUEID_MSGID_MAP_KEY corresponding to this id from redis - if err := p.client.Del(ctx, MessageKeyFor(p.redisStream, id)+UNIQUEID_MSGID_MAP_KEY).Err(); err != nil { - log.Error("Error deleting key from redis that flags that a request is being processed", "err", err) - } - delete(p.promises, id) - } p.promisesLock.Lock() defer p.promisesLock.Unlock() responded := 0 errored := 0 - for id, msgIDAndPromise := range p.promises { + for id, promise := range p.promises { if ctx.Err() != nil { return 0 } @@ -169,25 +157,25 @@ func (p *Producer[Request, Response]) checkResponses(ctx context.Context) time.D if pelData != nil && pelData.Lower != "" { allowedOldestID = pelData.Lower } - if cmpMsgId(msgIDAndPromise.msgID, allowedOldestID) == -1 { - msgIDAndPromise.promise.ProduceError(errors.New("error getting response, request has been waiting for too long")) + if cmpMsgId(id, allowedOldestID) == -1 { + promise.ProduceError(errors.New("error getting response, request has been waiting for too long")) log.Error("error getting response, request has been waiting past its TTL") errored++ - deletePromise(id) + delete(p.promises, id) } } continue } var resp Response if err := json.Unmarshal([]byte(res), &resp); err != nil { - msgIDAndPromise.promise.ProduceError(fmt.Errorf("error unmarshalling: %w", err)) + promise.ProduceError(fmt.Errorf("error unmarshalling: %w", err)) log.Error("Error unmarshaling", "value", res, "error", err) errored++ } else { - msgIDAndPromise.promise.Produce(resp) + promise.Produce(resp) responded++ } - deletePromise(id) + delete(p.promises, id) } // XDEL on consumer side already deletes acked messages (mark as deleted) but doesnt claim the memory back, XTRIM helps in claiming this memory in normal conditions // pelData might be outdated when we do the xtrim, but thats ok as the messages are also being trimmed by other producers @@ -230,39 +218,7 @@ func (p *Producer[Request, Response]) promisesLen() int { return len(p.promises) } -func (p *Producer[Request, Response]) produce(ctx context.Context, id string, value Request) (*containers.Promise[Response], error) { - if id != "" { - msgKey := MessageKeyFor(p.redisStream, id) - - // If the request has already been solved by a consumer - if res, err := p.client.Get(ctx, msgKey).Result(); err == nil { - var resp Response - if err := json.Unmarshal([]byte(res), &resp); err != nil { - log.Error("Error unmarshaling", "value", res, "error", err) - return nil, fmt.Errorf("error unmarshalling: %w", err) - } else { - pr := containers.NewPromise[Response](nil) - pr.Produce(resp) - return &pr, nil - } - } else if !errors.Is(err, redis.Nil) { - log.Error("error while checking for response to a request in redis", "err", err) - } - - // Check for duplicate unsolved request messages in stream - if res, err := p.client.Get(ctx, msgKey+UNIQUEID_MSGID_MAP_KEY).Result(); err == nil { - log.Info("Request already submitted by another producer", "msgId", res, "requestUniqueId", id) - p.promisesLock.Lock() - defer p.promisesLock.Unlock() - pr := containers.NewPromise[Response](nil) - p.promises[id] = &MsgIdAndPromise[Response]{ - msgID: res, - promise: &pr, - } - return &pr, nil - } - } - +func (p *Producer[Request, Response]) produce(ctx context.Context, value Request) (*containers.Promise[Response], error) { val, err := json.Marshal(value) if err != nil { return nil, fmt.Errorf("marshaling value: %w", err) @@ -277,30 +233,15 @@ func (p *Producer[Request, Response]) produce(ctx context.Context, id string, va if err != nil { return nil, fmt.Errorf("adding values to redis: %w", err) } - - if id == "" { - // If unique id doesn't exist, use the newly created msgId as unique id and follow the same steps as before - log.Info("Request doesn't have a unique identifier (SelfHash field set), defaulting to using redis stream messageId", "msgId", msgId) - id = msgId - } - - // Try adding key that flags that request is being processed - if err := p.client.Set(ctx, MessageKeyFor(p.redisStream, id)+UNIQUEID_MSGID_MAP_KEY, msgId, p.cfg.ResponseEntryTimeout).Err(); err != nil { - log.Error("Error adding key to redis that flags that a request is being processed, stream may encounter duplicate requests", "err", err) - } - - pr := containers.NewPromise[Response](nil) - p.promises[id] = &MsgIdAndPromise[Response]{ - msgID: msgId, - promise: &pr, - } - return &pr, nil + promise := containers.NewPromise[Response](nil) + p.promises[msgId] = &promise + return &promise, nil } -func (p *Producer[Request, Response]) Produce(ctx context.Context, id string, value Request) (*containers.Promise[Response], error) { +func (p *Producer[Request, Response]) Produce(ctx context.Context, value Request) (*containers.Promise[Response], error) { log.Debug("Redis stream producing", "value", value) p.once.Do(func() { p.StopWaiter.CallIteratively(p.checkResponses) }) - return p.produce(ctx, id, value) + return p.produce(ctx, value) } diff --git a/pubsub/pubsub_test.go b/pubsub/pubsub_test.go index 13258aea3d..391bd7555c 100644 --- a/pubsub/pubsub_test.go +++ b/pubsub/pubsub_test.go @@ -2,9 +2,6 @@ package pubsub import ( "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" "errors" "fmt" "os" @@ -28,17 +25,6 @@ var ( type testRequest struct { Request string IsInvalid bool - SelfHash string // Is a unique identifier which can be used to compare any two validationInputs -} - -// SetSelfHash should be only called once. In the context of redis streams- by the producer -func (t *testRequest) SetSelfHash() { - jsonData, err := json.Marshal(t) - if err != nil { - return - } - hash := sha256.Sum256(jsonData) - t.SelfHash = hex.EncodeToString(hash[:]) } type testResponse struct { @@ -117,13 +103,10 @@ func msgForIndex(idx int) string { return fmt.Sprintf("msg: %d", idx) } -func wantMessages(n int, group string, withDuplicates bool) []string { +func wantMessages(n int, group string) []string { var ret []string for i := 0; i < n; i++ { ret = append(ret, group+msgForIndex(i)) - if withDuplicates && i%3 == 0 { - ret = append(ret, msgForIndex(i)) - } } sort.Strings(ret) return ret @@ -145,8 +128,7 @@ func produceMessages(ctx context.Context, msgs []string, producer *Producer[test if withInvalidEntries && i%50 == 0 { req.IsInvalid = true } - req.SetSelfHash() - promise, err := producer.Produce(ctx, req.SelfHash, req) + promise, err := producer.Produce(ctx, req) if err != nil { return nil, err } @@ -199,7 +181,7 @@ func consume(ctx context.Context, t *testing.T, consumers []*Consumer[testReques gotMessages[idx][res.ID] = res.Value.Request if !res.Value.IsInvalid { resp := fmt.Sprintf("result for: %v", res.ID) - if err := c.SetResult(ctx, res.Value.SelfHash, res.ID, testResponse{Response: resp}); err != nil { + if err := c.SetResult(ctx, res.ID, testResponse{Response: resp}); err != nil { t.Errorf("Error setting a result: %v", err) } wantResponses[idx] = append(wantResponses[idx], resp) @@ -218,7 +200,6 @@ func TestRedisProduceComplex(t *testing.T) { name string entriesCount []int numProducers int - withDuplicates bool // If this is set, then every fourth entry (while generation) of each entries list is equal killConsumers bool withInvalidEntries bool // If this is set, then every 50th entry is invalid (requests that can't be solved by any consumer) }{ @@ -228,34 +209,27 @@ func TestRedisProduceComplex(t *testing.T) { numProducers: 1, }, { - name: "one producer, some consumers killed, others should take over their work", - entriesCount: []int{messagesCount}, - numProducers: 1, - killConsumers: true, - }, - { - name: "two producers, all consumers are active, all unique entries", + name: "two producers, all consumers are active", entriesCount: []int{20, 20}, numProducers: 2, }, { - name: "two producers, all consumers are active, some duplicate entries", - entriesCount: []int{20, 20}, - numProducers: 2, - withDuplicates: true, + name: "one producer, some consumers killed, others should take over their work", + entriesCount: []int{messagesCount}, + numProducers: 1, + killConsumers: true, }, + { - name: "two producers, some consumers killed, others should take over their work, some duplicate entries, unequal number of requests from producers", - entriesCount: []int{messagesCount, 2 * messagesCount}, - numProducers: 2, - withDuplicates: true, - killConsumers: true, + name: "two producers, some consumers killed, others should take over their work, unequal number of requests from producers", + entriesCount: []int{messagesCount, 2 * messagesCount}, + numProducers: 2, + killConsumers: true, }, { - name: "two producers, some consumers killed, others should take over their work, some duplicate entries, some invalid entries, unequal number of requests from producers", + name: "two producers, some consumers killed, others should take over their work, some invalid entries, unequal number of requests from producers", entriesCount: []int{messagesCount, 2 * messagesCount}, numProducers: 2, - withDuplicates: true, killConsumers: true, withInvalidEntries: true, }, @@ -281,10 +255,10 @@ func TestRedisProduceComplex(t *testing.T) { var entries [][]string if tc.numProducers == 2 { - entries = append(entries, wantMessages(tc.entriesCount[0], "1.", tc.withDuplicates)) - entries = append(entries, wantMessages(tc.entriesCount[1], "2.", tc.withDuplicates)) + entries = append(entries, wantMessages(tc.entriesCount[0], "1.")) + entries = append(entries, wantMessages(tc.entriesCount[1], "2.")) } else { - entries = append(entries, wantMessages(tc.entriesCount[0], "", tc.withDuplicates)) + entries = append(entries, wantMessages(tc.entriesCount[0], "")) } var promises [][]*containers.Promise[testResponse] @@ -322,7 +296,7 @@ func TestRedisProduceComplex(t *testing.T) { for i := 0; i < tc.numProducers; i++ { grs, errIndexes := awaitResponses(ctx, promises[i]) if tc.withInvalidEntries { - if errIndexes[len(errIndexes)-1]+50 <= len(entries[i]) { + if errIndexes[len(errIndexes)-1]+50 < len(entries[i]) { t.Fatalf("Unexpected number of invalid requests while awaiting responses") } for j, idx := range errIndexes { @@ -353,13 +327,12 @@ func TestRedisProduceComplex(t *testing.T) { for i := 0; i < tc.numProducers; i++ { combinedEntries = append(combinedEntries, entries[i]...) } - wantMsgs := removeDuplicates(combinedEntries) + wantMsgs := combinedEntries if diff := cmp.Diff(wantMsgs, got); diff != "" { t.Errorf("Unexpected diff (-want +got):\n%s\n", diff) } - // Consumers are not supposed to get duplicate requests - gotResponses = removeDuplicates(gotResponses) + sort.Strings(gotResponses) wantResp := flatten(wantResponses) if diff := cmp.Diff(wantResp, gotResponses); diff != "" { t.Errorf("Unexpected diff in responses:\n%s\n", diff) diff --git a/validator/client/redis/producer.go b/validator/client/redis/producer.go index ffa6146f81..c5726ffe8b 100644 --- a/validator/client/redis/producer.go +++ b/validator/client/redis/producer.go @@ -136,8 +136,7 @@ func (c *ValidationClient) Launch(entry *validator.ValidationInput, moduleRoot c errPromise := containers.NewReadyPromise(validator.GoGlobalState{}, fmt.Errorf("no validation is configured for wasm root %v", moduleRoot)) return server_common.NewValRun(errPromise, moduleRoot) } - entry.SetSelfHash() - promise, err := producer.Produce(c.GetContext(), entry.SelfHash, entry) + promise, err := producer.Produce(c.GetContext(), entry) if err != nil { errPromise := containers.NewReadyPromise(validator.GoGlobalState{}, fmt.Errorf("error producing input: %w", err)) return server_common.NewValRun(errPromise, moduleRoot) diff --git a/validator/validation_entry.go b/validator/validation_entry.go index d51a9cbe55..d340993fa2 100644 --- a/validator/validation_entry.go +++ b/validator/validation_entry.go @@ -1,13 +1,8 @@ package validator import ( - "encoding/json" - "fmt" - - "github.com/cespare/xxhash/v2" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/log" "github.com/offchainlabs/nitro/arbutil" ) @@ -27,19 +22,4 @@ type ValidationInput struct { DelayedMsg []byte StartState GoGlobalState DebugChain bool - - SelfHash string // Is a unique identifier which can be used to compare any two instances of validationInput -} - -// SetSelfHash should be only called once. In the context of redis streams- by the producer, before submitting a request -func (v *ValidationInput) SetSelfHash() { - if v.SelfHash != "" { - log.Warn("SetSelfHash called more then once") - return // exiting early as hash has already been set - } - jsonData, err := json.Marshal(v) - if err != nil { - return - } - v.SelfHash = fmt.Sprintf("%d", xxhash.Sum64(jsonData)) } diff --git a/validator/valnode/redis/consumer.go b/validator/valnode/redis/consumer.go index 558cc6ba12..4d19905ab7 100644 --- a/validator/valnode/redis/consumer.go +++ b/validator/valnode/redis/consumer.go @@ -156,7 +156,7 @@ func (s *ValidationServer) Start(ctx_in context.Context) { log.Error("Error validating", "request value", work.req.Value, "error", err) close(work.req.AckNotifier) } else { - err := s.consumers[work.moduleRoot].SetResult(ctx, work.req.Value.SelfHash, work.req.ID, res) + err := s.consumers[work.moduleRoot].SetResult(ctx, work.req.ID, res) // Even in error we close ackNotifier as there's no retry mechanism here and closing it will alow other consumers to autoclaim close(work.req.AckNotifier) if err != nil { From 8612635a7efc72b91c1c0c8a54575b4fe51b031c Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Mon, 16 Sep 2024 16:42:48 +0530 Subject: [PATCH 203/489] Changes based on PR comments --- .github/workflows/submodule-pin-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index 76c777baad..a6f53e7a25 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -43,7 +43,7 @@ jobs: divergent=1 fi done - if $divergent; then + if [ $divergent -eq 1 ]; then status_state="success" else resp="$(curl -sSL --fail-with-body \ From 7307caa628fd785a6747c65036b86002f8e3fb35 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Mon, 16 Sep 2024 16:45:13 +0530 Subject: [PATCH 204/489] test the change --- .github/workflows/submodule-pin-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index a6f53e7a25..40fe2d7cb2 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -1,7 +1,7 @@ name: Merge Checks on: - pull_request_target: + pull_request: branches: [ master ] types: [synchronize, opened, reopened] From b0f166ac36c238064ab6fc1c547fc2a3740015ec Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Mon, 16 Sep 2024 16:45:27 +0530 Subject: [PATCH 205/489] revert test the change --- .github/workflows/submodule-pin-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index 40fe2d7cb2..a6f53e7a25 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -1,7 +1,7 @@ name: Merge Checks on: - pull_request: + pull_request_target: branches: [ master ] types: [synchronize, opened, reopened] From 35671c8804a8c534f1388f89221a4933c946e02a Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Mon, 16 Sep 2024 17:33:37 +0530 Subject: [PATCH 206/489] change to 0 --- .github/workflows/submodule-pin-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index a6f53e7a25..90419b530e 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -43,7 +43,7 @@ jobs: divergent=1 fi done - if [ $divergent -eq 1 ]; then + if [ $divergent -eq 0 ]; then status_state="success" else resp="$(curl -sSL --fail-with-body \ From 7460b7b2fb4aedea85254c3a6e5e1091a38600af Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Mon, 16 Sep 2024 17:33:53 +0530 Subject: [PATCH 207/489] test the change --- .github/workflows/submodule-pin-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index 90419b530e..a3638a4929 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -1,7 +1,7 @@ name: Merge Checks on: - pull_request_target: + pull_request: branches: [ master ] types: [synchronize, opened, reopened] From 7c41e5d657ebfe3cf52a1bd349c057158abce98a Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Mon, 16 Sep 2024 17:34:06 +0530 Subject: [PATCH 208/489] revert test the change --- .github/workflows/submodule-pin-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index a3638a4929..90419b530e 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -1,7 +1,7 @@ name: Merge Checks on: - pull_request: + pull_request_target: branches: [ master ] types: [synchronize, opened, reopened] From 9b8a8f6207ce03ad42e8fbf856d2aba4cdd1529b Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Mon, 16 Sep 2024 10:53:02 -0300 Subject: [PATCH 209/489] Fix warn message formatting --- arbnode/seq_coordinator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arbnode/seq_coordinator.go b/arbnode/seq_coordinator.go index a582b64ffa..176ace114b 100644 --- a/arbnode/seq_coordinator.go +++ b/arbnode/seq_coordinator.go @@ -491,7 +491,7 @@ func (c *SeqCoordinator) updateWithLockout(ctx context.Context, nextChosen strin // Before proceeding, first try deleting finalized messages from redis and setting the finalizedMsgCount key finalized, err := c.sync.GetFinalizedMsgCount(ctx) if err != nil { - log.Warn("Error getting finalizedMessageCount from syncMonitor: %w", err) + log.Warn("Error getting finalizedMessageCount from syncMonitor", "err", err) } else if finalized == 0 { log.Warn("SyncMonitor returned zero finalizedMessageCount") } else if err := c.deleteFinalizedMsgsFromRedis(ctx, finalized); err != nil { From f0f5af5798d3bd6f4f6be5a1c16c3cba82567458 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Mon, 16 Sep 2024 14:31:25 -0300 Subject: [PATCH 210/489] Use positive in config name (disable -> enable) Rename --node.batch-poster.disable-dap-fallback-store-data-on-chain to --node.batch-poster.enable-dap-fallback-store-data-on-chain --- arbnode/batch_poster.go | 12 ++++++------ arbstate/daprovider/writer.go | 6 +++--- system_tests/das_test.go | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/arbnode/batch_poster.go b/arbnode/batch_poster.go index 44b360e76e..4e4a8d8572 100644 --- a/arbnode/batch_poster.go +++ b/arbnode/batch_poster.go @@ -141,8 +141,8 @@ type BatchPosterDangerousConfig struct { } type BatchPosterConfig struct { - Enable bool `koanf:"enable"` - DisableDapFallbackStoreDataOnChain bool `koanf:"disable-dap-fallback-store-data-on-chain" reload:"hot"` + Enable bool `koanf:"enable"` + EnableDapFallbackStoreDataOnChain bool `koanf:"enable-dap-fallback-store-data-on-chain" reload:"hot"` // Max batch size. MaxSize int `koanf:"max-size" reload:"hot"` // Maximum 4844 blob enabled batch size. @@ -205,7 +205,7 @@ type BatchPosterConfigFetcher func() *BatchPosterConfig func BatchPosterConfigAddOptions(prefix string, f *pflag.FlagSet) { f.Bool(prefix+".enable", DefaultBatchPosterConfig.Enable, "enable posting batches to l1") - f.Bool(prefix+".disable-dap-fallback-store-data-on-chain", DefaultBatchPosterConfig.DisableDapFallbackStoreDataOnChain, "If unable to batch to DA provider, disable fallback storing data on chain") + f.Bool(prefix+".enable-dap-fallback-store-data-on-chain", DefaultBatchPosterConfig.EnableDapFallbackStoreDataOnChain, "If unable to batch to DA provider, enable fallback storing data on chain") f.Int(prefix+".max-size", DefaultBatchPosterConfig.MaxSize, "maximum batch size") f.Int(prefix+".max-4844-batch-size", DefaultBatchPosterConfig.Max4844BatchSize, "maximum 4844 blob enabled batch size") f.Duration(prefix+".max-delay", DefaultBatchPosterConfig.MaxDelay, "maximum batch posting delay") @@ -231,8 +231,8 @@ func BatchPosterConfigAddOptions(prefix string, f *pflag.FlagSet) { } var DefaultBatchPosterConfig = BatchPosterConfig{ - Enable: false, - DisableDapFallbackStoreDataOnChain: false, + Enable: false, + EnableDapFallbackStoreDataOnChain: true, // This default is overridden for L3 chains in applyChainParameters in cmd/nitro/nitro.go MaxSize: 100000, // Try to fill 3 blobs per batch @@ -1366,7 +1366,7 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) return false, fmt.Errorf("%w: nonce changed from %d to %d while creating batch", storage.ErrStorageRace, nonce, gotNonce) } // #nosec G115 - sequencerMsg, err = b.dapWriter.Store(ctx, sequencerMsg, uint64(time.Now().Add(config.DASRetentionPeriod).Unix()), config.DisableDapFallbackStoreDataOnChain) + sequencerMsg, err = b.dapWriter.Store(ctx, sequencerMsg, uint64(time.Now().Add(config.DASRetentionPeriod).Unix()), config.EnableDapFallbackStoreDataOnChain) if err != nil { batchPosterDAFailureCounter.Inc(1) return false, err diff --git a/arbstate/daprovider/writer.go b/arbstate/daprovider/writer.go index a26e53c94d..1d83c0348f 100644 --- a/arbstate/daprovider/writer.go +++ b/arbstate/daprovider/writer.go @@ -17,7 +17,7 @@ type Writer interface { ctx context.Context, message []byte, timeout uint64, - disableFallbackStoreDataOnChain bool, + enableFallbackStoreDataOnChain bool, ) ([]byte, error) } @@ -31,10 +31,10 @@ type writerForDAS struct { dasWriter DASWriter } -func (d *writerForDAS) Store(ctx context.Context, message []byte, timeout uint64, disableFallbackStoreDataOnChain bool) ([]byte, error) { +func (d *writerForDAS) Store(ctx context.Context, message []byte, timeout uint64, enableFallbackStoreDataOnChain bool) ([]byte, error) { cert, err := d.dasWriter.Store(ctx, message, timeout) if errors.Is(err, ErrBatchToDasFailed) { - if disableFallbackStoreDataOnChain { + if !enableFallbackStoreDataOnChain { return nil, errors.New("unable to batch to DAS and fallback storing data on chain is disabled") } log.Warn("Falling back to storing data on chain", "err", err) diff --git a/system_tests/das_test.go b/system_tests/das_test.go index d59fca3e47..b16a97720b 100644 --- a/system_tests/das_test.go +++ b/system_tests/das_test.go @@ -348,7 +348,7 @@ func TestDASBatchPosterFallback(t *testing.T) { builder.nodeConfig.DataAvailability.RestAggregator.Enable = true builder.nodeConfig.DataAvailability.RestAggregator.Urls = []string{restServerUrl} builder.nodeConfig.DataAvailability.ParentChainNodeURL = "none" - builder.nodeConfig.BatchPoster.DisableDapFallbackStoreDataOnChain = true // Disable DAS fallback + builder.nodeConfig.BatchPoster.EnableDapFallbackStoreDataOnChain = false // Disable DAS fallback builder.nodeConfig.BatchPoster.ErrorDelay = time.Millisecond * 250 // Increase error delay because we expect errors builder.L2Info = NewArbTestInfo(t, builder.chainConfig.ChainID) builder.L2Info.GenerateAccount("User2") @@ -388,7 +388,7 @@ func TestDASBatchPosterFallback(t *testing.T) { // Enable the DAP fallback and check the transaction on the second node. // (We don't need to restart the node because of the hot-reload.) - builder.nodeConfig.BatchPoster.DisableDapFallbackStoreDataOnChain = false + builder.nodeConfig.BatchPoster.EnableDapFallbackStoreDataOnChain = true _, err = WaitForTx(ctx, l2B.Client, tx.Hash(), time.Second*3) Require(t, err) l2balance, err := l2B.Client.BalanceAt(ctx, l2info.GetAddress("User2"), nil) From 143c68be28db76d12c71fa5f9c4d9c2e303b2f66 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Mon, 16 Sep 2024 16:00:25 -0300 Subject: [PATCH 211/489] Enable DAP fallback on test config --- arbnode/batch_poster.go | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/arbnode/batch_poster.go b/arbnode/batch_poster.go index 4e4a8d8572..8c80d65009 100644 --- a/arbnode/batch_poster.go +++ b/arbnode/batch_poster.go @@ -267,26 +267,27 @@ var DefaultBatchPosterL1WalletConfig = genericconf.WalletConfig{ } var TestBatchPosterConfig = BatchPosterConfig{ - Enable: true, - MaxSize: 100000, - Max4844BatchSize: DefaultBatchPosterConfig.Max4844BatchSize, - PollInterval: time.Millisecond * 10, - ErrorDelay: time.Millisecond * 10, - MaxDelay: 0, - WaitForMaxDelay: false, - CompressionLevel: 2, - DASRetentionPeriod: daprovider.DefaultDASRetentionPeriod, - GasRefunderAddress: "", - ExtraBatchGas: 10_000, - Post4844Blobs: true, - IgnoreBlobPrice: false, - DataPoster: dataposter.TestDataPosterConfig, - ParentChainWallet: DefaultBatchPosterL1WalletConfig, - L1BlockBound: "", - L1BlockBoundBypass: time.Hour, - UseAccessLists: true, - GasEstimateBaseFeeMultipleBips: arbmath.OneInUBips * 3 / 2, - CheckBatchCorrectness: true, + Enable: true, + EnableDapFallbackStoreDataOnChain: true, + MaxSize: 100000, + Max4844BatchSize: DefaultBatchPosterConfig.Max4844BatchSize, + PollInterval: time.Millisecond * 10, + ErrorDelay: time.Millisecond * 10, + MaxDelay: 0, + WaitForMaxDelay: false, + CompressionLevel: 2, + DASRetentionPeriod: daprovider.DefaultDASRetentionPeriod, + GasRefunderAddress: "", + ExtraBatchGas: 10_000, + Post4844Blobs: true, + IgnoreBlobPrice: false, + DataPoster: dataposter.TestDataPosterConfig, + ParentChainWallet: DefaultBatchPosterL1WalletConfig, + L1BlockBound: "", + L1BlockBoundBypass: time.Hour, + UseAccessLists: true, + GasEstimateBaseFeeMultipleBips: arbmath.OneInUBips * 3 / 2, + CheckBatchCorrectness: true, } type BatchPosterOpts struct { From 984a7f1cec03d86fa40f2a0634f1fa3fcf1c69ff Mon Sep 17 00:00:00 2001 From: Pepper Lebeck-Jobe Date: Wed, 18 Sep 2024 15:14:58 +0200 Subject: [PATCH 212/489] Fix broken imports of ethdb and rawdb --- staker/stateless_block_validator.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/staker/stateless_block_validator.go b/staker/stateless_block_validator.go index 0cd97ec8d3..c4b0948e43 100644 --- a/staker/stateless_block_validator.go +++ b/staker/stateless_block_validator.go @@ -12,6 +12,7 @@ import ( "github.com/offchainlabs/nitro/arbstate/daprovider" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" @@ -468,7 +469,7 @@ func (v *StatelessBlockValidator) ValidationInputsAt(ctx context.Context, pos ar if err != nil { return server_api.InputJSON{}, err } - input, err := entry.ToInput([]rawdb.Target{rawdb.TargetWavm}) + input, err := entry.ToInput([]ethdb.WasmTarget{rawdb.TargetWavm}) if err != nil { return server_api.InputJSON{}, err } From c45d80fd4ff16fd4b25dc8e4e200be19d5f0cd7c Mon Sep 17 00:00:00 2001 From: Pepper Lebeck-Jobe Date: Wed, 18 Sep 2024 15:35:34 +0200 Subject: [PATCH 213/489] Update Cargo.lock --- arbitrator/wasm-libraries/Cargo.lock | 195 ++++++++++++++++++++------- 1 file changed, 148 insertions(+), 47 deletions(-) diff --git a/arbitrator/wasm-libraries/Cargo.lock b/arbitrator/wasm-libraries/Cargo.lock index b234424f69..c79b389f6a 100644 --- a/arbitrator/wasm-libraries/Cargo.lock +++ b/arbitrator/wasm-libraries/Cargo.lock @@ -46,6 +46,21 @@ version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "ansi_term" version = "0.12.1" @@ -121,6 +136,12 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bincode" version = "1.3.3" @@ -260,6 +281,19 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "num-traits", + "serde", + "windows-targets", +] + [[package]] name = "clap" version = "2.34.0" @@ -281,6 +315,12 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "corosensei" version = "0.1.4" @@ -436,38 +476,14 @@ dependencies = [ "typenum", ] -[[package]] -name = "darling" -version = "0.13.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" -dependencies = [ - "darling_core 0.13.4", - "darling_macro 0.13.4", -] - [[package]] name = "darling" version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" dependencies = [ - "darling_core 0.20.10", - "darling_macro 0.20.10", -] - -[[package]] -name = "darling_core" -version = "0.13.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim 0.10.0", - "syn 1.0.109", + "darling_core", + "darling_macro", ] [[package]] @@ -480,27 +496,17 @@ dependencies = [ "ident_case", "proc-macro2", "quote", + "strsim 0.11.1", "syn 2.0.72", ] -[[package]] -name = "darling_macro" -version = "0.13.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" -dependencies = [ - "darling_core 0.13.4", - "quote", - "syn 1.0.109", -] - [[package]] name = "darling_macro" version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ - "darling_core 0.20.10", + "darling_core", "quote", "syn 2.0.72", ] @@ -518,6 +524,16 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", + "serde", +] + [[package]] name = "derivative" version = "2.2.0" @@ -622,7 +638,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59c3b24c345d8c314966bdc1832f6c2635bfcce8e7cf363bd115987bba2ee242" dependencies = [ - "darling 0.20.10", + "darling", "proc-macro2", "quote", "syn 2.0.72", @@ -760,6 +776,29 @@ dependencies = [ "caller-env", ] +[[package]] +name = "iana-time-zone" +version = "0.1.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -780,6 +819,7 @@ checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", "hashbrown 0.12.3", + "serde", ] [[package]] @@ -790,6 +830,7 @@ checksum = "de3fc2e30ba82dd1b3911c8de1ffc143c74a914a14e99514d7637e3099df5ea0" dependencies = [ "equivalent", "hashbrown 0.14.5", + "serde", ] [[package]] @@ -1000,6 +1041,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-derive" version = "0.4.2" @@ -1128,6 +1175,12 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "proc-macro-crate" version = "3.1.0" @@ -1473,24 +1526,32 @@ dependencies = [ [[package]] name = "serde_with" -version = "1.14.0" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678b5a069e50bf00ecd22d0cd8ddf7c236f68581b03db652061ed5eb13a312ff" +checksum = "69cecfa94848272156ea67b2b1a53f20fc7bc638c4a46d2f8abde08f05f4b857" dependencies = [ + "base64", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.3.0", "serde", + "serde_derive", + "serde_json", "serde_with_macros", + "time", ] [[package]] name = "serde_with_macros" -version = "1.5.2" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" +checksum = "a8fee4991ef4f274617a51ad4af30519438dacb2f56ac773b08a1922ff743350" dependencies = [ - "darling 0.13.4", + "darling", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.72", ] [[package]] @@ -1602,9 +1663,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "strsim" -version = "0.10.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "structopt" @@ -1693,6 +1754,37 @@ dependencies = [ "syn 2.0.72", ] +[[package]] +name = "time" +version = "0.3.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tiny-keccak" version = "2.0.2" @@ -2111,6 +2203,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.33.0" From f9e6eea54e31d50834f64697b5cd029c5274e160 Mon Sep 17 00:00:00 2001 From: Pepper Lebeck-Jobe Date: Wed, 18 Sep 2024 18:32:16 +0200 Subject: [PATCH 214/489] Address review feedback --- arbitrator/arbutil/src/types.rs | 101 ++++++++++++++++++++++++++++ arbitrator/prover/src/prepare.rs | 1 - arbnode/api.go | 15 +---- staker/stateless_block_validator.go | 5 +- system_tests/common_test.go | 3 +- 5 files changed, 107 insertions(+), 18 deletions(-) diff --git a/arbitrator/arbutil/src/types.rs b/arbitrator/arbutil/src/types.rs index 207b8ab949..e92d4dfbc5 100644 --- a/arbitrator/arbutil/src/types.rs +++ b/arbitrator/arbutil/src/types.rs @@ -84,6 +84,32 @@ impl From for Bytes32 { } } +impl FromStr for Bytes32 { + type Err = &'static str; + + fn from_str(s: &str) -> Result { + // Remove the "0x" prefix if present + let s = s.strip_prefix("0x").unwrap_or(s); + + // Pad with leading zeros if the string is shorter than 64 characters (32 bytes) + let padded = format!("{:0>64}", s); + + // Decode the hex string using the hex crate + let decoded_bytes = hex::decode(padded).map_err(|_| "Invalid hex string")?; + + // Ensure the decoded bytes is exactly 32 bytes + if decoded_bytes.len() != 32 { + return Err("Hex string too long for Bytes32"); + } + + // Create a 32-byte array and fill it with the decoded bytes. + let mut b = [0u8; 32]; + b.copy_from_slice(&decoded_bytes); + + Ok(Bytes32(b)) + } +} +/* impl FromStr for Bytes32 { type Err = hex::FromHexError; @@ -98,6 +124,7 @@ impl FromStr for Bytes32 { Ok(Self(b)) } } +*/ impl TryFrom<&[u8]> for Bytes32 { type Error = std::array::TryFromSliceError; @@ -265,3 +292,77 @@ impl From for Bytes20 { <[u8; 20]>::from(x).into() } } + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_bytes32() { + let b = Bytes32::from(0x12345678u32); + let expected = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0x12, 0x34, 0x56, 0x78, + ]; + assert_eq!(b, Bytes32(expected)); + } + + #[test] + fn test_from_str_short() { + // Short hex string + let b = Bytes32::from_str("0x12345678").unwrap(); + let expected = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0x12, 0x34, 0x56, 0x78, + ]; + assert_eq!(b, Bytes32(expected)); + } + + #[test] + fn test_from_str_very_short() { + // Short hex string + let b = Bytes32::from_str("0x1").unwrap(); + let expected = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0x1, + ]; + assert_eq!(b, Bytes32(expected)); + } + + #[test] + fn test_from_str_no_prefix() { + // Short hex string + let b = Bytes32::from_str("12345678").unwrap(); + let expected = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0x12, 0x34, 0x56, 0x78, + ]; + assert_eq!(b, Bytes32(expected)); + } + + #[test] + fn test_from_str_full() { + // Full-length hex string + let b = + Bytes32::from_str("0x0000000000000000000000000000000000000000000000000000000012345678") + .unwrap(); + let expected = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0x12, 0x34, 0x56, 0x78, + ]; + assert_eq!(b, Bytes32(expected)); + } + + #[test] + fn test_from_str_invalid_non_hex() { + let s = "0x123g5678"; // Invalid character 'g' + assert!(Bytes32::from_str(s).is_err()); + } + + #[test] + fn test_from_str_too_big() { + let s = + "0123456789ABCDEF0123456789ABCDEF01234567890123456789ABCDEF01234567890123456789ABCDEF0"; // 65 characters + assert!(Bytes32::from_str(s).is_err()); + } +} diff --git a/arbitrator/prover/src/prepare.rs b/arbitrator/prover/src/prepare.rs index ecfb517735..a485267f39 100644 --- a/arbitrator/prover/src/prepare.rs +++ b/arbitrator/prover/src/prepare.rs @@ -42,7 +42,6 @@ pub fn prepare_machine(preimages: PathBuf, machines: PathBuf) -> eyre::Result Date: Wed, 18 Sep 2024 18:47:30 +0200 Subject: [PATCH 215/489] Remove commented out implementation. --- arbitrator/arbutil/src/types.rs | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/arbitrator/arbutil/src/types.rs b/arbitrator/arbutil/src/types.rs index e92d4dfbc5..722a89b81e 100644 --- a/arbitrator/arbutil/src/types.rs +++ b/arbitrator/arbutil/src/types.rs @@ -109,22 +109,6 @@ impl FromStr for Bytes32 { Ok(Bytes32(b)) } } -/* -impl FromStr for Bytes32 { - type Err = hex::FromHexError; - - fn from_str(s: &str) -> Result { - let trimmed = match s.strip_prefix("0x") { - Some(t) => t, - None => s, - }; - let bytes = hex::decode(trimmed)?; - let mut b = [0u8; 32]; - b.copy_from_slice(&bytes); - Ok(Self(b)) - } -} -*/ impl TryFrom<&[u8]> for Bytes32 { type Error = std::array::TryFromSliceError; From 109098e7b1d98391c9bcaede903434ed1a7f07e8 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 18 Sep 2024 12:57:26 -0600 Subject: [PATCH 216/489] init-reorg: fix logic --- cmd/nitro/nitro.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index a13a92fdc1..26301a4e27 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -1006,14 +1006,15 @@ func initReorg(initConfig conf.InitConfig, chainConfig *params.ChainConfig, inbo return nil } // Reorg out the batch containing the next message - var missing bool + var found bool var err error - batchCount, missing, err = inboxTracker.FindInboxBatchContainingMessage(messageIndex + 1) + batchCount, found, err = inboxTracker.FindInboxBatchContainingMessage(messageIndex + 1) if err != nil { return err } - if missing { - return fmt.Errorf("cannot reorg to unknown message index %v", messageIndex) + if !found { + log.Warn("init-reorg: no need to reorg, because message ahead of chain", "messageIndex", messageIndex) + return nil } } return inboxTracker.ReorgBatchesTo(batchCount) From 2cdce538f45244c29dc409e4bac56f42dde10921 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Wed, 18 Sep 2024 14:02:39 -0500 Subject: [PATCH 217/489] Improve stability of getNodeCreationBlock for L3s --- staker/rollup_watcher.go | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/staker/rollup_watcher.go b/staker/rollup_watcher.go index 5ef28a49dc..cd7e5d059d 100644 --- a/staker/rollup_watcher.go +++ b/staker/rollup_watcher.go @@ -4,16 +4,19 @@ package staker import ( + "bytes" "context" "encoding/binary" "errors" "fmt" "math/big" + "strings" "sync/atomic" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rpc" "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/solgen/go/rollupgen" "github.com/offchainlabs/nitro/util/headerreader" @@ -51,6 +54,7 @@ type RollupWatcher struct { client arbutil.L1Interface baseCallOpts bind.CallOpts unSupportedL3Method atomic.Bool + supportedL3Method atomic.Bool } func NewRollupWatcher(address common.Address, client arbutil.L1Interface, callOpts bind.CallOpts) (*RollupWatcher, error) { @@ -73,15 +77,40 @@ func (r *RollupWatcher) getCallOpts(ctx context.Context) *bind.CallOpts { return &opts } +const noNodeErr string = "NO_NODE" + +func looksLikeNoNodeError(err error) bool { + if err == nil { + return false + } + if strings.Contains(err.Error(), noNodeErr) { + return true + } + errWithData, ok := err.(rpc.DataError) + if !ok { + return false + } + dataString, ok := errWithData.ErrorData().(string) + if !ok { + return false + } + data := common.FromHex(dataString) + return bytes.Contains(data, []byte(noNodeErr)) +} + func (r *RollupWatcher) getNodeCreationBlock(ctx context.Context, nodeNum uint64) (*big.Int, error) { callOpts := r.getCallOpts(ctx) if !r.unSupportedL3Method.Load() { createdAtBlock, err := r.GetNodeCreationBlockForLogLookup(callOpts, nodeNum) if err == nil { + r.supportedL3Method.Store(true) return createdAtBlock, nil } - log.Trace("failed to call getNodeCreationBlockForLogLookup, falling back on node CreatedAtBlock field", "err", err) - if headerreader.ExecutionRevertedRegexp.MatchString(err.Error()) { + if headerreader.ExecutionRevertedRegexp.MatchString(err.Error()) && !looksLikeNoNodeError(err) { + if r.supportedL3Method.Load() { + return nil, fmt.Errorf("getNodeCreationBlockForLogLookup failed despite previously succeeding: %w", err) + } + log.Trace("failed to call getNodeCreationBlockForLogLookup, falling back on node CreatedAtBlock field", "err", err) r.unSupportedL3Method.Store(true) } else { return nil, err From 32a601ccc8ab957c285ac573b03cea8b0b6c5acc Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Wed, 18 Sep 2024 14:05:07 -0500 Subject: [PATCH 218/489] Fix linter --- staker/rollup_watcher.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/staker/rollup_watcher.go b/staker/rollup_watcher.go index cd7e5d059d..3219566549 100644 --- a/staker/rollup_watcher.go +++ b/staker/rollup_watcher.go @@ -86,7 +86,8 @@ func looksLikeNoNodeError(err error) bool { if strings.Contains(err.Error(), noNodeErr) { return true } - errWithData, ok := err.(rpc.DataError) + var errWithData rpc.DataError + ok := errors.As(err, &errWithData) if !ok { return false } From 2427b619a47d8c770a100e58eabbdb3222128023 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Mon, 23 Sep 2024 16:48:59 +0530 Subject: [PATCH 219/489] Sort storage_flush_cache --- arbos/util/storage_cache.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arbos/util/storage_cache.go b/arbos/util/storage_cache.go index bf05a5824d..dfbdc0892b 100644 --- a/arbos/util/storage_cache.go +++ b/arbos/util/storage_cache.go @@ -4,7 +4,9 @@ package util import ( + "bytes" "github.com/ethereum/go-ethereum/common" + "slices" ) type storageCacheEntry struct { @@ -67,6 +69,10 @@ func (s *storageCache) Flush() []storageCacheStores { }) } } + sortFunc := func(a, b storageCacheStores) int { + return bytes.Compare(a.Key.Bytes(), b.Key.Bytes()) + } + slices.SortFunc(stores, sortFunc) return stores } From dc9a109c2e5e97c9ae292f6eb0aa3aa1b1b72c1d Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Mon, 23 Sep 2024 16:59:01 +0530 Subject: [PATCH 220/489] Only run hash state scheme in race CI --- .github/workflows/ci.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index acd6295b7c..4619e8c2fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -166,19 +166,6 @@ jobs: fi done - - name: run tests with race detection and path state scheme - if: matrix.test-mode == 'race' - env: - TEST_STATE_SCHEME: path - run: | - packages=`go list ./...` - for package in $packages; do - echo running tests for $package - if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -race -timeout 30m > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then - exit 1 - fi - done - - name: run tests with race detection and hash state scheme if: matrix.test-mode == 'race' env: From f8fee5edcbec3dc8da0ac8ce4d7119ad95fa10c5 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 10 Sep 2024 12:47:02 -0300 Subject: [PATCH 221/489] Make scripts/startup-testnode.bash shellcheck compliant --- scripts/startup-testnode.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/startup-testnode.bash b/scripts/startup-testnode.bash index 701e7ff59a..5313a9ec5d 100755 --- a/scripts/startup-testnode.bash +++ b/scripts/startup-testnode.bash @@ -5,9 +5,9 @@ timeout 60 ./nitro-testnode/test-node.bash --init --dev || exit_status=$? -if [ -n "$exit_status" ] && [ $exit_status -ne 0 ] && [ $exit_status -ne 124 ]; then +if [ -n "$exit_status" ] && [ "$exit_status" -ne 0 ] && [ "$exit_status" -ne 124 ]; then echo "Startup failed." - exit $exit_status + exit "$exit_status" fi echo "Startup succeeded." From 6228fd9127eced651576628ff87c97bb090c7b71 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 10 Sep 2024 13:01:16 -0300 Subject: [PATCH 222/489] Make scripts/fuzz.bash shellcheck compliant --- scripts/fuzz.bash | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/scripts/fuzz.bash b/scripts/fuzz.bash index 6271b917b6..a73c208e88 100755 --- a/scripts/fuzz.bash +++ b/scripts/fuzz.bash @@ -2,12 +2,12 @@ set -e -mydir=`dirname $0` +mydir=$(dirname "$0") cd "$mydir" function printusage { - echo Usage: $0 --build \[--binary-path PATH\] - echo " " $0 \ \[--binary-path PATH\] \[--fuzzcache-path PATH\] \[--nitro-path PATH\] \[--duration DURATION\] + echo Usage: "$0" --build \[--binary-path PATH\] + echo " " "$0" \ \[--binary-path PATH\] \[--fuzzcache-path PATH\] \[--nitro-path PATH\] \[--duration DURATION\] echo echo fuzzer names: echo " " FuzzPrecompiles @@ -22,7 +22,6 @@ if [[ $# -eq 0 ]]; then exit fi -fuzz_executable=../target/bin/system_test.fuzz binpath=../target/bin/ fuzzcachepath=../target/var/fuzz-cache nitropath=../ @@ -72,7 +71,7 @@ while [[ $# -gt 0 ]]; do shift ;; FuzzPrecompiles | FuzzStateTransition) - if [[ ! -z "$test_name" ]]; then + if [[ -n "$test_name" ]]; then echo can only run one fuzzer at a time exit 1 fi @@ -81,7 +80,7 @@ while [[ $# -gt 0 ]]; do shift ;; FuzzInboxMultiplexer) - if [[ ! -z "$test_name" ]]; then + if [[ -n "$test_name" ]]; then echo can only run one fuzzer at a time exit 1 fi @@ -102,17 +101,17 @@ fi if $run_build; then for build_group in system_tests arbstate; do - go test -c ${nitropath}/${build_group} -fuzz Fuzz -o "$binpath"/${build_group}.fuzz + go test -c "${nitropath}"/${build_group} -fuzz Fuzz -o "$binpath"/${build_group}.fuzz done fi -if [[ ! -z $test_group ]]; then - timeout "$((60 * duration))" "$binpath"/${test_group}.fuzz -test.run "^$" -test.fuzzcachedir "$fuzzcachepath" -test.fuzz $test_name || exit_status=$? +if [[ -n $test_group ]]; then + timeout "$((60 * duration))" "$binpath"/${test_group}.fuzz -test.run "^$" -test.fuzzcachedir "$fuzzcachepath" -test.fuzz "$test_name" || exit_status=$? fi -if [ -n "$exit_status" ] && [ $exit_status -ne 0 ] && [ $exit_status -ne 124 ]; then +if [ -n "$exit_status" ] && [ "$exit_status" -ne 0 ] && [ "$exit_status" -ne 124 ]; then echo "Fuzzing failed." - exit $exit_status + exit "$exit_status" fi echo "Fuzzing succeeded." From 4df4c39bc02a144420a4297416ba7949b9d62634 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 10 Sep 2024 14:06:00 -0300 Subject: [PATCH 223/489] Make convert-databases.bash shellcheck compliant --- scripts/convert-databases.bash | 43 ++++++++++++++++------------------ 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/scripts/convert-databases.bash b/scripts/convert-databases.bash index 3020b389b4..baddcdcacd 100755 --- a/scripts/convert-databases.bash +++ b/scripts/convert-databases.bash @@ -33,7 +33,7 @@ printStatus() { } printUsage() { -echo Usage: $0 \[OPTIONS..\] +echo Usage: "$0" \[OPTIONS..\] echo echo OPTIONS: echo "--dbconv dbconv binary path (default: \"$DEFAULT_DBCONV\")" @@ -42,15 +42,15 @@ echo Usage: $0 \[OPTIONS..\] echo "--force remove destination directory if it exists" echo "--skip-existing skip convertion of databases which directories already exist in the destination directory" echo "--clean sets what should be removed in case of error, possible values:" - echo " \"failed\" - remove database which conversion failed (default)" + echo " \"failed\" - remove database which conversion failed (default)" echo " \"none\" - remove nothing, leave unfinished and potentially corrupted databases" echo " \"all\" - remove whole destination directory" } removeDir() { cmd="rm -r \"$1\"" - echo $cmd - eval $cmd + echo "$cmd" + eval "$cmd" return $? } @@ -62,7 +62,7 @@ cleanup() { ;; failed) echo "== Note: removing only failed destination directory" - dstdir=$(echo $dst/$1 | tr -s /) + dstdir=$(echo "$dst"/"$1" | tr -s /) removeDir "$dstdir" ;; none) @@ -127,8 +127,8 @@ if $force && $skip_existing; then exit 1 fi -if [ $clean != "all" ] && [ $clean != "failed" ] && [ $clean != "none" ] ; then - echo Error: Invalid --clean value: $clean +if [ "$clean" != "all" ] && [ "$clean" != "failed" ] && [ "$clean" != "none" ] ; then + echo Error: Invalid --clean value: "$clean" printUsage exit 1 fi @@ -138,8 +138,8 @@ if ! [ -e "$dbconv" ]; then exit 1 fi -if ! [ -n "$dst" ]; then - echo Error: Missing destination directory \(\-\-dst\) +if [ -z "$dst" ]; then + echo "Error: Missing destination directory (--dst)" printUsage exit 1 fi @@ -168,9 +168,8 @@ fi if [ -e "$dst" ] && ! $skip_existing; then if $force; then - echo == Warning! Destination already exists, --force is set, removing all files under path: "$dst" - removeDir "$dst" - if [ $? -ne 0 ]; then + echo "== Warning! Destination already exists, --force is set, removing all files under path: $dst" + if ! removeDir "$dst"; then echo Error: failed to remove "$dst" exit 1 fi @@ -183,14 +182,13 @@ fi convert_result= convert () { srcdir="$src"/$1 - dstdir=$(echo $dst/$1 | tr -s /) - if ! [ -e $dstdir ]; then + dstdir=$(echo "$dst"/"$1" | tr -s /) + if ! [ -e "$dstdir" ]; then echo "== Converting $1 db" cmd="$dbconv --src.db-engine=leveldb --src.data \"$srcdir\" --dst.db-engine=pebble --dst.data \"$dstdir\" --convert --compact" - echo $cmd - eval $cmd - if [ $? -ne 0 ]; then - cleanup $1 + echo "$cmd" + if ! eval "$cmd"; then + cleanup "$1" convert_result="FAILED" return 1 fi @@ -221,9 +219,8 @@ if ! [ -e "$dst"/l2chaindata/ancient ]; then ancient_dst=$(echo "$dst"/l2chaindata/ | tr -s /) echo "== Copying l2chaindata ancients" cmd="cp -r \"$ancient_src\" \"$ancient_dst\"" - echo $cmd - eval $cmd - if [ $? -ne 0 ]; then + echo "$cmd" + if ! eval "$cmd"; then l2chaindata_ancient_status="FAILED (failed to copy)" cleanup "l2chaindata" printStatus @@ -249,7 +246,7 @@ if [ $res -ne 0 ]; then exit 1 fi -if [ -e $src/wasm ]; then +if [ -e "$src"/wasm ]; then convert "wasm" res=$? wasm_status=$convert_result @@ -262,7 +259,7 @@ else wasm_status="not found in source directory" fi -if [ -e $src/classic-msg ]; then +if [ -e "$src"/classic-msg ]; then convert "classic-msg" res=$? classicmsg_status=$convert_result From 6efc51b4acc8f727e5a42d847ce6284a17df26f2 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 10 Sep 2024 14:49:41 -0300 Subject: [PATCH 224/489] Make scripts/build-brotli.sh shellcheck compliant --- scripts/build-brotli.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/scripts/build-brotli.sh b/scripts/build-brotli.sh index 7160936baa..1a23a88ae0 100755 --- a/scripts/build-brotli.sh +++ b/scripts/build-brotli.sh @@ -2,7 +2,7 @@ set -e -mydir=`dirname $0` +mydir=$(dirname "$0") cd "$mydir" BUILD_WASM=false @@ -35,7 +35,7 @@ usage(){ echo "all relative paths are relative to script location" } -while getopts "s:t:c:D:wldhf" option; do +while getopts "n:s:t:c:D:wldhf" option; do case $option in h) usage @@ -62,6 +62,9 @@ while getopts "s:t:c:D:wldhf" option; do s) SOURCE_DIR="$OPTARG" ;; + *) + usage + ;; esac done @@ -74,7 +77,7 @@ if [ ! -d "$TARGET_DIR" ]; then mkdir -p "${TARGET_DIR}lib" ln -s "lib" "${TARGET_DIR}lib64" # Fedora build fi -TARGET_DIR_ABS=`cd -P "$TARGET_DIR"; pwd` +TARGET_DIR_ABS=$(cd -P "$TARGET_DIR"; pwd) if $USE_DOCKER; then @@ -94,9 +97,9 @@ cd "$SOURCE_DIR" if $BUILD_WASM; then mkdir -p buildfiles/build-wasm mkdir -p buildfiles/install-wasm - TEMP_INSTALL_DIR_ABS=`cd -P buildfiles/install-wasm; pwd` + TEMP_INSTALL_DIR_ABS=$(cd -P buildfiles/install-wasm; pwd) cd buildfiles/build-wasm - cmake ../../ -DCMAKE_C_COMPILER=emcc -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_FLAGS=-fPIC -DCMAKE_INSTALL_PREFIX="$TEMP_INSTALL_DIR_ABS" -DCMAKE_AR=`which emar` -DCMAKE_RANLIB=`which touch` + cmake ../../ -DCMAKE_C_COMPILER=emcc -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_FLAGS=-fPIC -DCMAKE_INSTALL_PREFIX="$TEMP_INSTALL_DIR_ABS" -DCMAKE_AR="$(which emar)" -DCMAKE_RANLIB="$(which touch)" make -j make install cp -rv "$TEMP_INSTALL_DIR_ABS/lib" "$TARGET_DIR_ABS/lib-wasm" From f7d434344b16c9f259aea77a4ebf5aea8bbca912 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 10 Sep 2024 15:31:15 -0300 Subject: [PATCH 225/489] ShellCheck CI --- .github/workflows/shellcheck-ci.yml | 30 +++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/shellcheck-ci.yml diff --git a/.github/workflows/shellcheck-ci.yml b/.github/workflows/shellcheck-ci.yml new file mode 100644 index 0000000000..c86b4bde1c --- /dev/null +++ b/.github/workflows/shellcheck-ci.yml @@ -0,0 +1,30 @@ +name: ShellCheck CI +run-name: ShellCheck CI triggered from @${{ github.actor }} of ${{ github.head_ref }} + +on: + workflow_dispatch: + merge_group: + pull_request: + push: + branches: + - master + +jobs: + arbitrator: + name: Run ShellCheck + runs-on: ubuntu-8 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Run ShellCheck + uses: ludeeus/action-shellcheck@master + with: + ignore_paths: >- + ./fastcache/** + ./contracts/** + ./safe-smart-account/** + ./go-ethereum/** + ./nitro-testnode/** + ./brotli/** + ./arbitrator/** From 82a2c255093c623800e12ac1f7753ce8a4264f76 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 11 Sep 2024 12:58:05 -0300 Subject: [PATCH 226/489] Rename GitHub action job to shellcheck --- .github/workflows/shellcheck-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/shellcheck-ci.yml b/.github/workflows/shellcheck-ci.yml index c86b4bde1c..d1c7b58580 100644 --- a/.github/workflows/shellcheck-ci.yml +++ b/.github/workflows/shellcheck-ci.yml @@ -10,7 +10,7 @@ on: - master jobs: - arbitrator: + shellcheck: name: Run ShellCheck runs-on: ubuntu-8 steps: From ad0cd310dfb866f90a80dd76b14e03dfd22f84a8 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 5 Sep 2024 10:17:28 -0300 Subject: [PATCH 227/489] Adds chainSupportsBlobs to DeployOnL1 --- cmd/deploy/deploy.go | 1 + deploy/deploy.go | 21 ++++++++++++--------- system_tests/common_test.go | 1 + 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/cmd/deploy/deploy.go b/cmd/deploy/deploy.go index e0388e3d4b..811e54dff5 100644 --- a/cmd/deploy/deploy.go +++ b/cmd/deploy/deploy.go @@ -189,6 +189,7 @@ func main() { arbnode.GenerateRollupConfig(*prod, moduleRoot, ownerAddress, &chainConfig, chainConfigJson, loserEscrowAddress), nativeToken, maxDataSize, + true, ) if err != nil { flag.Usage() diff --git a/deploy/deploy.go b/deploy/deploy.go index f2099f976a..041e1049f5 100644 --- a/deploy/deploy.go +++ b/deploy/deploy.go @@ -31,7 +31,7 @@ func andTxSucceeded(ctx context.Context, l1Reader *headerreader.HeaderReader, tx return nil } -func deployBridgeCreator(ctx context.Context, l1Reader *headerreader.HeaderReader, auth *bind.TransactOpts, maxDataSize *big.Int) (common.Address, error) { +func deployBridgeCreator(ctx context.Context, l1Reader *headerreader.HeaderReader, auth *bind.TransactOpts, maxDataSize *big.Int, chainSupportsBlobs bool) (common.Address, error) { client := l1Reader.Client() /// deploy eth based templates @@ -41,10 +41,13 @@ func deployBridgeCreator(ctx context.Context, l1Reader *headerreader.HeaderReade return common.Address{}, fmt.Errorf("bridge deploy error: %w", err) } - reader4844, tx, _, err := yulgen.DeployReader4844(auth, client) - err = andTxSucceeded(ctx, l1Reader, tx, err) - if err != nil { - return common.Address{}, fmt.Errorf("blob basefee reader deploy error: %w", err) + var reader4844 common.Address + if chainSupportsBlobs { + reader4844, tx, _, err = yulgen.DeployReader4844(auth, client) + err = andTxSucceeded(ctx, l1Reader, tx, err) + if err != nil { + return common.Address{}, fmt.Errorf("blob basefee reader deploy error: %w", err) + } } seqInboxTemplateEthBased, tx, _, err := bridgegen.DeploySequencerInbox(auth, client, maxDataSize, reader4844, false) err = andTxSucceeded(ctx, l1Reader, tx, err) @@ -166,8 +169,8 @@ func deployChallengeFactory(ctx context.Context, l1Reader *headerreader.HeaderRe return ospEntryAddr, challengeManagerAddr, nil } -func deployRollupCreator(ctx context.Context, l1Reader *headerreader.HeaderReader, auth *bind.TransactOpts, maxDataSize *big.Int) (*rollupgen.RollupCreator, common.Address, common.Address, common.Address, error) { - bridgeCreator, err := deployBridgeCreator(ctx, l1Reader, auth, maxDataSize) +func deployRollupCreator(ctx context.Context, l1Reader *headerreader.HeaderReader, auth *bind.TransactOpts, maxDataSize *big.Int, chainSupportsBlobs bool) (*rollupgen.RollupCreator, common.Address, common.Address, common.Address, error) { + bridgeCreator, err := deployBridgeCreator(ctx, l1Reader, auth, maxDataSize, chainSupportsBlobs) if err != nil { return nil, common.Address{}, common.Address{}, common.Address{}, fmt.Errorf("bridge creator deploy error: %w", err) } @@ -239,12 +242,12 @@ func deployRollupCreator(ctx context.Context, l1Reader *headerreader.HeaderReade return rollupCreator, rollupCreatorAddress, validatorUtils, validatorWalletCreator, nil } -func DeployOnL1(ctx context.Context, parentChainReader *headerreader.HeaderReader, deployAuth *bind.TransactOpts, batchPosters []common.Address, batchPosterManager common.Address, authorizeValidators uint64, config rollupgen.Config, nativeToken common.Address, maxDataSize *big.Int) (*chaininfo.RollupAddresses, error) { +func DeployOnL1(ctx context.Context, parentChainReader *headerreader.HeaderReader, deployAuth *bind.TransactOpts, batchPosters []common.Address, batchPosterManager common.Address, authorizeValidators uint64, config rollupgen.Config, nativeToken common.Address, maxDataSize *big.Int, chainSupportsBlobs bool) (*chaininfo.RollupAddresses, error) { if config.WasmModuleRoot == (common.Hash{}) { return nil, errors.New("no machine specified") } - rollupCreator, _, validatorUtils, validatorWalletCreator, err := deployRollupCreator(ctx, parentChainReader, deployAuth, maxDataSize) + rollupCreator, _, validatorUtils, validatorWalletCreator, err := deployRollupCreator(ctx, parentChainReader, deployAuth, maxDataSize, chainSupportsBlobs) if err != nil { return nil, fmt.Errorf("error deploying rollup creator: %w", err) } diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 2607ae5c8b..0895610cfc 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -1040,6 +1040,7 @@ func DeployOnTestL1( arbnode.GenerateRollupConfig(prodConfirmPeriodBlocks, wasmModuleRoot, l1info.GetAddress("RollupOwner"), chainConfig, serializedChainConfig, common.Address{}), nativeToken, maxDataSize, + true, ) Require(t, err) l1info.SetContract("Bridge", addresses.Bridge) From 6a9deba4672e73ff92ca29b92d92e54046a93641 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 5 Sep 2024 10:21:36 -0300 Subject: [PATCH 228/489] Renames DeployOnL1 to DeployOnParentChain --- cmd/deploy/deploy.go | 2 +- deploy/deploy.go | 2 +- system_tests/common_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/deploy/deploy.go b/cmd/deploy/deploy.go index 811e54dff5..c70ceb1d94 100644 --- a/cmd/deploy/deploy.go +++ b/cmd/deploy/deploy.go @@ -179,7 +179,7 @@ func main() { defer l1Reader.StopAndWait() nativeToken := common.HexToAddress(*nativeTokenAddressString) - deployedAddresses, err := deploycode.DeployOnL1( + deployedAddresses, err := deploycode.DeployOnParentChain( ctx, l1Reader, l1TransactionOpts, diff --git a/deploy/deploy.go b/deploy/deploy.go index 041e1049f5..c4f7fc891e 100644 --- a/deploy/deploy.go +++ b/deploy/deploy.go @@ -242,7 +242,7 @@ func deployRollupCreator(ctx context.Context, l1Reader *headerreader.HeaderReade return rollupCreator, rollupCreatorAddress, validatorUtils, validatorWalletCreator, nil } -func DeployOnL1(ctx context.Context, parentChainReader *headerreader.HeaderReader, deployAuth *bind.TransactOpts, batchPosters []common.Address, batchPosterManager common.Address, authorizeValidators uint64, config rollupgen.Config, nativeToken common.Address, maxDataSize *big.Int, chainSupportsBlobs bool) (*chaininfo.RollupAddresses, error) { +func DeployOnParentChain(ctx context.Context, parentChainReader *headerreader.HeaderReader, deployAuth *bind.TransactOpts, batchPosters []common.Address, batchPosterManager common.Address, authorizeValidators uint64, config rollupgen.Config, nativeToken common.Address, maxDataSize *big.Int, chainSupportsBlobs bool) (*chaininfo.RollupAddresses, error) { if config.WasmModuleRoot == (common.Hash{}) { return nil, errors.New("no machine specified") } diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 0895610cfc..ed984a07e0 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -1030,7 +1030,7 @@ func DeployOnTestL1( nativeToken := common.Address{} maxDataSize := big.NewInt(117964) - addresses, err := deploy.DeployOnL1( + addresses, err := deploy.DeployOnParentChain( ctx, l1Reader, &l1TransactionOpts, From 9630bfbd12294ed3c52422c451ebec8765a18c95 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 5 Sep 2024 10:22:26 -0300 Subject: [PATCH 229/489] Renames l1Reader to parentChainReader on deploy --- deploy/deploy.go | 84 ++++++++++++++++++++++++------------------------ 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/deploy/deploy.go b/deploy/deploy.go index c4f7fc891e..bb4b2e6594 100644 --- a/deploy/deploy.go +++ b/deploy/deploy.go @@ -20,23 +20,23 @@ import ( "github.com/offchainlabs/nitro/util/headerreader" ) -func andTxSucceeded(ctx context.Context, l1Reader *headerreader.HeaderReader, tx *types.Transaction, err error) error { +func andTxSucceeded(ctx context.Context, parentChainReader *headerreader.HeaderReader, tx *types.Transaction, err error) error { if err != nil { return fmt.Errorf("error submitting tx: %w", err) } - _, err = l1Reader.WaitForTxApproval(ctx, tx) + _, err = parentChainReader.WaitForTxApproval(ctx, tx) if err != nil { return fmt.Errorf("error executing tx: %w", err) } return nil } -func deployBridgeCreator(ctx context.Context, l1Reader *headerreader.HeaderReader, auth *bind.TransactOpts, maxDataSize *big.Int, chainSupportsBlobs bool) (common.Address, error) { - client := l1Reader.Client() +func deployBridgeCreator(ctx context.Context, parentChainReader *headerreader.HeaderReader, auth *bind.TransactOpts, maxDataSize *big.Int, chainSupportsBlobs bool) (common.Address, error) { + client := parentChainReader.Client() /// deploy eth based templates bridgeTemplate, tx, _, err := bridgegen.DeployBridge(auth, client) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, fmt.Errorf("bridge deploy error: %w", err) } @@ -44,36 +44,36 @@ func deployBridgeCreator(ctx context.Context, l1Reader *headerreader.HeaderReade var reader4844 common.Address if chainSupportsBlobs { reader4844, tx, _, err = yulgen.DeployReader4844(auth, client) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, fmt.Errorf("blob basefee reader deploy error: %w", err) } } seqInboxTemplateEthBased, tx, _, err := bridgegen.DeploySequencerInbox(auth, client, maxDataSize, reader4844, false) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, fmt.Errorf("sequencer inbox eth based deploy error: %w", err) } seqInboxTemplateERC20Based, tx, _, err := bridgegen.DeploySequencerInbox(auth, client, maxDataSize, reader4844, true) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, fmt.Errorf("sequencer inbox erc20 based deploy error: %w", err) } inboxTemplate, tx, _, err := bridgegen.DeployInbox(auth, client, maxDataSize) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, fmt.Errorf("inbox deploy error: %w", err) } rollupEventBridgeTemplate, tx, _, err := rollupgen.DeployRollupEventInbox(auth, client) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, fmt.Errorf("rollup event bridge deploy error: %w", err) } outboxTemplate, tx, _, err := bridgegen.DeployOutbox(auth, client) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, fmt.Errorf("outbox deploy error: %w", err) } @@ -88,25 +88,25 @@ func deployBridgeCreator(ctx context.Context, l1Reader *headerreader.HeaderReade /// deploy ERC20 based templates erc20BridgeTemplate, tx, _, err := bridgegen.DeployERC20Bridge(auth, client) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, fmt.Errorf("bridge deploy error: %w", err) } erc20InboxTemplate, tx, _, err := bridgegen.DeployERC20Inbox(auth, client, maxDataSize) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, fmt.Errorf("inbox deploy error: %w", err) } erc20RollupEventBridgeTemplate, tx, _, err := rollupgen.DeployERC20RollupEventInbox(auth, client) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, fmt.Errorf("rollup event bridge deploy error: %w", err) } erc20OutboxTemplate, tx, _, err := bridgegen.DeployERC20Outbox(auth, client) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, fmt.Errorf("outbox deploy error: %w", err) } @@ -120,7 +120,7 @@ func deployBridgeCreator(ctx context.Context, l1Reader *headerreader.HeaderReade } bridgeCreatorAddr, tx, _, err := rollupgen.DeployBridgeCreator(auth, client, ethBasedTemplates, erc20BasedTemplates) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, fmt.Errorf("bridge creator deploy error: %w", err) } @@ -128,40 +128,40 @@ func deployBridgeCreator(ctx context.Context, l1Reader *headerreader.HeaderReade return bridgeCreatorAddr, nil } -func deployChallengeFactory(ctx context.Context, l1Reader *headerreader.HeaderReader, auth *bind.TransactOpts) (common.Address, common.Address, error) { - client := l1Reader.Client() +func deployChallengeFactory(ctx context.Context, parentChainReader *headerreader.HeaderReader, auth *bind.TransactOpts) (common.Address, common.Address, error) { + client := parentChainReader.Client() osp0, tx, _, err := ospgen.DeployOneStepProver0(auth, client) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, common.Address{}, fmt.Errorf("osp0 deploy error: %w", err) } ospMem, tx, _, err := ospgen.DeployOneStepProverMemory(auth, client) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, common.Address{}, fmt.Errorf("ospMemory deploy error: %w", err) } ospMath, tx, _, err := ospgen.DeployOneStepProverMath(auth, client) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, common.Address{}, fmt.Errorf("ospMath deploy error: %w", err) } ospHostIo, tx, _, err := ospgen.DeployOneStepProverHostIo(auth, client) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, common.Address{}, fmt.Errorf("ospHostIo deploy error: %w", err) } challengeManagerAddr, tx, _, err := challengegen.DeployChallengeManager(auth, client) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, common.Address{}, fmt.Errorf("challenge manager deploy error: %w", err) } ospEntryAddr, tx, _, err := ospgen.DeployOneStepProofEntry(auth, client, osp0, ospMem, ospMath, ospHostIo) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, common.Address{}, fmt.Errorf("ospEntry deploy error: %w", err) } @@ -169,55 +169,55 @@ func deployChallengeFactory(ctx context.Context, l1Reader *headerreader.HeaderRe return ospEntryAddr, challengeManagerAddr, nil } -func deployRollupCreator(ctx context.Context, l1Reader *headerreader.HeaderReader, auth *bind.TransactOpts, maxDataSize *big.Int, chainSupportsBlobs bool) (*rollupgen.RollupCreator, common.Address, common.Address, common.Address, error) { - bridgeCreator, err := deployBridgeCreator(ctx, l1Reader, auth, maxDataSize, chainSupportsBlobs) +func deployRollupCreator(ctx context.Context, parentChainReader *headerreader.HeaderReader, auth *bind.TransactOpts, maxDataSize *big.Int, chainSupportsBlobs bool) (*rollupgen.RollupCreator, common.Address, common.Address, common.Address, error) { + bridgeCreator, err := deployBridgeCreator(ctx, parentChainReader, auth, maxDataSize, chainSupportsBlobs) if err != nil { return nil, common.Address{}, common.Address{}, common.Address{}, fmt.Errorf("bridge creator deploy error: %w", err) } - ospEntryAddr, challengeManagerAddr, err := deployChallengeFactory(ctx, l1Reader, auth) + ospEntryAddr, challengeManagerAddr, err := deployChallengeFactory(ctx, parentChainReader, auth) if err != nil { return nil, common.Address{}, common.Address{}, common.Address{}, err } - rollupAdminLogic, tx, _, err := rollupgen.DeployRollupAdminLogic(auth, l1Reader.Client()) - err = andTxSucceeded(ctx, l1Reader, tx, err) + rollupAdminLogic, tx, _, err := rollupgen.DeployRollupAdminLogic(auth, parentChainReader.Client()) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return nil, common.Address{}, common.Address{}, common.Address{}, fmt.Errorf("rollup admin logic deploy error: %w", err) } - rollupUserLogic, tx, _, err := rollupgen.DeployRollupUserLogic(auth, l1Reader.Client()) - err = andTxSucceeded(ctx, l1Reader, tx, err) + rollupUserLogic, tx, _, err := rollupgen.DeployRollupUserLogic(auth, parentChainReader.Client()) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return nil, common.Address{}, common.Address{}, common.Address{}, fmt.Errorf("rollup user logic deploy error: %w", err) } - rollupCreatorAddress, tx, rollupCreator, err := rollupgen.DeployRollupCreator(auth, l1Reader.Client()) - err = andTxSucceeded(ctx, l1Reader, tx, err) + rollupCreatorAddress, tx, rollupCreator, err := rollupgen.DeployRollupCreator(auth, parentChainReader.Client()) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return nil, common.Address{}, common.Address{}, common.Address{}, fmt.Errorf("rollup creator deploy error: %w", err) } - upgradeExecutor, tx, _, err := upgrade_executorgen.DeployUpgradeExecutor(auth, l1Reader.Client()) - err = andTxSucceeded(ctx, l1Reader, tx, err) + upgradeExecutor, tx, _, err := upgrade_executorgen.DeployUpgradeExecutor(auth, parentChainReader.Client()) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return nil, common.Address{}, common.Address{}, common.Address{}, fmt.Errorf("upgrade executor deploy error: %w", err) } - validatorUtils, tx, _, err := rollupgen.DeployValidatorUtils(auth, l1Reader.Client()) - err = andTxSucceeded(ctx, l1Reader, tx, err) + validatorUtils, tx, _, err := rollupgen.DeployValidatorUtils(auth, parentChainReader.Client()) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return nil, common.Address{}, common.Address{}, common.Address{}, fmt.Errorf("validator utils deploy error: %w", err) } - validatorWalletCreator, tx, _, err := rollupgen.DeployValidatorWalletCreator(auth, l1Reader.Client()) - err = andTxSucceeded(ctx, l1Reader, tx, err) + validatorWalletCreator, tx, _, err := rollupgen.DeployValidatorWalletCreator(auth, parentChainReader.Client()) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return nil, common.Address{}, common.Address{}, common.Address{}, fmt.Errorf("validator wallet creator deploy error: %w", err) } - l2FactoriesDeployHelper, tx, _, err := rollupgen.DeployDeployHelper(auth, l1Reader.Client()) - err = andTxSucceeded(ctx, l1Reader, tx, err) + l2FactoriesDeployHelper, tx, _, err := rollupgen.DeployDeployHelper(auth, parentChainReader.Client()) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return nil, common.Address{}, common.Address{}, common.Address{}, fmt.Errorf("deploy helper creator deploy error: %w", err) } @@ -234,7 +234,7 @@ func deployRollupCreator(ctx context.Context, l1Reader *headerreader.HeaderReade validatorWalletCreator, l2FactoriesDeployHelper, ) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return nil, common.Address{}, common.Address{}, common.Address{}, fmt.Errorf("rollup set template error: %w", err) } From e1edcb3845ef797640558e3148f6c9c4876f0169 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 5 Sep 2024 11:35:05 -0300 Subject: [PATCH 230/489] Basic L3 tests --- system_tests/common_test.go | 261 +++++++++++++++++++++++++++++++----- system_tests/l3_test.go | 37 +++++ 2 files changed, 264 insertions(+), 34 deletions(-) create mode 100644 system_tests/l3_test.go diff --git a/system_tests/common_test.go b/system_tests/common_test.go index ed984a07e0..91b421a7c4 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -235,6 +235,7 @@ type NodeBuilder struct { valnodeConfig *valnode.Config L1Info info L2Info info + L3Info info // L1, L2 Node parameters dataDir string @@ -248,6 +249,54 @@ type NodeBuilder struct { // Created nodes L1 *TestClient L2 *TestClient + L3 *TestClient +} + +type NitroConfig struct { + chainConfig *params.ChainConfig + nodeConfig *arbnode.Config + execConfig *gethexec.Config + stackConfig *node.Config + valnodeConfig *valnode.Config + + withProdConfirmPeriodBlocks bool + isSequencer bool +} + +func L3NitroConfigDefaultTest(t *testing.T) *NitroConfig { + chainConfig := ¶ms.ChainConfig{ + ChainID: big.NewInt(333333), + HomesteadBlock: big.NewInt(0), + DAOForkBlock: nil, + DAOForkSupport: true, + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(0), + MuirGlacierBlock: big.NewInt(0), + BerlinBlock: big.NewInt(0), + LondonBlock: big.NewInt(0), + ArbitrumChainParams: params.ArbitrumDevTestParams(), + Clique: ¶ms.CliqueConfig{ + Period: 0, + Epoch: 0, + }, + } + + valnodeConfig := valnode.TestValidationConfig + return &NitroConfig{ + chainConfig: chainConfig, + nodeConfig: arbnode.ConfigDefaultL2Test(), + execConfig: ExecConfigDefaultTest(t), + stackConfig: testhelpers.CreateStackConfigForTest(t.TempDir()), + valnodeConfig: &valnodeConfig, + + withProdConfirmPeriodBlocks: false, + isSequencer: true, + } } func NewNodeBuilder(ctx context.Context) *NodeBuilder { @@ -337,10 +386,146 @@ func (b *NodeBuilder) BuildL1(t *testing.T) { b.L1Info, b.L1.Client, b.L1.L1Backend, b.L1.Stack = createTestL1BlockChain(t, b.L1Info) locator, err := server_common.NewMachineLocator(b.valnodeConfig.Wasm.RootPath) Require(t, err) - b.addresses, b.initMessage = DeployOnTestL1(t, b.ctx, b.L1Info, b.L1.Client, b.chainConfig, locator.LatestWasmModuleRoot(), b.withProdConfirmPeriodBlocks) + b.addresses, b.initMessage = deployOnParentChain( + t, + b.ctx, + b.L1Info, + b.L1.Client, + &headerreader.TestConfig, + b.chainConfig, + locator.LatestWasmModuleRoot(), + b.withProdConfirmPeriodBlocks, + true, + ) b.L1.cleanup = func() { requireClose(t, b.L1.Stack) } } +func buildOnParentChain( + t *testing.T, + ctx context.Context, + + dataDir string, + + parentChainInfo info, + parentChainTestClient *TestClient, + parentChainId *big.Int, + + chainConfig *params.ChainConfig, + stackConfig *node.Config, + execConfig *gethexec.Config, + nodeConfig *arbnode.Config, + valnodeConfig *valnode.Config, + isSequencer bool, + chainInfo info, + + initMessage *arbostypes.ParsedInitMessage, + addresses *chaininfo.RollupAddresses, +) *TestClient { + if parentChainTestClient == nil { + t.Fatal("must build parent chain before building chain") + } + + chainTestClient := NewTestClient(ctx) + + var chainDb ethdb.Database + var arbDb ethdb.Database + var blockchain *core.BlockChain + _, chainTestClient.Stack, chainDb, arbDb, blockchain = createNonL1BlockChainWithStackConfig( + t, chainInfo, dataDir, chainConfig, initMessage, stackConfig, execConfig) + + var sequencerTxOptsPtr *bind.TransactOpts + var dataSigner signature.DataSignerFunc + if isSequencer { + sequencerTxOpts := parentChainInfo.GetDefaultTransactOpts("Sequencer", ctx) + sequencerTxOptsPtr = &sequencerTxOpts + dataSigner = signature.DataSignerFromPrivateKey(parentChainInfo.GetInfoWithPrivKey("Sequencer").PrivateKey) + } else { + nodeConfig.BatchPoster.Enable = false + nodeConfig.Sequencer = false + nodeConfig.DelayedSequencer.Enable = false + execConfig.Sequencer.Enable = false + } + + var validatorTxOptsPtr *bind.TransactOpts + if nodeConfig.Staker.Enable { + validatorTxOpts := parentChainInfo.GetDefaultTransactOpts("Validator", ctx) + validatorTxOptsPtr = &validatorTxOpts + } + + AddValNodeIfNeeded(t, ctx, nodeConfig, true, "", valnodeConfig.Wasm.RootPath) + + Require(t, execConfig.Validate()) + execConfigToBeUsedInConfigFetcher := execConfig + execConfigFetcher := func() *gethexec.Config { return execConfigToBeUsedInConfigFetcher } + execNode, err := gethexec.CreateExecutionNode(ctx, chainTestClient.Stack, chainDb, blockchain, parentChainTestClient.Client, execConfigFetcher) + Require(t, err) + + fatalErrChan := make(chan error, 10) + chainTestClient.ConsensusNode, err = arbnode.CreateNode( + ctx, chainTestClient.Stack, execNode, arbDb, NewFetcherFromConfig(nodeConfig), blockchain.Config(), parentChainTestClient.Client, + addresses, validatorTxOptsPtr, sequencerTxOptsPtr, dataSigner, fatalErrChan, parentChainId, nil) + Require(t, err) + + err = chainTestClient.ConsensusNode.Start(ctx) + Require(t, err) + + chainTestClient.Client = ClientForStack(t, chainTestClient.Stack) + + StartWatchChanErr(t, ctx, fatalErrChan, chainTestClient.ConsensusNode) + + chainTestClient.ExecNode = getExecNode(t, chainTestClient.ConsensusNode) + chainTestClient.cleanup = func() { chainTestClient.ConsensusNode.StopAndWait() } + + return chainTestClient +} + +func (b *NodeBuilder) BuildL3OnL2(t *testing.T, nitroConfig *NitroConfig) func() { + b.L3Info = NewArbTestInfo(t, nitroConfig.chainConfig.ChainID) + + locator, err := server_common.NewMachineLocator(nitroConfig.valnodeConfig.Wasm.RootPath) + Require(t, err) + + parentChainReaderConfig := headerreader.TestConfig + parentChainReaderConfig.Dangerous.WaitForTxApprovalSafePoll = 0 + addresses, initMessage := deployOnParentChain( + t, + b.ctx, + b.L2Info, + b.L2.Client, + &parentChainReaderConfig, + nitroConfig.chainConfig, + locator.LatestWasmModuleRoot(), + nitroConfig.withProdConfirmPeriodBlocks, + false, + ) + + b.L3 = buildOnParentChain( + t, + b.ctx, + + b.dataDir, + + b.L2Info, + b.L2, + b.chainConfig.ChainID, + + nitroConfig.chainConfig, + nitroConfig.stackConfig, + nitroConfig.execConfig, + nitroConfig.nodeConfig, + nitroConfig.valnodeConfig, + nitroConfig.isSequencer, + b.L3Info, + + initMessage, + addresses, + ) + + return func() { + b.L3.cleanup() + } +} + func (b *NodeBuilder) BuildL2OnL1(t *testing.T) func() { if b.L1 == nil { t.Fatal("must build L1 before building L2") @@ -350,7 +535,7 @@ func (b *NodeBuilder) BuildL2OnL1(t *testing.T) func() { var l2chainDb ethdb.Database var l2arbDb ethdb.Database var l2blockchain *core.BlockChain - _, b.L2.Stack, l2chainDb, l2arbDb, l2blockchain = createL2BlockChainWithStackConfig( + _, b.L2.Stack, l2chainDb, l2arbDb, l2blockchain = createNonL1BlockChainWithStackConfig( t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, b.execConfig) var sequencerTxOptsPtr *bind.TransactOpts @@ -465,7 +650,7 @@ func (b *NodeBuilder) RestartL2Node(t *testing.T) { } b.L2.cleanup() - l2info, stack, chainDb, arbDb, blockchain := createL2BlockChainWithStackConfig(t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, b.execConfig) + l2info, stack, chainDb, arbDb, blockchain := createNonL1BlockChainWithStackConfig(t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, b.execConfig) execConfigFetcher := func() *gethexec.Config { return b.execConfig } execNode, err := gethexec.CreateExecutionNode(b.ctx, stack, chainDb, blockchain, nil, execConfigFetcher) @@ -1004,60 +1189,68 @@ func getInitMessage(ctx context.Context, t *testing.T, l1client client, addresse return initMessage } -func DeployOnTestL1( - t *testing.T, ctx context.Context, l1info info, l1client client, chainConfig *params.ChainConfig, wasmModuleRoot common.Hash, prodConfirmPeriodBlocks bool, +func deployOnParentChain( + t *testing.T, + ctx context.Context, + parentChainInfo info, + parentChainClient client, + parentChainReaderConfig *headerreader.Config, + chainConfig *params.ChainConfig, + wasmModuleRoot common.Hash, + prodConfirmPeriodBlocks bool, + chainSupportsBlobs bool, ) (*chaininfo.RollupAddresses, *arbostypes.ParsedInitMessage) { - l1info.GenerateAccount("RollupOwner") - l1info.GenerateAccount("Sequencer") - l1info.GenerateAccount("Validator") - l1info.GenerateAccount("User") - - SendWaitTestTransactions(t, ctx, l1client, []*types.Transaction{ - l1info.PrepareTx("Faucet", "RollupOwner", 30000, big.NewInt(9223372036854775807), nil), - l1info.PrepareTx("Faucet", "Sequencer", 30000, big.NewInt(9223372036854775807), nil), - l1info.PrepareTx("Faucet", "Validator", 30000, big.NewInt(9223372036854775807), nil), - l1info.PrepareTx("Faucet", "User", 30000, big.NewInt(9223372036854775807), nil)}) - - l1TransactionOpts := l1info.GetDefaultTransactOpts("RollupOwner", ctx) + parentChainInfo.GenerateAccount("RollupOwner") + parentChainInfo.GenerateAccount("Sequencer") + parentChainInfo.GenerateAccount("Validator") + parentChainInfo.GenerateAccount("User") + + SendWaitTestTransactions(t, ctx, parentChainClient, []*types.Transaction{ + parentChainInfo.PrepareTx("Faucet", "RollupOwner", parentChainInfo.TransferGas, big.NewInt(9223372036854775807), nil), + parentChainInfo.PrepareTx("Faucet", "Sequencer", parentChainInfo.TransferGas, big.NewInt(9223372036854775807), nil), + parentChainInfo.PrepareTx("Faucet", "Validator", parentChainInfo.TransferGas, big.NewInt(9223372036854775807), nil), + parentChainInfo.PrepareTx("Faucet", "User", parentChainInfo.TransferGas, big.NewInt(9223372036854775807), nil)}) + + parentChainTransactionOpts := parentChainInfo.GetDefaultTransactOpts("RollupOwner", ctx) serializedChainConfig, err := json.Marshal(chainConfig) Require(t, err) - arbSys, _ := precompilesgen.NewArbSys(types.ArbSysAddress, l1client) - l1Reader, err := headerreader.New(ctx, l1client, func() *headerreader.Config { return &headerreader.TestConfig }, arbSys) + arbSys, _ := precompilesgen.NewArbSys(types.ArbSysAddress, parentChainClient) + parentChainReader, err := headerreader.New(ctx, parentChainClient, func() *headerreader.Config { return parentChainReaderConfig }, arbSys) Require(t, err) - l1Reader.Start(ctx) - defer l1Reader.StopAndWait() + parentChainReader.Start(ctx) + defer parentChainReader.StopAndWait() nativeToken := common.Address{} maxDataSize := big.NewInt(117964) addresses, err := deploy.DeployOnParentChain( ctx, - l1Reader, - &l1TransactionOpts, - []common.Address{l1info.GetAddress("Sequencer")}, - l1info.GetAddress("RollupOwner"), + parentChainReader, + &parentChainTransactionOpts, + []common.Address{parentChainInfo.GetAddress("Sequencer")}, + parentChainInfo.GetAddress("RollupOwner"), 0, - arbnode.GenerateRollupConfig(prodConfirmPeriodBlocks, wasmModuleRoot, l1info.GetAddress("RollupOwner"), chainConfig, serializedChainConfig, common.Address{}), + arbnode.GenerateRollupConfig(prodConfirmPeriodBlocks, wasmModuleRoot, parentChainInfo.GetAddress("RollupOwner"), chainConfig, serializedChainConfig, common.Address{}), nativeToken, maxDataSize, - true, + chainSupportsBlobs, ) Require(t, err) - l1info.SetContract("Bridge", addresses.Bridge) - l1info.SetContract("SequencerInbox", addresses.SequencerInbox) - l1info.SetContract("Inbox", addresses.Inbox) - l1info.SetContract("UpgradeExecutor", addresses.UpgradeExecutor) - initMessage := getInitMessage(ctx, t, l1client, addresses) + parentChainInfo.SetContract("Bridge", addresses.Bridge) + parentChainInfo.SetContract("SequencerInbox", addresses.SequencerInbox) + parentChainInfo.SetContract("Inbox", addresses.Inbox) + parentChainInfo.SetContract("UpgradeExecutor", addresses.UpgradeExecutor) + initMessage := getInitMessage(ctx, t, parentChainClient, addresses) return addresses, initMessage } func createL2BlockChain( t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, execConfig *gethexec.Config, ) (*BlockchainTestInfo, *node.Node, ethdb.Database, ethdb.Database, *core.BlockChain) { - return createL2BlockChainWithStackConfig(t, l2info, dataDir, chainConfig, nil, nil, execConfig) + return createNonL1BlockChainWithStackConfig(t, l2info, dataDir, chainConfig, nil, nil, execConfig) } -func createL2BlockChainWithStackConfig( +func createNonL1BlockChainWithStackConfig( t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, initMessage *arbostypes.ParsedInitMessage, stackConfig *node.Config, execConfig *gethexec.Config, ) (*BlockchainTestInfo, *node.Node, ethdb.Database, ethdb.Database, *core.BlockChain) { if l2info == nil { diff --git a/system_tests/l3_test.go b/system_tests/l3_test.go new file mode 100644 index 0000000000..5737ed73c4 --- /dev/null +++ b/system_tests/l3_test.go @@ -0,0 +1,37 @@ +package arbtest + +import ( + "context" + "math/big" + "testing" +) + +func TestBasicL3(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, true) + cleanupL1AndL2 := builder.Build(t) + defer cleanupL1AndL2() + + l3NodeNitroConfig := L3NitroConfigDefaultTest(t) + l3NodeNitroConfig.nodeConfig.ParentChainReader.Enable = true + cleanupL3 := builder.BuildL3OnL2(t, l3NodeNitroConfig) + defer cleanupL3() + + builder.L3Info.GenerateAccount("User2") + tx := builder.L3Info.PrepareTx("Owner", "User2", builder.L3Info.TransferGas, big.NewInt(1e12), nil) + + err := builder.L3.Client.SendTransaction(ctx, tx) + Require(t, err) + + _, err = builder.L3.EnsureTxSucceeded(tx) + Require(t, err) + + l2balance, err := builder.L3.Client.BalanceAt(ctx, builder.L3Info.GetAddress("User2"), nil) + Require(t, err) + if l2balance.Cmp(big.NewInt(1e12)) != 0 { + t.Fatal("Unexpected balance:", l2balance) + } +} From 88ad39c40c1a2d333316e8c6af4ecc76e9e2fdb9 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 5 Sep 2024 13:10:36 -0300 Subject: [PATCH 231/489] Uses buildOnParentChain on BuildL2OnL1 --- system_tests/common_test.go | 65 ++++++++++--------------------------- 1 file changed, 17 insertions(+), 48 deletions(-) diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 91b421a7c4..f89fea2733 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -527,59 +527,28 @@ func (b *NodeBuilder) BuildL3OnL2(t *testing.T, nitroConfig *NitroConfig) func() } func (b *NodeBuilder) BuildL2OnL1(t *testing.T) func() { - if b.L1 == nil { - t.Fatal("must build L1 before building L2") - } - b.L2 = NewTestClient(b.ctx) - - var l2chainDb ethdb.Database - var l2arbDb ethdb.Database - var l2blockchain *core.BlockChain - _, b.L2.Stack, l2chainDb, l2arbDb, l2blockchain = createNonL1BlockChainWithStackConfig( - t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, b.execConfig) - - var sequencerTxOptsPtr *bind.TransactOpts - var dataSigner signature.DataSignerFunc - if b.isSequencer { - sequencerTxOpts := b.L1Info.GetDefaultTransactOpts("Sequencer", b.ctx) - sequencerTxOptsPtr = &sequencerTxOpts - dataSigner = signature.DataSignerFromPrivateKey(b.L1Info.GetInfoWithPrivKey("Sequencer").PrivateKey) - } else { - b.nodeConfig.BatchPoster.Enable = false - b.nodeConfig.Sequencer = false - b.nodeConfig.DelayedSequencer.Enable = false - b.execConfig.Sequencer.Enable = false - } - - var validatorTxOptsPtr *bind.TransactOpts - if b.nodeConfig.Staker.Enable { - validatorTxOpts := b.L1Info.GetDefaultTransactOpts("Validator", b.ctx) - validatorTxOptsPtr = &validatorTxOpts - } - - AddValNodeIfNeeded(t, b.ctx, b.nodeConfig, true, "", b.valnodeConfig.Wasm.RootPath) - - Require(t, b.execConfig.Validate()) - execConfig := b.execConfig - execConfigFetcher := func() *gethexec.Config { return execConfig } - execNode, err := gethexec.CreateExecutionNode(b.ctx, b.L2.Stack, l2chainDb, l2blockchain, b.L1.Client, execConfigFetcher) - Require(t, err) + b.L2 = buildOnParentChain( + t, + b.ctx, - fatalErrChan := make(chan error, 10) - b.L2.ConsensusNode, err = arbnode.CreateNode( - b.ctx, b.L2.Stack, execNode, l2arbDb, NewFetcherFromConfig(b.nodeConfig), l2blockchain.Config(), b.L1.Client, - b.addresses, validatorTxOptsPtr, sequencerTxOptsPtr, dataSigner, fatalErrChan, big.NewInt(1337), nil) - Require(t, err) + b.dataDir, - err = b.L2.ConsensusNode.Start(b.ctx) - Require(t, err) + b.L1Info, + b.L1, + big.NewInt(1337), - b.L2.Client = ClientForStack(t, b.L2.Stack) + b.chainConfig, + b.l2StackConfig, + b.execConfig, + b.nodeConfig, + b.valnodeConfig, + b.isSequencer, + b.L2Info, - StartWatchChanErr(t, b.ctx, fatalErrChan, b.L2.ConsensusNode) + b.initMessage, + b.addresses, + ) - b.L2.ExecNode = getExecNode(t, b.L2.ConsensusNode) - b.L2.cleanup = func() { b.L2.ConsensusNode.StopAndWait() } return func() { b.L2.cleanup() if b.L1 != nil && b.L1.cleanup != nil { From 479f85a1636ae746bb6e66c678455b0b4c08635e Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 5 Sep 2024 13:13:22 -0300 Subject: [PATCH 232/489] Rename l1Client to parentChainClient in getInitMessage --- system_tests/common_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/system_tests/common_test.go b/system_tests/common_test.go index f89fea2733..9f2a2bb2be 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -1143,8 +1143,8 @@ func createTestL1BlockChain(t *testing.T, l1info info) (info, *ethclient.Client, return l1info, l1Client, l1backend, stack } -func getInitMessage(ctx context.Context, t *testing.T, l1client client, addresses *chaininfo.RollupAddresses) *arbostypes.ParsedInitMessage { - bridge, err := arbnode.NewDelayedBridge(l1client, addresses.Bridge, addresses.DeployedAt) +func getInitMessage(ctx context.Context, t *testing.T, parentChainClient client, addresses *chaininfo.RollupAddresses) *arbostypes.ParsedInitMessage { + bridge, err := arbnode.NewDelayedBridge(parentChainClient, addresses.Bridge, addresses.DeployedAt) Require(t, err) deployedAtBig := arbmath.UintToBig(addresses.DeployedAt) messages, err := bridge.LookupMessagesInRange(ctx, deployedAtBig, deployedAtBig, nil) From c1bd83e437f9479bbcf93a60bfd72f13dde33703 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 5 Sep 2024 13:14:41 -0300 Subject: [PATCH 233/489] Rename l2info to info in createNonL1BlockChainWithStackConfig --- system_tests/common_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 9f2a2bb2be..160815f4bb 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -1220,10 +1220,10 @@ func createL2BlockChain( } func createNonL1BlockChainWithStackConfig( - t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, initMessage *arbostypes.ParsedInitMessage, stackConfig *node.Config, execConfig *gethexec.Config, + t *testing.T, info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, initMessage *arbostypes.ParsedInitMessage, stackConfig *node.Config, execConfig *gethexec.Config, ) (*BlockchainTestInfo, *node.Node, ethdb.Database, ethdb.Database, *core.BlockChain) { - if l2info == nil { - l2info = NewArbTestInfo(t, chainConfig.ChainID) + if info == nil { + info = NewArbTestInfo(t, chainConfig.ChainID) } if stackConfig == nil { stackConfig = testhelpers.CreateStackConfigForTest(dataDir) @@ -1243,7 +1243,7 @@ func createNonL1BlockChainWithStackConfig( arbDb, err := stack.OpenDatabaseWithExtraOptions("arbitrumdata", 0, 0, "arbitrumdata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("arbitrumdata")) Require(t, err) - initReader := statetransfer.NewMemoryInitDataReader(&l2info.ArbInitData) + initReader := statetransfer.NewMemoryInitDataReader(&info.ArbInitData) if initMessage == nil { serializedChainConfig, err := json.Marshal(chainConfig) Require(t, err) @@ -1258,7 +1258,7 @@ func createNonL1BlockChainWithStackConfig( blockchain, err := gethexec.WriteOrTestBlockChain(chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest(t).TxLookupLimit, 0) Require(t, err) - return l2info, stack, chainDb, arbDb, blockchain + return info, stack, chainDb, arbDb, blockchain } func ClientForStack(t *testing.T, backend *node.Node) *ethclient.Client { From 5852df193419b66e1641821afbe2d466b4ee5441 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 5 Sep 2024 15:13:54 -0300 Subject: [PATCH 234/489] Adds l3Config to NodeBuilder --- system_tests/common_test.go | 26 ++++++++++++++------------ system_tests/l3_test.go | 4 +--- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 160815f4bb..2eba898e95 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -233,6 +233,7 @@ type NodeBuilder struct { l1StackConfig *node.Config l2StackConfig *node.Config valnodeConfig *valnode.Config + l3Config *NitroConfig L1Info info L2Info info L3Info info @@ -289,7 +290,7 @@ func L3NitroConfigDefaultTest(t *testing.T) *NitroConfig { valnodeConfig := valnode.TestValidationConfig return &NitroConfig{ chainConfig: chainConfig, - nodeConfig: arbnode.ConfigDefaultL2Test(), + nodeConfig: arbnode.ConfigDefaultL1Test(), execConfig: ExecConfigDefaultTest(t), stackConfig: testhelpers.CreateStackConfigForTest(t.TempDir()), valnodeConfig: &valnodeConfig, @@ -322,6 +323,7 @@ func (b *NodeBuilder) DefaultConfig(t *testing.T, withL1 bool) *NodeBuilder { cp := valnode.TestValidationConfig b.valnodeConfig = &cp b.execConfig = ExecConfigDefaultTest(t) + b.l3Config = L3NitroConfigDefaultTest(t) return b } @@ -479,10 +481,10 @@ func buildOnParentChain( return chainTestClient } -func (b *NodeBuilder) BuildL3OnL2(t *testing.T, nitroConfig *NitroConfig) func() { - b.L3Info = NewArbTestInfo(t, nitroConfig.chainConfig.ChainID) +func (b *NodeBuilder) BuildL3OnL2(t *testing.T) func() { + b.L3Info = NewArbTestInfo(t, b.l3Config.chainConfig.ChainID) - locator, err := server_common.NewMachineLocator(nitroConfig.valnodeConfig.Wasm.RootPath) + locator, err := server_common.NewMachineLocator(b.l3Config.valnodeConfig.Wasm.RootPath) Require(t, err) parentChainReaderConfig := headerreader.TestConfig @@ -493,9 +495,9 @@ func (b *NodeBuilder) BuildL3OnL2(t *testing.T, nitroConfig *NitroConfig) func() b.L2Info, b.L2.Client, &parentChainReaderConfig, - nitroConfig.chainConfig, + b.l3Config.chainConfig, locator.LatestWasmModuleRoot(), - nitroConfig.withProdConfirmPeriodBlocks, + b.l3Config.withProdConfirmPeriodBlocks, false, ) @@ -509,12 +511,12 @@ func (b *NodeBuilder) BuildL3OnL2(t *testing.T, nitroConfig *NitroConfig) func() b.L2, b.chainConfig.ChainID, - nitroConfig.chainConfig, - nitroConfig.stackConfig, - nitroConfig.execConfig, - nitroConfig.nodeConfig, - nitroConfig.valnodeConfig, - nitroConfig.isSequencer, + b.l3Config.chainConfig, + b.l3Config.stackConfig, + b.l3Config.execConfig, + b.l3Config.nodeConfig, + b.l3Config.valnodeConfig, + b.l3Config.isSequencer, b.L3Info, initMessage, diff --git a/system_tests/l3_test.go b/system_tests/l3_test.go index 5737ed73c4..6ed10e3193 100644 --- a/system_tests/l3_test.go +++ b/system_tests/l3_test.go @@ -15,9 +15,7 @@ func TestBasicL3(t *testing.T) { cleanupL1AndL2 := builder.Build(t) defer cleanupL1AndL2() - l3NodeNitroConfig := L3NitroConfigDefaultTest(t) - l3NodeNitroConfig.nodeConfig.ParentChainReader.Enable = true - cleanupL3 := builder.BuildL3OnL2(t, l3NodeNitroConfig) + cleanupL3 := builder.BuildL3OnL2(t) defer cleanupL3() builder.L3Info.GenerateAccount("User2") From 467ec77ab125b7cc40808888a14f2c5ecf93d871 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 5 Sep 2024 16:35:54 -0300 Subject: [PATCH 235/489] Build2ndNodeOnL3 --- system_tests/common_test.go | 151 ++++++++++++++++++++++++++---------- system_tests/l3_test.go | 34 ++++++-- 2 files changed, 135 insertions(+), 50 deletions(-) diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 2eba898e95..f894eb065f 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -238,13 +238,15 @@ type NodeBuilder struct { L2Info info L3Info info - // L1, L2 Node parameters + // L1, L2, L3 Node parameters dataDir string isSequencer bool takeOwnership bool withL1 bool addresses *chaininfo.RollupAddresses + l3Addresses *chaininfo.RollupAddresses initMessage *arbostypes.ParsedInitMessage + l3InitMessage *arbostypes.ParsedInitMessage withProdConfirmPeriodBlocks bool // Created nodes @@ -489,7 +491,7 @@ func (b *NodeBuilder) BuildL3OnL2(t *testing.T) func() { parentChainReaderConfig := headerreader.TestConfig parentChainReaderConfig.Dangerous.WaitForTxApprovalSafePoll = 0 - addresses, initMessage := deployOnParentChain( + b.l3Addresses, b.l3InitMessage = deployOnParentChain( t, b.ctx, b.L2Info, @@ -519,8 +521,8 @@ func (b *NodeBuilder) BuildL3OnL2(t *testing.T) func() { b.l3Config.isSequencer, b.L3Info, - initMessage, - addresses, + b.l3InitMessage, + b.l3Addresses, ) return func() { @@ -646,13 +648,25 @@ func (b *NodeBuilder) RestartL2Node(t *testing.T) { b.L2Info = l2info } -func (b *NodeBuilder) Build2ndNode(t *testing.T, params *SecondNodeParams) (*TestClient, func()) { - if b.L2 == nil { - t.Fatal("builder did not previously build a L2 Node") - } - if b.withL1 && b.L1 == nil { - t.Fatal("builder did not previously build a L1 Node") - } +func build2ndNode( + t *testing.T, + ctx context.Context, + + firstNodeStackConfig *node.Config, + firsNodeExecConfig *gethexec.Config, + firstNodeNodeConfig *arbnode.Config, + firstNodeInfo info, + firstNodeTestClient *TestClient, + valnodeConfig *valnode.Config, + + parentChainTestClient *TestClient, + parentChainInfo info, + + params *SecondNodeParams, + + addresses *chaininfo.RollupAddresses, + initMessage *arbostypes.ParsedInitMessage, +) (*TestClient, func()) { if params.nodeConfig == nil { params.nodeConfig = arbnode.ConfigDefaultL1NonSequencerTest() } @@ -660,18 +674,18 @@ func (b *NodeBuilder) Build2ndNode(t *testing.T, params *SecondNodeParams) (*Tes params.nodeConfig.DataAvailability = *params.dasConfig } if params.stackConfig == nil { - params.stackConfig = b.l2StackConfig + params.stackConfig = firstNodeStackConfig // should use different dataDir from the previously used ones params.stackConfig.DataDir = t.TempDir() } if params.initData == nil { - params.initData = &b.L2Info.ArbInitData + params.initData = &firstNodeInfo.ArbInitData } if params.execConfig == nil { - params.execConfig = b.execConfig + params.execConfig = firsNodeExecConfig } if params.addresses == nil { - params.addresses = b.addresses + params.addresses = addresses } if params.execConfig.RPC.MaxRecreateStateDepth == arbitrum.UninitializedMaxRecreateStateDepth { if params.execConfig.Caching.Archive { @@ -680,16 +694,69 @@ func (b *NodeBuilder) Build2ndNode(t *testing.T, params *SecondNodeParams) (*Tes params.execConfig.RPC.MaxRecreateStateDepth = arbitrum.DefaultNonArchiveNodeMaxRecreateStateDepth } } - if b.nodeConfig.BatchPoster.Enable && params.nodeConfig.BatchPoster.Enable && params.nodeConfig.BatchPoster.RedisUrl == "" { + if firstNodeNodeConfig.BatchPoster.Enable && params.nodeConfig.BatchPoster.Enable && params.nodeConfig.BatchPoster.RedisUrl == "" { t.Fatal("The batch poster must use Redis when enabled for multiple nodes") } - l2 := NewTestClient(b.ctx) - l2.Client, l2.ConsensusNode = - Create2ndNodeWithConfig(t, b.ctx, b.L2.ConsensusNode, b.L1.Stack, b.L1Info, params.initData, params.nodeConfig, params.execConfig, params.stackConfig, b.valnodeConfig, params.addresses, b.initMessage) - l2.ExecNode = getExecNode(t, l2.ConsensusNode) - l2.cleanup = func() { l2.ConsensusNode.StopAndWait() } - return l2, func() { l2.cleanup() } + testClient := NewTestClient(ctx) + testClient.Client, testClient.ConsensusNode = + Create2ndNodeWithConfig(t, ctx, firstNodeTestClient.ConsensusNode, parentChainTestClient.Stack, parentChainInfo, params.initData, params.nodeConfig, params.execConfig, params.stackConfig, valnodeConfig, params.addresses, initMessage) + testClient.ExecNode = getExecNode(t, testClient.ConsensusNode) + testClient.cleanup = func() { testClient.ConsensusNode.StopAndWait() } + return testClient, func() { testClient.cleanup() } +} + +func (b *NodeBuilder) Build2ndNode(t *testing.T, params *SecondNodeParams) (*TestClient, func()) { + if b.L2 == nil { + t.Fatal("builder did not previously built an L2 Node") + } + if b.withL1 && b.L1 == nil { + t.Fatal("builder did not previously built an L1 Node") + } + return build2ndNode( + t, + b.ctx, + + b.l2StackConfig, + b.execConfig, + b.nodeConfig, + b.L2Info, + b.L2, + b.valnodeConfig, + + b.L1, + b.L1Info, + + params, + + b.addresses, + b.initMessage, + ) +} + +func (b *NodeBuilder) Build2ndNodeOnL3(t *testing.T, params *SecondNodeParams) (*TestClient, func()) { + if b.L3 == nil { + t.Fatal("builder did not previously built an L3 Node") + } + return build2ndNode( + t, + b.ctx, + + b.l3Config.stackConfig, + b.l3Config.execConfig, + b.l3Config.nodeConfig, + b.L3Info, + b.L3, + b.l3Config.valnodeConfig, + + b.L2, + b.L2Info, + + params, + + b.l3Addresses, + b.l3InitMessage, + ) } func (b *NodeBuilder) BridgeBalance(t *testing.T, account string, amount *big.Int) (*types.Transaction, *types.Receipt) { @@ -1303,9 +1370,9 @@ func Create2ndNodeWithConfig( t *testing.T, ctx context.Context, first *arbnode.Node, - l1stack *node.Node, - l1info *BlockchainTestInfo, - l2InitData *statetransfer.ArbosInitializationInfo, + parentChainStack *node.Node, + parentChainInfo *BlockchainTestInfo, + chainInitData *statetransfer.ArbosInitializationInfo, nodeConfig *arbnode.Config, execConfig *gethexec.Config, stackConfig *node.Config, @@ -1320,34 +1387,34 @@ func Create2ndNodeWithConfig( execConfig = ExecConfigDefaultNonSequencerTest(t) } feedErrChan := make(chan error, 10) - l1rpcClient := l1stack.Attach() - l1client := ethclient.NewClient(l1rpcClient) + parentChainRpcClient := parentChainStack.Attach() + parentChainClient := ethclient.NewClient(parentChainRpcClient) if stackConfig == nil { stackConfig = testhelpers.CreateStackConfigForTest(t.TempDir()) } - l2stack, err := node.New(stackConfig) + chainStack, err := node.New(stackConfig) Require(t, err) - l2chainData, err := l2stack.OpenDatabaseWithExtraOptions("l2chaindata", 0, 0, "l2chaindata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("l2chaindata")) + chainData, err := chainStack.OpenDatabaseWithExtraOptions("l2chaindata", 0, 0, "l2chaindata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("l2chaindata")) Require(t, err) - wasmData, err := l2stack.OpenDatabaseWithExtraOptions("wasm", 0, 0, "wasm/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("wasm")) + wasmData, err := chainStack.OpenDatabaseWithExtraOptions("wasm", 0, 0, "wasm/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("wasm")) Require(t, err) - l2chainDb := rawdb.WrapDatabaseWithWasm(l2chainData, wasmData, 0, execConfig.StylusTarget.WasmTargets()) + chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmData, 0, execConfig.StylusTarget.WasmTargets()) - l2arbDb, err := l2stack.OpenDatabaseWithExtraOptions("arbitrumdata", 0, 0, "arbitrumdata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("arbitrumdata")) + arbDb, err := chainStack.OpenDatabaseWithExtraOptions("arbitrumdata", 0, 0, "arbitrumdata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("arbitrumdata")) Require(t, err) - initReader := statetransfer.NewMemoryInitDataReader(l2InitData) + initReader := statetransfer.NewMemoryInitDataReader(chainInitData) - dataSigner := signature.DataSignerFromPrivateKey(l1info.GetInfoWithPrivKey("Sequencer").PrivateKey) - sequencerTxOpts := l1info.GetDefaultTransactOpts("Sequencer", ctx) - validatorTxOpts := l1info.GetDefaultTransactOpts("Validator", ctx) + dataSigner := signature.DataSignerFromPrivateKey(parentChainInfo.GetInfoWithPrivKey("Sequencer").PrivateKey) + sequencerTxOpts := parentChainInfo.GetDefaultTransactOpts("Sequencer", ctx) + validatorTxOpts := parentChainInfo.GetDefaultTransactOpts("Validator", ctx) firstExec := getExecNode(t, first) chainConfig := firstExec.ArbInterface.BlockChain().Config() - coreCacheConfig := gethexec.DefaultCacheConfigFor(l2stack, &execConfig.Caching) - l2blockchain, err := gethexec.WriteOrTestBlockChain(l2chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest(t).TxLookupLimit, 0) + coreCacheConfig := gethexec.DefaultCacheConfigFor(chainStack, &execConfig.Caching) + blockchain, err := gethexec.WriteOrTestBlockChain(chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest(t).TxLookupLimit, 0) Require(t, err) AddValNodeIfNeeded(t, ctx, nodeConfig, true, "", valnodeConfig.Wasm.RootPath) @@ -1355,19 +1422,19 @@ func Create2ndNodeWithConfig( Require(t, execConfig.Validate()) Require(t, nodeConfig.Validate()) configFetcher := func() *gethexec.Config { return execConfig } - currentExec, err := gethexec.CreateExecutionNode(ctx, l2stack, l2chainDb, l2blockchain, l1client, configFetcher) + currentExec, err := gethexec.CreateExecutionNode(ctx, chainStack, chainDb, blockchain, parentChainClient, configFetcher) Require(t, err) - currentNode, err := arbnode.CreateNode(ctx, l2stack, currentExec, l2arbDb, NewFetcherFromConfig(nodeConfig), l2blockchain.Config(), l1client, addresses, &validatorTxOpts, &sequencerTxOpts, dataSigner, feedErrChan, big.NewInt(1337), nil) + currentNode, err := arbnode.CreateNode(ctx, chainStack, currentExec, arbDb, NewFetcherFromConfig(nodeConfig), blockchain.Config(), parentChainClient, addresses, &validatorTxOpts, &sequencerTxOpts, dataSigner, feedErrChan, big.NewInt(1337), nil) Require(t, err) err = currentNode.Start(ctx) Require(t, err) - l2client := ClientForStack(t, l2stack) + chainClient := ClientForStack(t, chainStack) StartWatchChanErr(t, ctx, feedErrChan, currentNode) - return l2client, currentNode + return chainClient, currentNode } func GetBalance(t *testing.T, ctx context.Context, client *ethclient.Client, account common.Address) *big.Int { diff --git a/system_tests/l3_test.go b/system_tests/l3_test.go index 6ed10e3193..97eabcee78 100644 --- a/system_tests/l3_test.go +++ b/system_tests/l3_test.go @@ -4,9 +4,12 @@ import ( "context" "math/big" "testing" + "time" + + "github.com/offchainlabs/nitro/arbnode" ) -func TestBasicL3(t *testing.T) { +func TestSimpleL3(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -15,19 +18,34 @@ func TestBasicL3(t *testing.T) { cleanupL1AndL2 := builder.Build(t) defer cleanupL1AndL2() - cleanupL3 := builder.BuildL3OnL2(t) - defer cleanupL3() + cleanupL3FirstNode := builder.BuildL3OnL2(t) + defer cleanupL3FirstNode() + firstNodeTestClient := builder.L3 + + secondNodeNodeConfig := arbnode.ConfigDefaultL1NonSequencerTest() + secondNodeTestClient, cleanupL3SecondNode := builder.Build2ndNodeOnL3(t, &SecondNodeParams{nodeConfig: secondNodeNodeConfig}) + defer cleanupL3SecondNode() - builder.L3Info.GenerateAccount("User2") - tx := builder.L3Info.PrepareTx("Owner", "User2", builder.L3Info.TransferGas, big.NewInt(1e12), nil) + accountName := "User2" + builder.L3Info.GenerateAccount(accountName) + tx := builder.L3Info.PrepareTx("Owner", accountName, builder.L3Info.TransferGas, big.NewInt(1e12), nil) - err := builder.L3.Client.SendTransaction(ctx, tx) + err := firstNodeTestClient.Client.SendTransaction(ctx, tx) Require(t, err) - _, err = builder.L3.EnsureTxSucceeded(tx) + // Checks that first node has the correct balance + _, err = firstNodeTestClient.EnsureTxSucceeded(tx) Require(t, err) + l2balance, err := firstNodeTestClient.Client.BalanceAt(ctx, builder.L3Info.GetAddress(accountName), nil) + Require(t, err) + if l2balance.Cmp(big.NewInt(1e12)) != 0 { + t.Fatal("Unexpected balance:", l2balance) + } - l2balance, err := builder.L3.Client.BalanceAt(ctx, builder.L3Info.GetAddress("User2"), nil) + // Checks that second node has the correct balance + _, err = WaitForTx(ctx, secondNodeTestClient.Client, tx.Hash(), time.Second*15) + Require(t, err) + l2balance, err = secondNodeTestClient.Client.BalanceAt(ctx, builder.L3Info.GetAddress(accountName), nil) Require(t, err) if l2balance.Cmp(big.NewInt(1e12)) != 0 { t.Fatal("Unexpected balance:", l2balance) From e8bc32c6603a50a094e4f94f1657b0cfad1af108 Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Sat, 7 Sep 2024 00:24:10 +0800 Subject: [PATCH 236/489] feat: support google cloud storage as das --- das/das.go | 8 +- das/factory.go | 9 ++ das/google_cloud_storage_service.go | 126 ++++++++++++++++++++++++++++ go.mod | 56 +++++++++---- go.sum | 122 ++++++++++++++++++++------- 5 files changed, 272 insertions(+), 49 deletions(-) create mode 100644 das/google_cloud_storage_service.go diff --git a/das/das.go b/das/das.go index 6bd02fbc75..0b03c05ad6 100644 --- a/das/das.go +++ b/das/das.go @@ -41,9 +41,10 @@ type DataAvailabilityConfig struct { LocalCache CacheConfig `koanf:"local-cache"` RedisCache RedisConfig `koanf:"redis-cache"` - LocalDBStorage LocalDBStorageConfig `koanf:"local-db-storage"` - LocalFileStorage LocalFileStorageConfig `koanf:"local-file-storage"` - S3Storage S3StorageServiceConfig `koanf:"s3-storage"` + LocalDBStorage LocalDBStorageConfig `koanf:"local-db-storage"` + LocalFileStorage LocalFileStorageConfig `koanf:"local-file-storage"` + S3Storage S3StorageServiceConfig `koanf:"s3-storage"` + GoogleCloudStorage GoogleCloudStorageServiceConfig `koanf:"google-cloud-storage"` MigrateLocalDBToFileStorage bool `koanf:"migrate-local-db-to-file-storage"` @@ -114,6 +115,7 @@ func dataAvailabilityConfigAddOptions(prefix string, f *flag.FlagSet, r role) { LocalDBStorageConfigAddOptions(prefix+".local-db-storage", f) LocalFileStorageConfigAddOptions(prefix+".local-file-storage", f) S3ConfigAddOptions(prefix+".s3-storage", f) + GoogleCloudConfigAddOptions(prefix+".google-cloud-storage", f) f.Bool(prefix+".migrate-local-db-to-file-storage", DefaultDataAvailabilityConfig.MigrateLocalDBToFileStorage, "daserver will migrate all data on startup from local-db-storage to local-file-storage, then mark local-db-storage as unusable") // Key config for storage diff --git a/das/factory.go b/das/factory.go index 7f696912b3..0d78a18246 100644 --- a/das/factory.go +++ b/das/factory.go @@ -65,6 +65,15 @@ func CreatePersistentStorageService( storageServices = append(storageServices, s) } + if config.GoogleCloudStorage.Enable { + s, err := NewGoogleCloudStorageService(config.GoogleCloudStorage) + if err != nil { + return nil, nil, err + } + lifecycleManager.Register(s) + storageServices = append(storageServices, s) + } + if len(storageServices) > 1 { s, err := NewRedundantStorageService(ctx, storageServices) if err != nil { diff --git a/das/google_cloud_storage_service.go b/das/google_cloud_storage_service.go new file mode 100644 index 0000000000..8c8c4716db --- /dev/null +++ b/das/google_cloud_storage_service.go @@ -0,0 +1,126 @@ +package das + +import ( + googlestorage "cloud.google.com/go/storage" + "context" + "encoding/hex" + "fmt" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "github.com/google/go-cmp/cmp" + "github.com/offchainlabs/nitro/arbstate/daprovider" + "github.com/offchainlabs/nitro/das/dastree" + "github.com/offchainlabs/nitro/util/pretty" + flag "github.com/spf13/pflag" + "google.golang.org/api/option" + "io" + "sort" +) + +type GoogleCloudStorageServiceConfig struct { + Enable bool `koanf:"enable"` + AccessToken string `koanf:"access-token"` + Bucket string `koanf:"bucket"` + ObjectPrefix string `koanf:"object-prefix"` + DiscardAfterTimeout bool `koanf:"discard-after-timeout"` +} + +var DefaultGoogleCloudStorageServiceConfig = GoogleCloudStorageServiceConfig{} + +func GoogleCloudConfigAddOptions(prefix string, f *flag.FlagSet) { + f.Bool(prefix+".enable", DefaultGoogleCloudStorageServiceConfig.Enable, "enable storage/retrieval of sequencer batch data from an Google Cloud Storage bucket") + f.String(prefix+".access-token", DefaultGoogleCloudStorageServiceConfig.AccessToken, "Google Cloud Storage access token") + f.String(prefix+".bucket", DefaultGoogleCloudStorageServiceConfig.Bucket, "Google Cloud Storage bucket") + f.String(prefix+".object-prefix", DefaultGoogleCloudStorageServiceConfig.ObjectPrefix, "prefix to add to Google Cloud Storage objects") + f.Bool(prefix+".discard-after-timeout", DefaultGoogleCloudStorageServiceConfig.DiscardAfterTimeout, "discard data after its expiry timeout") + +} + +type GoogleCloudStorageService struct { + client *googlestorage.Client + bucket string + objectPrefix string + discardAfterTimeout bool +} + +func NewGoogleCloudStorageService(config GoogleCloudStorageServiceConfig) (StorageService, error) { + client, err := googlestorage.NewClient(context.Background(), option.WithCredentialsJSON([]byte(config.AccessToken))) + if err != nil { + return nil, err + } + return &GoogleCloudStorageService{ + client: client, + bucket: config.Bucket, + objectPrefix: config.ObjectPrefix, + discardAfterTimeout: config.DiscardAfterTimeout, + }, nil +} + +func (gcs *GoogleCloudStorageService) Put(ctx context.Context, value []byte, timeout uint64) error { + logPut("das.GoogleCloudStorageService.Store", value, timeout, gcs) + bucket := gcs.client.Bucket(gcs.bucket).Object(gcs.objectPrefix + EncodeStorageServiceKey(dastree.Hash(value))) + w := bucket.NewWriter(ctx) + if _, err := fmt.Fprintln(w, hex.EncodeToString(value)); err != nil { + log.Error("das.GoogleCloudStorageService.Store", "err", err) + return err + } + return w.Close() +} + +func (gcs *GoogleCloudStorageService) GetByHash(ctx context.Context, key common.Hash) ([]byte, error) { + log.Trace("das.GoogleCloudStorageService.GetByHash", "key", pretty.PrettyHash(key), "this", gcs) + bucket := gcs.client.Bucket(gcs.bucket).Object(gcs.objectPrefix + EncodeStorageServiceKey(key)) + reader, err := bucket.NewReader(ctx) + if err != nil { + return nil, err + } + buf, err := io.ReadAll(reader) + if err != nil { + return nil, err + } + return hex.DecodeString(string(buf)) +} + +func (gcs *GoogleCloudStorageService) ExpirationPolicy(ctx context.Context) (daprovider.ExpirationPolicy, error) { + if gcs.discardAfterTimeout { + return daprovider.DiscardAfterDataTimeout, nil + } + return daprovider.KeepForever, nil +} + +func (gcs *GoogleCloudStorageService) Sync(ctx context.Context) error { + return nil +} + +func (gcs *GoogleCloudStorageService) Close(ctx context.Context) error { + return gcs.client.Close() +} + +func (gcs *GoogleCloudStorageService) String() string { + return fmt.Sprintf("GoogleCloudStorageService(:%s)", gcs.bucket) +} + +func (gcs *GoogleCloudStorageService) HealthCheck(ctx context.Context) error { + bucket := gcs.client.Bucket(gcs.bucket) + // check if we have bucket permissions + permissions := []string{ + "storage.buckets.get", + "storage.buckets.list", + "storage.objects.create", + "storage.objects.delete", + "storage.objects.list", + "storage.objects.get", + } + perms, err := bucket.IAM().TestPermissions(ctx, permissions) + if err != nil { + return err + } + sort.Strings(permissions) + sort.Strings(perms) + if !cmp.Equal(perms, permissions) { + return fmt.Errorf("permissions mismatch (-want +got):\n%s", cmp.Diff(permissions, perms)) + } + // check if bucket exists (and others) + _, err = bucket.Attrs(ctx) + return err +} diff --git a/go.mod b/go.mod index da49b0d8b9..6407eec991 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ replace github.com/VictoriaMetrics/fastcache => ./fastcache replace github.com/ethereum/go-ethereum => ./go-ethereum require ( + cloud.google.com/go/storage v1.43.0 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible github.com/Shopify/toxiproxy v2.1.4+incompatible github.com/alicebob/miniredis/v2 v2.32.1 @@ -30,7 +31,7 @@ require ( github.com/gobwas/ws-examples v0.0.0-20190625122829-a9e8908d9484 github.com/google/btree v1.1.2 github.com/google/go-cmp v0.6.0 - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.6.0 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/holiman/uint256 v1.2.4 github.com/knadh/koanf v1.4.0 @@ -42,15 +43,38 @@ require ( github.com/spf13/pflag v1.0.5 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/wealdtech/go-merkletree v1.0.0 - golang.org/x/crypto v0.21.0 + golang.org/x/crypto v0.24.0 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa - golang.org/x/sys v0.18.0 - golang.org/x/term v0.18.0 - golang.org/x/tools v0.16.0 + golang.org/x/sys v0.21.0 + golang.org/x/term v0.21.0 + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d gopkg.in/natefinch/lumberjack.v2 v2.0.0 ) -require github.com/google/go-querystring v1.1.0 // indirect +require ( + cloud.google.com/go v0.115.0 // indirect + cloud.google.com/go/auth v0.6.1 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect + cloud.google.com/go/compute/metadata v0.3.0 // indirect + cloud.google.com/go/iam v1.1.8 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/go-querystring v1.1.0 // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/gax-go/v2 v2.12.5 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.opentelemetry.io/otel/trace v1.24.0 // indirect + google.golang.org/api v0.187.0 // indirect + google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d // indirect + google.golang.org/grpc v1.64.0 // indirect +) require ( github.com/DataDog/zstd v1.4.5 // indirect @@ -107,9 +131,9 @@ require ( github.com/gofrs/flock v0.8.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect - github.com/golang/glog v1.0.0 // indirect - github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/glog v1.2.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect github.com/google/flatbuffers v1.12.1 // indirect github.com/google/go-github/v62 v62.0.0 @@ -159,13 +183,13 @@ require ( github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/yuin/gopher-lua v1.1.1 // indirect - go.opencensus.io v0.22.5 // indirect - golang.org/x/mod v0.14.0 // indirect - golang.org/x/net v0.23.0 // indirect + go.opencensus.io v0.24.0 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/net v0.26.0 // indirect golang.org/x/oauth2 v0.22.0 - golang.org/x/sync v0.5.0 - golang.org/x/text v0.14.0 // indirect - golang.org/x/time v0.3.0 // indirect - google.golang.org/protobuf v1.33.0 // indirect + golang.org/x/sync v0.7.0 + golang.org/x/text v0.16.0 // indirect + golang.org/x/time v0.5.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) diff --git a/go.sum b/go.sum index c0193be769..6120382844 100644 --- a/go.sum +++ b/go.sum @@ -13,14 +13,26 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.115.0 h1:CnFSK6Xo3lDYRoBKEcAtia6VSC837/ZkJuRduSFnr14= +cloud.google.com/go v0.115.0/go.mod h1:8jIM5vVgoAEoiVxQ/O4BFTfHqulPZgs/ufEzMcFMdWU= +cloud.google.com/go/auth v0.6.1 h1:T0Zw1XM5c1GlpN2HYr2s+m3vr1p2wy+8VN+Z1FKxW38= +cloud.google.com/go/auth v0.6.1/go.mod h1:eFHG7zDzbXHKmjJddFG/rBlcGp6t25SwRUiEQSlO4x4= +cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4= +cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/iam v1.1.8 h1:r7umDwhj+BQyz0ScZMp4QrGXjSTI3ZINnpgU2nlB/K0= +cloud.google.com/go/iam v1.1.8/go.mod h1:GvE6lyMmfxXauzNq8NbgJbeVQNspG+tcdL/W8QO1+zE= +cloud.google.com/go/longrunning v0.5.7 h1:WLbHekDbjK1fVFD3ibpFFVoyizlLRl73I7YKuAKilhU= +cloud.google.com/go/longrunning v0.5.7/go.mod h1:8GClkudohy1Fxm3owmBGid8W0pSgodEMwEAztp38Xng= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -30,6 +42,8 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs= +cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -233,6 +247,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fjl/memsize v0.0.2 h1:27txuSD9or+NZlnOWdKUxeBzTAUkWCVh+4Gf2dWFOzA= github.com/fjl/memsize v0.0.2/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -269,6 +285,11 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= @@ -303,12 +324,13 @@ github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzq github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= -github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= +github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -332,8 +354,8 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= @@ -353,6 +375,7 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -368,8 +391,11 @@ github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -381,12 +407,18 @@ github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8q github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b h1:RMpPgZTSApbPf7xaVel+QkoGPRLFLrwFO89uDUHEGf0= github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.12.5 h1:8gw9KZK8TiVKB6q3zHY3SBzLnrGp6HQjyfYBYGmXdxA= +github.com/googleapis/gax-go/v2 v2.12.5/go.mod h1:BUDKcWo+RaKq5SC9vVYL0wLADa3VcfswbOMMRmB9H3E= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= @@ -670,14 +702,19 @@ github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobt github.com/status-im/keycard-go v0.2.0/go.mod h1:wlp8ZLbsmrF6g6WjugPAx+IzoLrkdf9+mHxBEeo3Hbg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= @@ -729,8 +766,20 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= +go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= golang.org/x/crypto v0.0.0-20180214000028-650f4a345ab4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -743,8 +792,8 @@ golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -781,8 +830,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180406214816-61147c48b25b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -819,6 +868,7 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -827,8 +877,8 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -851,8 +901,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -928,14 +978,14 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -947,14 +997,15 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1003,8 +1054,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM= -golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1025,6 +1076,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.187.0 h1:Mxs7VATVC2v7CY+7Xwm4ndkX71hpElcvx0D1Ji/p1eo= +google.golang.org/api v0.187.0/go.mod h1:KIHlTc4x7N7gKKuVsdmfBXN13yEEWXWFURWY6SBp2gk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1063,6 +1116,12 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d h1:PksQg4dV6Sem3/HkBX+Ltq8T0ke0PKIRBNBatoDTVls= +google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d/go.mod h1:s7iA721uChleev562UJO2OYB0PPT9CMFjV+Ce7VJH5M= +google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 h1:MuYw1wJzT+ZkybKfaOXKp5hJiZDn2iHaXRw0mRYdHSc= +google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4/go.mod h1:px9SlOOZBg1wM1zdnr8jEL4CNGUBZ+ZKYtNPApNQc4c= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d h1:k3zyW3BYYR30e8v3x0bTDdE9vpYFjZHK+HcyqkrppWk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -1078,7 +1137,10 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1091,8 +1153,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From e6039aaddc0f3a2efc7433306959077e7ce9566b Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Sat, 7 Sep 2024 02:01:27 +0800 Subject: [PATCH 237/489] refactor: adjust interface for easily unit tests --- cmd/genericconf/config.go | 18 +++++++ das/google_cloud_storage_service.go | 62 ++++++++++++++++++------ das/google_cloud_storage_service_test.go | 57 ++++++++++++++++++++++ 3 files changed, 122 insertions(+), 15 deletions(-) create mode 100644 das/google_cloud_storage_service_test.go diff --git a/cmd/genericconf/config.go b/cmd/genericconf/config.go index 06e1fcd12d..89dd0f336e 100644 --- a/cmd/genericconf/config.go +++ b/cmd/genericconf/config.go @@ -65,6 +65,24 @@ var DefaultS3Config = S3Config{ SecretKey: "", } +type GoogleCloudStorageConfig struct { + AccessToken string `koanf:"access-token"` + Bucket string `koanf:"bucket"` + ObjectPrefix string `koanf:"object-prefix"` +} + +var DefaultGoogleCloudStorageConfig = GoogleCloudStorageConfig{ + AccessToken: "", + Bucket: "", + ObjectPrefix: "", +} + +func GoogleCloudConfigAddOptions(prefix string, f *flag.FlagSet) { + f.String(prefix+".access-token", DefaultGoogleCloudStorageConfig.AccessToken, "Google Cloud Storage access token") + f.String(prefix+".bucket", DefaultGoogleCloudStorageConfig.Bucket, "Google Cloud Storage bucket") + f.String(prefix+".object-prefix", DefaultGoogleCloudStorageConfig.ObjectPrefix, "prefix to add to Google Cloud Storage objects") +} + func HandlerFromLogType(logType string, output io.Writer) (slog.Handler, error) { if logType == "plaintext" { return log.NewTerminalHandler(output, false), nil diff --git a/das/google_cloud_storage_service.go b/das/google_cloud_storage_service.go index 8c8c4716db..62a26422a0 100644 --- a/das/google_cloud_storage_service.go +++ b/das/google_cloud_storage_service.go @@ -17,6 +17,45 @@ import ( "sort" ) +type GoogleCloudStorageOperator interface { + Bucket(name string) *googlestorage.BucketHandle + Upload(ctx context.Context, bucket, objectPrefix string, value []byte) error + Download(ctx context.Context, bucket, objectPrefix string, key common.Hash) ([]byte, error) + Close(ctx context.Context) error +} + +type GoogleCloudStorageClient struct { + client *googlestorage.Client +} + +func (g *GoogleCloudStorageClient) Bucket(name string) *googlestorage.BucketHandle { + return g.client.Bucket(name) +} + +func (g *GoogleCloudStorageClient) Upload(ctx context.Context, bucket, objectPrefix string, value []byte) error { + obj := g.client.Bucket(bucket).Object(objectPrefix + EncodeStorageServiceKey(dastree.Hash(value))) + w := obj.NewWriter(ctx) + + if _, err := fmt.Fprintln(w, hex.EncodeToString(value)); err != nil { + return err + } + return w.Close() + +} + +func (g *GoogleCloudStorageClient) Download(ctx context.Context, bucket, objectPrefix string, key common.Hash) ([]byte, error) { + obj := g.client.Bucket(bucket).Object(objectPrefix + EncodeStorageServiceKey(key)) + reader, err := obj.NewReader(ctx) + if err != nil { + return nil, err + } + return io.ReadAll(reader) +} + +func (g *GoogleCloudStorageClient) Close(ctx context.Context) error { + return g.client.Close() +} + type GoogleCloudStorageServiceConfig struct { Enable bool `koanf:"enable"` AccessToken string `koanf:"access-token"` @@ -33,11 +72,10 @@ func GoogleCloudConfigAddOptions(prefix string, f *flag.FlagSet) { f.String(prefix+".bucket", DefaultGoogleCloudStorageServiceConfig.Bucket, "Google Cloud Storage bucket") f.String(prefix+".object-prefix", DefaultGoogleCloudStorageServiceConfig.ObjectPrefix, "prefix to add to Google Cloud Storage objects") f.Bool(prefix+".discard-after-timeout", DefaultGoogleCloudStorageServiceConfig.DiscardAfterTimeout, "discard data after its expiry timeout") - } type GoogleCloudStorageService struct { - client *googlestorage.Client + operator GoogleCloudStorageOperator bucket string objectPrefix string discardAfterTimeout bool @@ -49,7 +87,7 @@ func NewGoogleCloudStorageService(config GoogleCloudStorageServiceConfig) (Stora return nil, err } return &GoogleCloudStorageService{ - client: client, + operator: &GoogleCloudStorageClient{client: client}, bucket: config.Bucket, objectPrefix: config.ObjectPrefix, discardAfterTimeout: config.DiscardAfterTimeout, @@ -58,24 +96,18 @@ func NewGoogleCloudStorageService(config GoogleCloudStorageServiceConfig) (Stora func (gcs *GoogleCloudStorageService) Put(ctx context.Context, value []byte, timeout uint64) error { logPut("das.GoogleCloudStorageService.Store", value, timeout, gcs) - bucket := gcs.client.Bucket(gcs.bucket).Object(gcs.objectPrefix + EncodeStorageServiceKey(dastree.Hash(value))) - w := bucket.NewWriter(ctx) - if _, err := fmt.Fprintln(w, hex.EncodeToString(value)); err != nil { + if err := gcs.operator.Upload(ctx, gcs.bucket, gcs.objectPrefix, value); err != nil { log.Error("das.GoogleCloudStorageService.Store", "err", err) return err } - return w.Close() + return nil } func (gcs *GoogleCloudStorageService) GetByHash(ctx context.Context, key common.Hash) ([]byte, error) { log.Trace("das.GoogleCloudStorageService.GetByHash", "key", pretty.PrettyHash(key), "this", gcs) - bucket := gcs.client.Bucket(gcs.bucket).Object(gcs.objectPrefix + EncodeStorageServiceKey(key)) - reader, err := bucket.NewReader(ctx) - if err != nil { - return nil, err - } - buf, err := io.ReadAll(reader) + buf, err := gcs.operator.Download(ctx, gcs.bucket, gcs.objectPrefix, key) if err != nil { + log.Error("das.GoogleCloudStorageService.GetByHash", "err", err) return nil, err } return hex.DecodeString(string(buf)) @@ -93,7 +125,7 @@ func (gcs *GoogleCloudStorageService) Sync(ctx context.Context) error { } func (gcs *GoogleCloudStorageService) Close(ctx context.Context) error { - return gcs.client.Close() + return gcs.operator.Close(ctx) } func (gcs *GoogleCloudStorageService) String() string { @@ -101,7 +133,7 @@ func (gcs *GoogleCloudStorageService) String() string { } func (gcs *GoogleCloudStorageService) HealthCheck(ctx context.Context) error { - bucket := gcs.client.Bucket(gcs.bucket) + bucket := gcs.operator.Bucket(gcs.bucket) // check if we have bucket permissions permissions := []string{ "storage.buckets.get", diff --git a/das/google_cloud_storage_service_test.go b/das/google_cloud_storage_service_test.go new file mode 100644 index 0000000000..eeabf9bc36 --- /dev/null +++ b/das/google_cloud_storage_service_test.go @@ -0,0 +1,57 @@ +package das + +import ( + googlestorage "cloud.google.com/go/storage" + "context" + "errors" + "github.com/ethereum/go-ethereum/common" + "github.com/offchainlabs/nitro/cmd/genericconf" + "github.com/offchainlabs/nitro/das/dastree" + "testing" +) + +type mockGCSClient struct { +} + +func (c *mockGCSClient) Bucket(name string) *googlestorage.BucketHandle { + //TODO implement me + panic("implement me") +} + +func (c *mockGCSClient) Download(ctx context.Context, bucket, objectPrefix string, key common.Hash) ([]byte, error) { + return nil, ErrNotFound +} + +func (c *mockGCSClient) Close(ctx context.Context) error { + //TODO implement me + panic("implement me") +} + +func (*mockGCSClient) Upload(ctx context.Context, bucket, objectPrefix string, value []byte) error { + return nil +} + +func NewTestGoogleCloudStorageService(ctx context.Context, googleCloudStorageConfig genericconf.GoogleCloudStorageConfig) (StorageService, error) { + return &GoogleCloudStorageService{ + bucket: googleCloudStorageConfig.Bucket, + objectPrefix: googleCloudStorageConfig.ObjectPrefix, + operator: &mockGCSClient{}, + }, nil +} + +func TestNewGoogleCloudStorageService(t *testing.T) { + ctx := context.Background() + //timeout := uint64(time.Now().Add(time.Hour).Unix()) + googleCloudService, err := NewTestGoogleCloudStorageService(ctx, genericconf.DefaultGoogleCloudStorageConfig) + Require(t, err) + + val1 := []byte("The first value") + val1CorrectKey := dastree.Hash(val1) + //val2IncorrectKey := dastree.Hash(append(val1, 0)) + + _, err = googleCloudService.GetByHash(ctx, val1CorrectKey) + if !errors.Is(err, ErrNotFound) { + t.Fatal(err) + } + +} From 9fe04abf8a59d1a49142ba3c136cbb63c09262a1 Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Sat, 7 Sep 2024 10:13:54 +0800 Subject: [PATCH 238/489] fix: use raw bytes instead of hex --- das/google_cloud_storage_service.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/das/google_cloud_storage_service.go b/das/google_cloud_storage_service.go index 62a26422a0..8f54a00efd 100644 --- a/das/google_cloud_storage_service.go +++ b/das/google_cloud_storage_service.go @@ -3,7 +3,6 @@ package das import ( googlestorage "cloud.google.com/go/storage" "context" - "encoding/hex" "fmt" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" @@ -36,7 +35,7 @@ func (g *GoogleCloudStorageClient) Upload(ctx context.Context, bucket, objectPre obj := g.client.Bucket(bucket).Object(objectPrefix + EncodeStorageServiceKey(dastree.Hash(value))) w := obj.NewWriter(ctx) - if _, err := fmt.Fprintln(w, hex.EncodeToString(value)); err != nil { + if _, err := fmt.Fprintln(w, value); err != nil { return err } return w.Close() @@ -110,7 +109,7 @@ func (gcs *GoogleCloudStorageService) GetByHash(ctx context.Context, key common. log.Error("das.GoogleCloudStorageService.GetByHash", "err", err) return nil, err } - return hex.DecodeString(string(buf)) + return buf, nil } func (gcs *GoogleCloudStorageService) ExpirationPolicy(ctx context.Context) (daprovider.ExpirationPolicy, error) { From 91d7c724ca6a223f6fadeee3b333d46fbacbe56a Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Sat, 7 Sep 2024 11:43:30 +0800 Subject: [PATCH 239/489] fix: test case --- das/google_cloud_storage_service_test.go | 41 ++++++++++++++++++------ 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/das/google_cloud_storage_service_test.go b/das/google_cloud_storage_service_test.go index eeabf9bc36..86c611e2c5 100644 --- a/das/google_cloud_storage_service_test.go +++ b/das/google_cloud_storage_service_test.go @@ -1,6 +1,7 @@ package das import ( + "bytes" googlestorage "cloud.google.com/go/storage" "context" "errors" @@ -8,26 +9,32 @@ import ( "github.com/offchainlabs/nitro/cmd/genericconf" "github.com/offchainlabs/nitro/das/dastree" "testing" + "time" ) type mockGCSClient struct { + storage map[string][]byte } func (c *mockGCSClient) Bucket(name string) *googlestorage.BucketHandle { - //TODO implement me - panic("implement me") + return nil } func (c *mockGCSClient) Download(ctx context.Context, bucket, objectPrefix string, key common.Hash) ([]byte, error) { - return nil, ErrNotFound + value, ok := c.storage[objectPrefix+EncodeStorageServiceKey(key)] + if !ok { + return nil, ErrNotFound + } + return value, nil } func (c *mockGCSClient) Close(ctx context.Context) error { - //TODO implement me - panic("implement me") + return nil } -func (*mockGCSClient) Upload(ctx context.Context, bucket, objectPrefix string, value []byte) error { +func (c *mockGCSClient) Upload(ctx context.Context, bucket, objectPrefix string, value []byte) error { + key := objectPrefix + EncodeStorageServiceKey(dastree.Hash(value)) + c.storage[key] = value return nil } @@ -35,23 +42,39 @@ func NewTestGoogleCloudStorageService(ctx context.Context, googleCloudStorageCon return &GoogleCloudStorageService{ bucket: googleCloudStorageConfig.Bucket, objectPrefix: googleCloudStorageConfig.ObjectPrefix, - operator: &mockGCSClient{}, + operator: &mockGCSClient{ + storage: make(map[string][]byte), + }, }, nil } func TestNewGoogleCloudStorageService(t *testing.T) { ctx := context.Background() - //timeout := uint64(time.Now().Add(time.Hour).Unix()) + timeout := uint64(time.Now().Add(time.Hour).Unix()) googleCloudService, err := NewTestGoogleCloudStorageService(ctx, genericconf.DefaultGoogleCloudStorageConfig) Require(t, err) val1 := []byte("The first value") val1CorrectKey := dastree.Hash(val1) - //val2IncorrectKey := dastree.Hash(append(val1, 0)) + val2IncorrectKey := dastree.Hash(append(val1, 0)) _, err = googleCloudService.GetByHash(ctx, val1CorrectKey) if !errors.Is(err, ErrNotFound) { t.Fatal(err) } + err = googleCloudService.Put(ctx, val1, timeout) + Require(t, err) + + _, err = googleCloudService.GetByHash(ctx, val2IncorrectKey) + if !errors.Is(err, ErrNotFound) { + t.Fatal(err) + } + + val, err := googleCloudService.GetByHash(ctx, val1CorrectKey) + Require(t, err) + if !bytes.Equal(val, val1) { + t.Fatal(val, val1) + } + } From 38b0c60a4d616d0b9eed668fa18ccd739132d3a0 Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Sat, 7 Sep 2024 12:09:32 +0800 Subject: [PATCH 240/489] refactor: use EnableExpire and MaxRetention for expiration polciy --- das/google_cloud_storage_service.go | 65 +++++++++++++++++++---------- 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/das/google_cloud_storage_service.go b/das/google_cloud_storage_service.go index 8f54a00efd..1ace445eed 100644 --- a/das/google_cloud_storage_service.go +++ b/das/google_cloud_storage_service.go @@ -10,10 +10,12 @@ import ( "github.com/offchainlabs/nitro/arbstate/daprovider" "github.com/offchainlabs/nitro/das/dastree" "github.com/offchainlabs/nitro/util/pretty" + "github.com/offchainlabs/nitro/util/stopwaiter" flag "github.com/spf13/pflag" "google.golang.org/api/option" "io" "sort" + "time" ) type GoogleCloudStorageOperator interface { @@ -56,11 +58,12 @@ func (g *GoogleCloudStorageClient) Close(ctx context.Context) error { } type GoogleCloudStorageServiceConfig struct { - Enable bool `koanf:"enable"` - AccessToken string `koanf:"access-token"` - Bucket string `koanf:"bucket"` - ObjectPrefix string `koanf:"object-prefix"` - DiscardAfterTimeout bool `koanf:"discard-after-timeout"` + Enable bool `koanf:"enable"` + AccessToken string `koanf:"access-token"` + Bucket string `koanf:"bucket"` + ObjectPrefix string `koanf:"object-prefix"` + EnableExpiry bool `koanf:"enable-expiry"` + MaxRetention time.Duration `koanf:"max-retention"` } var DefaultGoogleCloudStorageServiceConfig = GoogleCloudStorageServiceConfig{} @@ -70,14 +73,18 @@ func GoogleCloudConfigAddOptions(prefix string, f *flag.FlagSet) { f.String(prefix+".access-token", DefaultGoogleCloudStorageServiceConfig.AccessToken, "Google Cloud Storage access token") f.String(prefix+".bucket", DefaultGoogleCloudStorageServiceConfig.Bucket, "Google Cloud Storage bucket") f.String(prefix+".object-prefix", DefaultGoogleCloudStorageServiceConfig.ObjectPrefix, "prefix to add to Google Cloud Storage objects") - f.Bool(prefix+".discard-after-timeout", DefaultGoogleCloudStorageServiceConfig.DiscardAfterTimeout, "discard data after its expiry timeout") + f.Bool(prefix+".enable-expiry", DefaultLocalFileStorageConfig.EnableExpiry, "enable expiry of batches") + f.Duration(prefix+".max-retention", DefaultLocalFileStorageConfig.MaxRetention, "store requests with expiry times farther in the future than max-retention will be rejected") + } type GoogleCloudStorageService struct { - operator GoogleCloudStorageOperator - bucket string - objectPrefix string - discardAfterTimeout bool + operator GoogleCloudStorageOperator + bucket string + objectPrefix string + enableExpiry bool + maxRetention time.Duration + stopWaiter stopwaiter.StopWaiterSafe } func NewGoogleCloudStorageService(config GoogleCloudStorageServiceConfig) (StorageService, error) { @@ -86,10 +93,11 @@ func NewGoogleCloudStorageService(config GoogleCloudStorageServiceConfig) (Stora return nil, err } return &GoogleCloudStorageService{ - operator: &GoogleCloudStorageClient{client: client}, - bucket: config.Bucket, - objectPrefix: config.ObjectPrefix, - discardAfterTimeout: config.DiscardAfterTimeout, + operator: &GoogleCloudStorageClient{client: client}, + bucket: config.Bucket, + objectPrefix: config.ObjectPrefix, + enableExpiry: config.EnableExpiry, + maxRetention: config.MaxRetention, }, nil } @@ -113,10 +121,10 @@ func (gcs *GoogleCloudStorageService) GetByHash(ctx context.Context, key common. } func (gcs *GoogleCloudStorageService) ExpirationPolicy(ctx context.Context) (daprovider.ExpirationPolicy, error) { - if gcs.discardAfterTimeout { - return daprovider.DiscardAfterDataTimeout, nil + if gcs.enableExpiry { + return daprovider.KeepForever, nil } - return daprovider.KeepForever, nil + return daprovider.DiscardAfterDataTimeout, nil } func (gcs *GoogleCloudStorageService) Sync(ctx context.Context) error { @@ -144,14 +152,29 @@ func (gcs *GoogleCloudStorageService) HealthCheck(ctx context.Context) error { } perms, err := bucket.IAM().TestPermissions(ctx, permissions) if err != nil { - return err + return fmt.Errorf("could not check permissions: %v", err) } sort.Strings(permissions) sort.Strings(perms) if !cmp.Equal(perms, permissions) { return fmt.Errorf("permissions mismatch (-want +got):\n%s", cmp.Diff(permissions, perms)) } - // check if bucket exists (and others) - _, err = bucket.Attrs(ctx) - return err + // check if bucket exists (and others), and update expiration policy if enabled + attrs, err := bucket.Attrs(ctx) + if err != nil { + return err + } + lifecycleRule := googlestorage.LifecycleRule{ + Action: googlestorage.LifecycleAction{Type: "Delete"}, + Condition: googlestorage.LifecycleCondition{AgeInDays: int64(gcs.maxRetention.Hours() / 24)}, // Objects older than 30 days + } + attrs.Lifecycle.Rules = append(attrs.Lifecycle.Rules, lifecycleRule) + + bucketAttrsToUpdate := googlestorage.BucketAttrsToUpdate{ + Lifecycle: &attrs.Lifecycle, + } + if _, err := bucket.Update(ctx, bucketAttrsToUpdate); err != nil { + return fmt.Errorf("failed to update bucket lifecycle: %v", err) + } + return nil } From d466ea5480fab256aec4d351991ab26d5341e37d Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Sat, 7 Sep 2024 12:33:36 +0800 Subject: [PATCH 241/489] fix: enable expiry check --- das/google_cloud_storage_service.go | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/das/google_cloud_storage_service.go b/das/google_cloud_storage_service.go index 1ace445eed..58c15295af 100644 --- a/das/google_cloud_storage_service.go +++ b/das/google_cloud_storage_service.go @@ -164,17 +164,20 @@ func (gcs *GoogleCloudStorageService) HealthCheck(ctx context.Context) error { if err != nil { return err } - lifecycleRule := googlestorage.LifecycleRule{ - Action: googlestorage.LifecycleAction{Type: "Delete"}, - Condition: googlestorage.LifecycleCondition{AgeInDays: int64(gcs.maxRetention.Hours() / 24)}, // Objects older than 30 days + if gcs.enableExpiry { + lifecycleRule := googlestorage.LifecycleRule{ + Action: googlestorage.LifecycleAction{Type: "Delete"}, + Condition: googlestorage.LifecycleCondition{AgeInDays: int64(gcs.maxRetention.Hours() / 24)}, // Objects older than 30 days + } + attrs.Lifecycle.Rules = append(attrs.Lifecycle.Rules, lifecycleRule) + + bucketAttrsToUpdate := googlestorage.BucketAttrsToUpdate{ + Lifecycle: &attrs.Lifecycle, + } + if _, err := bucket.Update(ctx, bucketAttrsToUpdate); err != nil { + return fmt.Errorf("failed to update bucket lifecycle: %v", err) + } } - attrs.Lifecycle.Rules = append(attrs.Lifecycle.Rules, lifecycleRule) - bucketAttrsToUpdate := googlestorage.BucketAttrsToUpdate{ - Lifecycle: &attrs.Lifecycle, - } - if _, err := bucket.Update(ctx, bucketAttrsToUpdate); err != nil { - return fmt.Errorf("failed to update bucket lifecycle: %v", err) - } return nil } From a86c4ec0ada9d3ca96f990d3bf17eb223b76190c Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Sat, 7 Sep 2024 16:16:01 +0800 Subject: [PATCH 242/489] chore: make init client error more readable --- contracts | 2 +- das/google_cloud_storage_service.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contracts b/contracts index 23fc796282..f7894d3a6d 160000 --- a/contracts +++ b/contracts @@ -1 +1 @@ -Subproject commit 23fc79628292aa5d604d449fed48937ae7faeb2f +Subproject commit f7894d3a6d4035ba60f51a7f1334f0f2d4f02dce diff --git a/das/google_cloud_storage_service.go b/das/google_cloud_storage_service.go index 58c15295af..e66cc1cef8 100644 --- a/das/google_cloud_storage_service.go +++ b/das/google_cloud_storage_service.go @@ -90,7 +90,7 @@ type GoogleCloudStorageService struct { func NewGoogleCloudStorageService(config GoogleCloudStorageServiceConfig) (StorageService, error) { client, err := googlestorage.NewClient(context.Background(), option.WithCredentialsJSON([]byte(config.AccessToken))) if err != nil { - return nil, err + return nil, fmt.Errorf("error creating Google Cloud Storage client: %v", err) } return &GoogleCloudStorageService{ operator: &GoogleCloudStorageClient{client: client}, From e9e364356accfb392748c0ed565d0d82d5dc4581 Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Sat, 7 Sep 2024 20:21:01 +0800 Subject: [PATCH 243/489] refactor: support ADC --- das/google_cloud_storage_service.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/das/google_cloud_storage_service.go b/das/google_cloud_storage_service.go index e66cc1cef8..e9104f1334 100644 --- a/das/google_cloud_storage_service.go +++ b/das/google_cloud_storage_service.go @@ -88,7 +88,15 @@ type GoogleCloudStorageService struct { } func NewGoogleCloudStorageService(config GoogleCloudStorageServiceConfig) (StorageService, error) { - client, err := googlestorage.NewClient(context.Background(), option.WithCredentialsJSON([]byte(config.AccessToken))) + var client *googlestorage.Client + var err error + // Note that if the credentials are not specified, the client library will find credentials using ADC(Application Default Credentials) + // https://cloud.google.com/docs/authentication/provide-credentials-adc. + if config.AccessToken == "" { + client, err = googlestorage.NewClient(context.Background()) + } else { + client, err = googlestorage.NewClient(context.Background(), option.WithCredentialsJSON([]byte(config.AccessToken))) + } if err != nil { return nil, fmt.Errorf("error creating Google Cloud Storage client: %v", err) } From 88a2d724ed144e25a9ad38fc9dca466e2fa3838a Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Tue, 10 Sep 2024 10:20:59 +0800 Subject: [PATCH 244/489] fix: check expire time before pushing data --- cmd/genericconf/config.go | 9 ++++++--- das/google_cloud_storage_service.go | 16 +++++++++++++--- das/google_cloud_storage_service_test.go | 5 +++-- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/cmd/genericconf/config.go b/cmd/genericconf/config.go index 89dd0f336e..8bde4a972b 100644 --- a/cmd/genericconf/config.go +++ b/cmd/genericconf/config.go @@ -66,21 +66,24 @@ var DefaultS3Config = S3Config{ } type GoogleCloudStorageConfig struct { - AccessToken string `koanf:"access-token"` - Bucket string `koanf:"bucket"` - ObjectPrefix string `koanf:"object-prefix"` + AccessToken string `koanf:"access-token"` + Bucket string `koanf:"bucket"` + ObjectPrefix string `koanf:"object-prefix"` + MaxRetention time.Duration `koanf:"max-retention"` } var DefaultGoogleCloudStorageConfig = GoogleCloudStorageConfig{ AccessToken: "", Bucket: "", ObjectPrefix: "", + MaxRetention: time.Hour * 24, } func GoogleCloudConfigAddOptions(prefix string, f *flag.FlagSet) { f.String(prefix+".access-token", DefaultGoogleCloudStorageConfig.AccessToken, "Google Cloud Storage access token") f.String(prefix+".bucket", DefaultGoogleCloudStorageConfig.Bucket, "Google Cloud Storage bucket") f.String(prefix+".object-prefix", DefaultGoogleCloudStorageConfig.ObjectPrefix, "prefix to add to Google Cloud Storage objects") + f.Duration(prefix+".max-retention", DefaultGoogleCloudStorageConfig.MaxRetention, "store requests with expiry times farther in the future than max-retention will be rejected") } func HandlerFromLogType(logType string, output io.Writer) (slog.Handler, error) { diff --git a/das/google_cloud_storage_service.go b/das/google_cloud_storage_service.go index e9104f1334..d5ee9c4ace 100644 --- a/das/google_cloud_storage_service.go +++ b/das/google_cloud_storage_service.go @@ -14,6 +14,7 @@ import ( flag "github.com/spf13/pflag" "google.golang.org/api/option" "io" + "math" "sort" "time" ) @@ -109,9 +110,18 @@ func NewGoogleCloudStorageService(config GoogleCloudStorageServiceConfig) (Stora }, nil } -func (gcs *GoogleCloudStorageService) Put(ctx context.Context, value []byte, timeout uint64) error { - logPut("das.GoogleCloudStorageService.Store", value, timeout, gcs) - if err := gcs.operator.Upload(ctx, gcs.bucket, gcs.objectPrefix, value); err != nil { +func (gcs *GoogleCloudStorageService) Put(ctx context.Context, data []byte, expiry uint64) error { + logPut("das.GoogleCloudStorageService.Store", data, expiry, gcs) + if expiry > math.MaxInt64 { + return fmt.Errorf("request expiry time (%v) exceeds max int64", expiry) + } + // #nosec G115 + expiryTime := time.Unix(int64(expiry), 0) + currentTimePlusRetention := time.Now().Add(gcs.maxRetention) + if expiryTime.After(currentTimePlusRetention) { + return fmt.Errorf("requested expiry time (%v) exceeds current time plus maximum allowed retention period(%v)", expiryTime, currentTimePlusRetention) + } + if err := gcs.operator.Upload(ctx, gcs.bucket, gcs.objectPrefix, data); err != nil { log.Error("das.GoogleCloudStorageService.Store", "err", err) return err } diff --git a/das/google_cloud_storage_service_test.go b/das/google_cloud_storage_service_test.go index 86c611e2c5..5482b0c1c2 100644 --- a/das/google_cloud_storage_service_test.go +++ b/das/google_cloud_storage_service_test.go @@ -45,12 +45,13 @@ func NewTestGoogleCloudStorageService(ctx context.Context, googleCloudStorageCon operator: &mockGCSClient{ storage: make(map[string][]byte), }, + maxRetention: googleCloudStorageConfig.MaxRetention, }, nil } func TestNewGoogleCloudStorageService(t *testing.T) { ctx := context.Background() - timeout := uint64(time.Now().Add(time.Hour).Unix()) + expiry := uint64(time.Now().Add(time.Hour).Unix()) googleCloudService, err := NewTestGoogleCloudStorageService(ctx, genericconf.DefaultGoogleCloudStorageConfig) Require(t, err) @@ -63,7 +64,7 @@ func TestNewGoogleCloudStorageService(t *testing.T) { t.Fatal(err) } - err = googleCloudService.Put(ctx, val1, timeout) + err = googleCloudService.Put(ctx, val1, expiry) Require(t, err) _, err = googleCloudService.GetByHash(ctx, val2IncorrectKey) From 4ac2c217ef9b92e30917a9cfa7c7171fbb9966f4 Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Tue, 10 Sep 2024 10:35:25 +0800 Subject: [PATCH 245/489] fix: move health check to constructor --- das/google_cloud_storage_service.go | 45 +++++++++++++++-------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/das/google_cloud_storage_service.go b/das/google_cloud_storage_service.go index d5ee9c4ace..d5804d3002 100644 --- a/das/google_cloud_storage_service.go +++ b/das/google_cloud_storage_service.go @@ -101,13 +101,35 @@ func NewGoogleCloudStorageService(config GoogleCloudStorageServiceConfig) (Stora if err != nil { return nil, fmt.Errorf("error creating Google Cloud Storage client: %v", err) } - return &GoogleCloudStorageService{ + service := &GoogleCloudStorageService{ operator: &GoogleCloudStorageClient{client: client}, bucket: config.Bucket, objectPrefix: config.ObjectPrefix, enableExpiry: config.EnableExpiry, maxRetention: config.MaxRetention, - }, nil + } + if config.EnableExpiry { + lifecycleRule := googlestorage.LifecycleRule{ + Action: googlestorage.LifecycleAction{Type: "Delete"}, + Condition: googlestorage.LifecycleCondition{AgeInDays: int64(config.MaxRetention.Hours() / 24)}, // Objects older than 30 days + } + ctx := context.Background() + bucket := service.operator.Bucket(service.bucket) + // check if bucket exists (and others), and update expiration policy if enabled + attrs, err := bucket.Attrs(ctx) + if err != nil { + return nil, fmt.Errorf("error getting bucket attributes: %v", err) + } + attrs.Lifecycle.Rules = append(attrs.Lifecycle.Rules, lifecycleRule) + + bucketAttrsToUpdate := googlestorage.BucketAttrsToUpdate{ + Lifecycle: &attrs.Lifecycle, + } + if _, err := bucket.Update(ctx, bucketAttrsToUpdate); err != nil { + return nil, fmt.Errorf("failed to update bucket lifecycle: %v", err) + } + } + return service, nil } func (gcs *GoogleCloudStorageService) Put(ctx context.Context, data []byte, expiry uint64) error { @@ -177,25 +199,6 @@ func (gcs *GoogleCloudStorageService) HealthCheck(ctx context.Context) error { if !cmp.Equal(perms, permissions) { return fmt.Errorf("permissions mismatch (-want +got):\n%s", cmp.Diff(permissions, perms)) } - // check if bucket exists (and others), and update expiration policy if enabled - attrs, err := bucket.Attrs(ctx) - if err != nil { - return err - } - if gcs.enableExpiry { - lifecycleRule := googlestorage.LifecycleRule{ - Action: googlestorage.LifecycleAction{Type: "Delete"}, - Condition: googlestorage.LifecycleCondition{AgeInDays: int64(gcs.maxRetention.Hours() / 24)}, // Objects older than 30 days - } - attrs.Lifecycle.Rules = append(attrs.Lifecycle.Rules, lifecycleRule) - - bucketAttrsToUpdate := googlestorage.BucketAttrsToUpdate{ - Lifecycle: &attrs.Lifecycle, - } - if _, err := bucket.Update(ctx, bucketAttrsToUpdate); err != nil { - return fmt.Errorf("failed to update bucket lifecycle: %v", err) - } - } return nil } From 680ee95ddd3b123f237fe29ab586c6731c418f11 Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Tue, 10 Sep 2024 11:17:43 +0800 Subject: [PATCH 246/489] chore: fmt and lint --- das/google_cloud_storage_service.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/das/google_cloud_storage_service.go b/das/google_cloud_storage_service.go index d5804d3002..2c490f346c 100644 --- a/das/google_cloud_storage_service.go +++ b/das/google_cloud_storage_service.go @@ -10,7 +10,6 @@ import ( "github.com/offchainlabs/nitro/arbstate/daprovider" "github.com/offchainlabs/nitro/das/dastree" "github.com/offchainlabs/nitro/util/pretty" - "github.com/offchainlabs/nitro/util/stopwaiter" flag "github.com/spf13/pflag" "google.golang.org/api/option" "io" @@ -85,7 +84,6 @@ type GoogleCloudStorageService struct { objectPrefix string enableExpiry bool maxRetention time.Duration - stopWaiter stopwaiter.StopWaiterSafe } func NewGoogleCloudStorageService(config GoogleCloudStorageServiceConfig) (StorageService, error) { @@ -99,7 +97,7 @@ func NewGoogleCloudStorageService(config GoogleCloudStorageServiceConfig) (Stora client, err = googlestorage.NewClient(context.Background(), option.WithCredentialsJSON([]byte(config.AccessToken))) } if err != nil { - return nil, fmt.Errorf("error creating Google Cloud Storage client: %v", err) + return nil, fmt.Errorf("error creating Google Cloud Storage client: %w", err) } service := &GoogleCloudStorageService{ operator: &GoogleCloudStorageClient{client: client}, @@ -118,7 +116,7 @@ func NewGoogleCloudStorageService(config GoogleCloudStorageServiceConfig) (Stora // check if bucket exists (and others), and update expiration policy if enabled attrs, err := bucket.Attrs(ctx) if err != nil { - return nil, fmt.Errorf("error getting bucket attributes: %v", err) + return nil, fmt.Errorf("error getting bucket attributes: %w", err) } attrs.Lifecycle.Rules = append(attrs.Lifecycle.Rules, lifecycleRule) @@ -126,7 +124,7 @@ func NewGoogleCloudStorageService(config GoogleCloudStorageServiceConfig) (Stora Lifecycle: &attrs.Lifecycle, } if _, err := bucket.Update(ctx, bucketAttrsToUpdate); err != nil { - return nil, fmt.Errorf("failed to update bucket lifecycle: %v", err) + return nil, fmt.Errorf("failed to update bucket lifecycle: %w", err) } } return service, nil @@ -192,7 +190,7 @@ func (gcs *GoogleCloudStorageService) HealthCheck(ctx context.Context) error { } perms, err := bucket.IAM().TestPermissions(ctx, permissions) if err != nil { - return fmt.Errorf("could not check permissions: %v", err) + return fmt.Errorf("could not check permissions: %w", err) } sort.Strings(permissions) sort.Strings(perms) From bee25349260483fed2089ba17a5bf35cecc0fba1 Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Tue, 10 Sep 2024 11:19:54 +0800 Subject: [PATCH 247/489] chore: undo submodule commit for contracts --- contracts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts b/contracts index f7894d3a6d..23fc796282 160000 --- a/contracts +++ b/contracts @@ -1 +1 @@ -Subproject commit f7894d3a6d4035ba60f51a7f1334f0f2d4f02dce +Subproject commit 23fc79628292aa5d604d449fed48937ae7faeb2f From 660e3d9a00becfc150082615a2ae7b0ac500b9f0 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Tue, 24 Sep 2024 18:39:41 +0530 Subject: [PATCH 248/489] Fix soft-float cache --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4619e8c2fe..c7d7427c63 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,7 +91,7 @@ jobs: ~/.cargo/git/ arbitrator/target/ arbitrator/wasm-libraries/target/ - arbitrator/wasm-libraries/soft-float/SoftFloat/build + arbitrator/wasm-libraries/soft-float/ target/etc/initial-machine-cache/ key: ${{ runner.os }}-cargo-${{ steps.install-rust.outputs.rustc_hash }}-min-${{ hashFiles('arbitrator/Cargo.lock') }}-${{ matrix.test-mode }} restore-keys: ${{ runner.os }}-cargo-${{ steps.install-rust.outputs.rustc_hash }}- From 8fab4766214a0acd139524d6769bf287e02973b8 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Tue, 24 Sep 2024 18:51:41 +0530 Subject: [PATCH 249/489] skip test for testing --- .github/workflows/ci.yml | 150 +++++++++++++++++++-------------------- 1 file changed, 75 insertions(+), 75 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c7d7427c63..4808d0403e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -140,81 +140,81 @@ jobs: echo "GOGC=80" >> "$GITHUB_ENV" echo "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" >> "$GITHUB_ENV" - - name: run tests without race detection and path state scheme - if: matrix.test-mode == 'defaults' - env: - TEST_STATE_SCHEME: path - run: | - packages=`go list ./...` - for package in $packages; do - echo running tests for $package - if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -timeout 20m -tags=cionly > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then - exit 1 - fi - done - - - name: run tests without race detection and hash state scheme - if: matrix.test-mode == 'defaults' - env: - TEST_STATE_SCHEME: hash - run: | - packages=`go list ./...` - for package in $packages; do - echo running tests for $package - if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -timeout 20m -tags=cionly; then - exit 1 - fi - done - - - name: run tests with race detection and hash state scheme - if: matrix.test-mode == 'race' - env: - TEST_STATE_SCHEME: hash - run: | - packages=`go list ./...` - for package in $packages; do - echo running tests for $package - if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -race -timeout 30m; then - exit 1 - fi - done - - - name: run redis tests - if: matrix.test-mode == 'defaults' - run: TEST_REDIS=redis://localhost:6379/0 gotestsum --format short-verbose -- -p 1 -run TestRedis ./arbnode/... ./system_tests/... -coverprofile=coverage-redis.txt -covermode=atomic -coverpkg=./... - - - name: run challenge tests - if: matrix.test-mode == 'challenge' - run: | - packages=`go list ./...` - for package in $packages; do - echo running tests for $package - if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -tags=challengetest -run=TestChallenge > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then - exit 1 - fi - done - - - name: run stylus tests - if: matrix.test-mode == 'stylus' - run: | - packages=`go list ./...` - for package in $packages; do - echo running tests for $package - if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -timeout 60m -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -tags=stylustest -run="TestProgramArbitrator" > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then - exit 1 - fi - done - - - name: run long stylus tests - if: matrix.test-mode == 'long' - run: | - packages=`go list ./...` - for package in $packages; do - echo running tests for $package - if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -timeout 60m -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -tags=stylustest -run="TestProgramLong" > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then - exit 1 - fi - done +# - name: run tests without race detection and path state scheme +# if: matrix.test-mode == 'defaults' +# env: +# TEST_STATE_SCHEME: path +# run: | +# packages=`go list ./...` +# for package in $packages; do +# echo running tests for $package +# if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -timeout 20m -tags=cionly > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then +# exit 1 +# fi +# done + +# - name: run tests without race detection and hash state scheme +# if: matrix.test-mode == 'defaults' +# env: +# TEST_STATE_SCHEME: hash +# run: | +# packages=`go list ./...` +# for package in $packages; do +# echo running tests for $package +# if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -timeout 20m -tags=cionly; then +# exit 1 +# fi +# done + +# - name: run tests with race detection and hash state scheme +# if: matrix.test-mode == 'race' +# env: +# TEST_STATE_SCHEME: hash +# run: | +# packages=`go list ./...` +# for package in $packages; do +# echo running tests for $package +# if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -race -timeout 30m; then +# exit 1 +# fi +# done + +# - name: run redis tests +# if: matrix.test-mode == 'defaults' +# run: TEST_REDIS=redis://localhost:6379/0 gotestsum --format short-verbose -- -p 1 -run TestRedis ./arbnode/... ./system_tests/... -coverprofile=coverage-redis.txt -covermode=atomic -coverpkg=./... + +# - name: run challenge tests +# if: matrix.test-mode == 'challenge' +# run: | +# packages=`go list ./...` +# for package in $packages; do +# echo running tests for $package +# if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -tags=challengetest -run=TestChallenge > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then +# exit 1 +# fi +# done + +# - name: run stylus tests +# if: matrix.test-mode == 'stylus' +# run: | +# packages=`go list ./...` +# for package in $packages; do +# echo running tests for $package +# if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -timeout 60m -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -tags=stylustest -run="TestProgramArbitrator" > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then +# exit 1 +# fi +# done + +# - name: run long stylus tests +# if: matrix.test-mode == 'long' +# run: | +# packages=`go list ./...` +# for package in $packages; do +# echo running tests for $package +# if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -timeout 60m -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -tags=stylustest -run="TestProgramLong" > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then +# exit 1 +# fi +# done - name: Archive detailed run log uses: actions/upload-artifact@v3 From c0043e4d052df650b47f7f41b008b104a314f555 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Tue, 24 Sep 2024 19:21:24 +0530 Subject: [PATCH 250/489] Trigger Build From 93c255c8f5f66034e782b4877c6775a8ebead3c8 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Tue, 24 Sep 2024 19:34:12 +0530 Subject: [PATCH 251/489] Trigger Build From 009fa852ac5bf8597c1ef36a4f01943ffc871bf6 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Tue, 24 Sep 2024 20:03:48 +0530 Subject: [PATCH 252/489] fix rust cache --- .github/workflows/ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4808d0403e..453bd00617 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,8 +87,7 @@ jobs: uses: actions/cache@v3 with: path: | - ~/.cargo/registry/ - ~/.cargo/git/ + ~/.cargo/ arbitrator/target/ arbitrator/wasm-libraries/target/ arbitrator/wasm-libraries/soft-float/ From cb8c8f92281b9631f133466da37ac3fc40ac7c8e Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Tue, 24 Sep 2024 09:35:51 -0500 Subject: [PATCH 253/489] Arbitrum Nitro v3.2.0 --- .golangci.yml | 1 + Makefile | 31 +- arbitrator/arbutil/src/evm/api.rs | 2 +- arbitrator/arbutil/src/evm/mod.rs | 3 + arbitrator/arbutil/src/evm/req.rs | 5 +- arbitrator/arbutil/src/hostios.rs | 64 -- arbitrator/arbutil/src/lib.rs | 1 - arbitrator/jit/src/machine.rs | 2 + arbitrator/jit/src/program.rs | 93 ++- arbitrator/prover/Cargo.toml | 4 - arbitrator/prover/build.rs | 89 --- arbitrator/prover/src/binary.rs | 21 +- arbitrator/prover/src/host.rs | 9 +- arbitrator/prover/src/lib.rs | 12 +- arbitrator/prover/src/machine.rs | 269 +++---- arbitrator/prover/src/main.rs | 3 - arbitrator/prover/src/programs/mod.rs | 100 +-- arbitrator/prover/src/test.rs | 2 +- arbitrator/prover/src/value.rs | 12 +- arbitrator/prover/test-cases/forward-test.wat | 32 + .../prover/test-cases/forward/forward.wat | 8 + .../prover/test-cases/forward/target.wat | 27 + arbitrator/stylus/src/host.rs | 89 +-- arbitrator/stylus/src/lib.rs | 13 +- arbitrator/stylus/src/native.rs | 200 ++++-- arbitrator/stylus/src/test/api.rs | 2 +- arbitrator/stylus/src/test/mod.rs | 1 - arbitrator/stylus/src/test/native.rs | 4 +- arbitrator/tools/module_roots/src/main.rs | 1 - arbitrator/wasm-libraries/Cargo.lock | 668 +----------------- arbitrator/wasm-libraries/Cargo.toml | 1 + arbitrator/wasm-libraries/forward/.gitignore | 1 + arbitrator/wasm-libraries/forward/Cargo.toml | 8 + arbitrator/wasm-libraries/forward/src/main.rs | 207 ++++++ .../wasm-libraries/user-host-trait/src/lib.rs | 20 +- .../wasm-libraries/user-host/src/link.rs | 19 +- .../wasm-libraries/user-test/src/program.rs | 2 +- arbitrator/wasm-testsuite/src/main.rs | 2 - arbos/arbosState/arbosstate.go | 58 +- arbos/programs/native.go | 33 +- arbos/programs/programs.go | 6 +- arbos/programs/wasm.go | 11 +- arbos/programs/wasm_api.go | 4 +- arbos/programs/wasmstorehelper.go | 11 +- cmd/chaininfo/arbitrum_chain_info.json | 4 +- cmd/nitro/init.go | 8 +- cmd/nitro/nitro.go | 4 + contracts | 2 +- execution/gethexec/executionengine.go | 4 +- execution/gethexec/wasmstorerebuilder.go | 2 +- go-ethereum | 2 +- precompiles/ArbWasm.go | 8 +- system_tests/block_validator_test.go | 14 + validator/server_arb/machine.go | 2 +- 54 files changed, 935 insertions(+), 1266 deletions(-) delete mode 100644 arbitrator/arbutil/src/hostios.rs delete mode 100644 arbitrator/prover/build.rs create mode 100644 arbitrator/prover/test-cases/forward-test.wat create mode 100644 arbitrator/prover/test-cases/forward/forward.wat create mode 100644 arbitrator/prover/test-cases/forward/target.wat create mode 100644 arbitrator/wasm-libraries/forward/.gitignore create mode 100644 arbitrator/wasm-libraries/forward/Cargo.toml create mode 100644 arbitrator/wasm-libraries/forward/src/main.rs diff --git a/.golangci.yml b/.golangci.yml index 0594670137..8b10d7116c 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -44,6 +44,7 @@ linters-settings: gosec: excludes: - G404 # checks that random numbers are securely generated + - G115 govet: enable-all: true diff --git a/Makefile b/Makefile index b736b04906..0a71d64f12 100644 --- a/Makefile +++ b/Makefile @@ -60,7 +60,7 @@ replay_wasm=$(output_latest)/replay.wasm arb_brotli_files = $(wildcard arbitrator/brotli/src/*.* arbitrator/brotli/src/*/*.* arbitrator/brotli/*.toml arbitrator/brotli/*.rs) .make/cbrotli-lib .make/cbrotli-wasm arbitrator_generated_header=$(output_root)/include/arbitrator.h -arbitrator_wasm_libs=$(patsubst %, $(output_root)/machines/latest/%.wasm, wasi_stub host_io soft-float arbcompress user_host program_exec) +arbitrator_wasm_libs=$(patsubst %, $(output_root)/machines/latest/%.wasm, forward wasi_stub host_io soft-float arbcompress user_host program_exec) arbitrator_stylus_lib=$(output_root)/lib/libstylus.a prover_bin=$(output_root)/bin/prover arbitrator_jit=$(output_root)/bin/jit @@ -75,12 +75,16 @@ arbitrator_test_wasms=$(patsubst %.wat,%.wasm, $(arbitrator_tests_wat)) $(patsub arbitrator_tests_link_info = $(shell cat $(arbitrator_cases)/link.txt | xargs) arbitrator_tests_link_deps = $(patsubst %,$(arbitrator_cases)/%.wasm, $(arbitrator_tests_link_info)) +arbitrator_tests_forward_wats = $(wildcard $(arbitrator_cases)/forward/*.wat) +arbitrator_tests_forward_deps = $(arbitrator_tests_forward_wats:wat=wasm) + WASI_SYSROOT?=/opt/wasi-sdk/wasi-sysroot arbitrator_wasm_lib_flags=$(patsubst %, -l %, $(arbitrator_wasm_libs)) rust_arbutil_files = $(wildcard arbitrator/arbutil/src/*.* arbitrator/arbutil/src/*/*.* arbitrator/arbutil/*.toml arbitrator/caller-env/src/*.* arbitrator/caller-env/src/*/*.* arbitrator/caller-env/*.toml) .make/cbrotli-lib +prover_direct_includes = $(patsubst %,$(output_latest)/%.wasm, forward forward_stub) prover_dir = arbitrator/prover/ rust_prover_files = $(wildcard $(prover_dir)/src/*.* $(prover_dir)/src/*/*.* $(prover_dir)/*.toml $(prover_dir)/*.rs) $(rust_arbutil_files) $(prover_direct_includes) $(arb_brotli_files) @@ -88,9 +92,12 @@ wasm_lib = arbitrator/wasm-libraries wasm_lib_cargo = $(wasm_lib)/.cargo/config.toml wasm_lib_deps = $(wildcard $(wasm_lib)/$(1)/*.toml $(wasm_lib)/$(1)/src/*.rs $(wasm_lib)/$(1)/*.rs) $(wasm_lib_cargo) $(rust_arbutil_files) $(arb_brotli_files) .make/machines wasm_lib_go_abi = $(call wasm_lib_deps,go-abi) +wasm_lib_forward = $(call wasm_lib_deps,forward) wasm_lib_user_host_trait = $(call wasm_lib_deps,user-host-trait) wasm_lib_user_host = $(call wasm_lib_deps,user-host) $(wasm_lib_user_host_trait) +forward_dir = $(wasm_lib)/forward + stylus_files = $(wildcard $(stylus_dir)/*.toml $(stylus_dir)/src/*.rs) $(wasm_lib_user_host_trait) $(rust_prover_files) jit_dir = arbitrator/jit @@ -274,6 +281,7 @@ clean: rm -f arbitrator/wasm-libraries/soft-float/*.o rm -f arbitrator/wasm-libraries/soft-float/SoftFloat/build/Wasm-Clang/*.o rm -f arbitrator/wasm-libraries/soft-float/SoftFloat/build/Wasm-Clang/*.a + rm -f arbitrator/wasm-libraries/forward/*.wat rm -rf arbitrator/stylus/tests/*/target/ arbitrator/stylus/tests/*/*.wasm @rm -rf contracts/build contracts/cache solgen/go/ @rm -f .make/* @@ -391,7 +399,7 @@ $(output_latest)/host_io.wasm: $(DEP_PREDICATE) $(call wasm_lib_deps,host-io) $( cargo build --manifest-path arbitrator/wasm-libraries/Cargo.toml --release --target wasm32-wasi --config $(wasm_lib_cargo) --package host-io install arbitrator/wasm-libraries/$(wasm32_wasi)/host_io.wasm $@ -$(output_latest)/user_host.wasm: $(DEP_PREDICATE) $(wasm_lib_user_host) $(rust_prover_files) .make/machines +$(output_latest)/user_host.wasm: $(DEP_PREDICATE) $(wasm_lib_user_host) $(rust_prover_files) $(output_latest)/forward_stub.wasm .make/machines cargo build --manifest-path arbitrator/wasm-libraries/Cargo.toml --release --target wasm32-wasi --config $(wasm_lib_cargo) --package user-host install arbitrator/wasm-libraries/$(wasm32_wasi)/user_host.wasm $@ @@ -407,9 +415,17 @@ $(output_latest)/arbcompress.wasm: $(DEP_PREDICATE) $(call wasm_lib_deps,brotli) cargo build --manifest-path arbitrator/wasm-libraries/Cargo.toml --release --target wasm32-wasi --config $(wasm_lib_cargo) --package arbcompress install arbitrator/wasm-libraries/$(wasm32_wasi)/arbcompress.wasm $@ +$(output_latest)/forward.wasm: $(DEP_PREDICATE) $(wasm_lib_forward) .make/machines + cargo run --manifest-path $(forward_dir)/Cargo.toml -- --path $(forward_dir)/forward.wat + wat2wasm $(wasm_lib)/forward/forward.wat -o $@ + +$(output_latest)/forward_stub.wasm: $(DEP_PREDICATE) $(wasm_lib_forward) .make/machines + cargo run --manifest-path $(forward_dir)/Cargo.toml -- --path $(forward_dir)/forward_stub.wat --stub + wat2wasm $(wasm_lib)/forward/forward_stub.wat -o $@ + $(output_latest)/machine.wavm.br: $(DEP_PREDICATE) $(prover_bin) $(arbitrator_wasm_libs) $(replay_wasm) - $(prover_bin) $(replay_wasm) --generate-binaries $(output_latest) --with-forwarder \ - $(patsubst %,-l $(output_latest)/%.wasm, soft-float wasi_stub host_io user_host arbcompress program_exec) + $(prover_bin) $(replay_wasm) --generate-binaries $(output_latest) \ + $(patsubst %,-l $(output_latest)/%.wasm, forward soft-float wasi_stub host_io user_host arbcompress program_exec) $(arbitrator_cases)/%.wasm: $(arbitrator_cases)/%.wat wat2wasm $< -o $@ @@ -476,10 +492,10 @@ target/testdata/preimages.bin: python3 scripts/create-test-preimages.py $@ contracts/test/prover/proofs/rust-%.json: $(arbitrator_cases)/rust/$(wasm32_wasi)/%.wasm $(prover_bin) $(arbitrator_wasm_libs) target/testdata/preimages.bin - $(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --allow-hostapi --require-success --inbox-add-stub-headers --inbox $(arbitrator_cases)/rust/data/msg0.bin --inbox $(arbitrator_cases)/rust/data/msg1.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg0.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg1.bin --preimages target/testdata/preimages.bin --with-forwarder + $(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --allow-hostapi --require-success --inbox-add-stub-headers --inbox $(arbitrator_cases)/rust/data/msg0.bin --inbox $(arbitrator_cases)/rust/data/msg1.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg0.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg1.bin --preimages target/testdata/preimages.bin contracts/test/prover/proofs/go.json: $(arbitrator_cases)/go/testcase.wasm $(prover_bin) $(arbitrator_wasm_libs) target/testdata/preimages.bin $(arbitrator_tests_link_deps) $(arbitrator_cases)/user.wasm - $(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --require-success --preimages target/testdata/preimages.bin --stylus-modules $(arbitrator_cases)/user.wasm --with-forwarder + $(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --require-success --preimages target/testdata/preimages.bin --stylus-modules $(arbitrator_cases)/user.wasm # avoid testing user.wasm in onestepproofs. It can only run as stylus program. contracts/test/prover/proofs/user.json: @@ -492,6 +508,9 @@ contracts/test/prover/proofs/read-inboxmsg-10.json: contracts/test/prover/proofs/global-state.json: echo "[]" > $@ +contracts/test/prover/proofs/forward-test.json: $(arbitrator_cases)/forward-test.wasm $(arbitrator_tests_forward_deps) $(prover_bin) + $(prover_bin) $< -o $@ --allow-hostapi $(patsubst %,-l %, $(arbitrator_tests_forward_deps)) + contracts/test/prover/proofs/link.json: $(arbitrator_cases)/link.wasm $(arbitrator_tests_link_deps) $(prover_bin) $(prover_bin) $< -o $@ --allow-hostapi --stylus-modules $(arbitrator_tests_link_deps) --require-success diff --git a/arbitrator/arbutil/src/evm/api.rs b/arbitrator/arbutil/src/evm/api.rs index 093e7f2984..9d4c78c0de 100644 --- a/arbitrator/arbutil/src/evm/api.rs +++ b/arbitrator/arbutil/src/evm/api.rs @@ -77,7 +77,7 @@ pub trait EvmApi: Send + 'static { /// Reads the 32-byte value in the EVM state trie at offset `key`. /// Returns the value and the access cost in gas. /// Analogous to `vm.SLOAD`. - fn get_bytes32(&mut self, key: Bytes32) -> (Bytes32, u64); + fn get_bytes32(&mut self, key: Bytes32, evm_api_gas_to_use: u64) -> (Bytes32, u64); /// Stores the given value at the given key in Stylus VM's cache of the EVM state trie. /// Note that the actual values only get written after calls to `set_trie_slots`. diff --git a/arbitrator/arbutil/src/evm/mod.rs b/arbitrator/arbutil/src/evm/mod.rs index 1671e67072..36dadd906a 100644 --- a/arbitrator/arbutil/src/evm/mod.rs +++ b/arbitrator/arbutil/src/evm/mod.rs @@ -74,9 +74,12 @@ pub const GASPRICE_GAS: u64 = GAS_QUICK_STEP; // vm.GasQuickStep (see jump_table.go) pub const ORIGIN_GAS: u64 = GAS_QUICK_STEP; +pub const ARBOS_VERSION_STYLUS_CHARGING_FIXES: u64 = 32; + #[derive(Clone, Copy, Debug, Default)] #[repr(C)] pub struct EvmData { + pub arbos_version: u64, pub block_basefee: Bytes32, pub chainid: u64, pub block_coinbase: Bytes20, diff --git a/arbitrator/arbutil/src/evm/req.rs b/arbitrator/arbutil/src/evm/req.rs index 287db357f3..1cfceda6b7 100644 --- a/arbitrator/arbutil/src/evm/req.rs +++ b/arbitrator/arbutil/src/evm/req.rs @@ -7,7 +7,6 @@ use crate::{ storage::{StorageCache, StorageWord}, user::UserOutcomeKind, }, - pricing::EVM_API_INK, Bytes20, Bytes32, }; use eyre::{bail, eyre, Result}; @@ -99,13 +98,13 @@ impl> EvmApiRequestor { } impl> EvmApi for EvmApiRequestor { - fn get_bytes32(&mut self, key: Bytes32) -> (Bytes32, u64) { + fn get_bytes32(&mut self, key: Bytes32, evm_api_gas_to_use: u64) -> (Bytes32, u64) { let cache = &mut self.storage_cache; let mut cost = cache.read_gas(); let value = cache.entry(key).or_insert_with(|| { let (res, _, gas) = self.handler.request(EvmApiMethod::GetBytes32, key); - cost = cost.saturating_add(gas).saturating_add(EVM_API_INK); + cost = cost.saturating_add(gas).saturating_add(evm_api_gas_to_use); StorageWord::known(res.try_into().unwrap()) }); (value.value, cost) diff --git a/arbitrator/arbutil/src/hostios.rs b/arbitrator/arbutil/src/hostios.rs deleted file mode 100644 index 05bcb94f73..0000000000 --- a/arbitrator/arbutil/src/hostios.rs +++ /dev/null @@ -1,64 +0,0 @@ -use std::fmt::Display; - -pub enum ParamType { - I32, - I64, -} - -impl Display for ParamType { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - use ParamType::*; - match self { - I32 => write!(f, "i32"), - I64 => write!(f, "i64"), - } - } -} - -use ParamType::*; - -/// order matters! -pub const HOSTIOS: [(&str, &[ParamType], &[ParamType]); 42] = [ - ("read_args", &[I32], &[]), - ("write_result", &[I32, I32], &[]), - ("exit_early", &[I32], &[]), - ("storage_load_bytes32", &[I32, I32], &[]), - ("storage_cache_bytes32", &[I32, I32], &[]), - ("storage_flush_cache", &[I32], &[]), - ("transient_load_bytes32", &[I32, I32], &[]), - ("transient_store_bytes32", &[I32, I32], &[]), - ("call_contract", &[I32, I32, I32, I32, I64, I32], &[I32]), - ("delegate_call_contract", &[I32, I32, I32, I64, I32], &[I32]), - ("static_call_contract", &[I32, I32, I32, I64, I32], &[I32]), - ("create1", &[I32, I32, I32, I32, I32], &[]), - ("create2", &[I32, I32, I32, I32, I32, I32], &[]), - ("read_return_data", &[I32, I32, I32], &[I32]), - ("return_data_size", &[], &[I32]), - ("emit_log", &[I32, I32, I32], &[]), - ("account_balance", &[I32, I32], &[]), - ("account_code", &[I32, I32, I32, I32], &[I32]), - ("account_code_size", &[I32], &[I32]), - ("account_codehash", &[I32, I32], &[]), - ("evm_gas_left", &[], &[I64]), - ("evm_ink_left", &[], &[I64]), - ("block_basefee", &[I32], &[]), - ("chainid", &[], &[I64]), - ("block_coinbase", &[I32], &[]), - ("block_gas_limit", &[], &[I64]), - ("block_number", &[], &[I64]), - ("block_timestamp", &[], &[I64]), - ("contract_address", &[I32], &[]), - ("math_div", &[I32, I32], &[]), - ("math_mod", &[I32, I32], &[]), - ("math_pow", &[I32, I32], &[]), - ("math_add_mod", &[I32, I32, I32], &[]), - ("math_mul_mod", &[I32, I32, I32], &[]), - ("msg_reentrant", &[], &[I32]), - ("msg_sender", &[I32], &[]), - ("msg_value", &[I32], &[]), - ("native_keccak256", &[I32, I32, I32], &[]), - ("tx_gas_price", &[I32], &[]), - ("tx_ink_price", &[], &[I32]), - ("tx_origin", &[I32], &[]), - ("pay_for_memory_grow", &[I32], &[]), -]; diff --git a/arbitrator/arbutil/src/lib.rs b/arbitrator/arbutil/src/lib.rs index 8d8c1d0fca..9c48a9fefc 100644 --- a/arbitrator/arbutil/src/lib.rs +++ b/arbitrator/arbutil/src/lib.rs @@ -6,7 +6,6 @@ pub mod color; pub mod crypto; pub mod evm; pub mod format; -pub mod hostios; pub mod math; pub mod operator; pub mod pricing; diff --git a/arbitrator/jit/src/machine.rs b/arbitrator/jit/src/machine.rs index 2a3c5c5616..02523f740a 100644 --- a/arbitrator/jit/src/machine.rs +++ b/arbitrator/jit/src/machine.rs @@ -129,7 +129,9 @@ pub fn create(opts: &Opts, env: WasmEnv) -> (Instance, FunctionEnv, Sto "send_response" => func!(program::send_response), "create_stylus_config" => func!(program::create_stylus_config), "create_evm_data" => func!(program::create_evm_data), + "create_evm_data_v2" => func!(program::create_evm_data_v2), "activate" => func!(program::activate), + "activate_v2" => func!(program::activate_v2), }, }; diff --git a/arbitrator/jit/src/program.rs b/arbitrator/jit/src/program.rs index c608a3cf85..084afe96bc 100644 --- a/arbitrator/jit/src/program.rs +++ b/arbitrator/jit/src/program.rs @@ -16,8 +16,45 @@ use prover::{ programs::{config::PricingParams, prelude::*}, }; -/// activates a user program +const DEFAULT_STYLUS_ARBOS_VERSION: u64 = 31; + pub fn activate( + env: WasmEnvMut, + wasm_ptr: GuestPtr, + wasm_size: u32, + pages_ptr: GuestPtr, + asm_estimate_ptr: GuestPtr, + init_cost_ptr: GuestPtr, + cached_init_cost_ptr: GuestPtr, + stylus_version: u16, + debug: u32, + codehash: GuestPtr, + module_hash_ptr: GuestPtr, + gas_ptr: GuestPtr, + err_buf: GuestPtr, + err_buf_len: u32, +) -> Result { + activate_v2( + env, + wasm_ptr, + wasm_size, + pages_ptr, + asm_estimate_ptr, + init_cost_ptr, + cached_init_cost_ptr, + stylus_version, + DEFAULT_STYLUS_ARBOS_VERSION, + debug, + codehash, + module_hash_ptr, + gas_ptr, + err_buf, + err_buf_len, + ) +} + +/// activates a user program +pub fn activate_v2( mut env: WasmEnvMut, wasm_ptr: GuestPtr, wasm_size: u32, @@ -25,7 +62,8 @@ pub fn activate( asm_estimate_ptr: GuestPtr, init_cost_ptr: GuestPtr, cached_init_cost_ptr: GuestPtr, - version: u16, + stylus_version: u16, + arbos_version_for_gas: u64, debug: u32, codehash: GuestPtr, module_hash_ptr: GuestPtr, @@ -40,7 +78,15 @@ pub fn activate( let page_limit = mem.read_u16(pages_ptr); let gas_left = &mut mem.read_u64(gas_ptr); - match Module::activate(&wasm, codehash, version, page_limit, debug, gas_left) { + match Module::activate( + &wasm, + codehash, + stylus_version, + arbos_version_for_gas, + page_limit, + debug, + gas_left, + ) { Ok((module, data)) => { mem.write_u64(gas_ptr, *gas_left); mem.write_u16(pages_ptr, data.footprint); @@ -222,9 +268,47 @@ pub fn create_stylus_config( Ok(res as u64) } -/// Creates an `EvmData` handler from its component parts. pub fn create_evm_data( + env: WasmEnvMut, + block_basefee_ptr: GuestPtr, + chainid: u64, + block_coinbase_ptr: GuestPtr, + block_gas_limit: u64, + block_number: u64, + block_timestamp: u64, + contract_address_ptr: GuestPtr, + module_hash_ptr: GuestPtr, + msg_sender_ptr: GuestPtr, + msg_value_ptr: GuestPtr, + tx_gas_price_ptr: GuestPtr, + tx_origin_ptr: GuestPtr, + cached: u32, + reentrant: u32, +) -> Result { + create_evm_data_v2( + env, + DEFAULT_STYLUS_ARBOS_VERSION, + block_basefee_ptr, + chainid, + block_coinbase_ptr, + block_gas_limit, + block_number, + block_timestamp, + contract_address_ptr, + module_hash_ptr, + msg_sender_ptr, + msg_value_ptr, + tx_gas_price_ptr, + tx_origin_ptr, + cached, + reentrant, + ) +} + +/// Creates an `EvmData` handler from its component parts. +pub fn create_evm_data_v2( mut env: WasmEnvMut, + arbos_version: u64, block_basefee_ptr: GuestPtr, chainid: u64, block_coinbase_ptr: GuestPtr, @@ -243,6 +327,7 @@ pub fn create_evm_data( let (mut mem, _) = env.jit_env(); let evm_data = EvmData { + arbos_version, block_basefee: mem.read_bytes32(block_basefee_ptr), cached: cached != 0, chainid, diff --git a/arbitrator/prover/Cargo.toml b/arbitrator/prover/Cargo.toml index a634dba19e..5475647765 100644 --- a/arbitrator/prover/Cargo.toml +++ b/arbitrator/prover/Cargo.toml @@ -47,10 +47,6 @@ enum-iterator = "2.0.1" criterion = { version = "0.5.0", features = ["html_reports"] } rand = "0.8.4" -[build-dependencies] -wasmer = { path = "../tools/wasmer/lib/api" } -arbutil = { path = "../arbutil/" } - [[bench]] name = "merkle_bench" harness = false diff --git a/arbitrator/prover/build.rs b/arbitrator/prover/build.rs deleted file mode 100644 index db0560b2c5..0000000000 --- a/arbitrator/prover/build.rs +++ /dev/null @@ -1,89 +0,0 @@ -use arbutil::hostios::HOSTIOS; -use std::{env, fmt::Write, fs, path::Path}; - -pub fn gen_forwarder(out_path: &Path) { - let mut wat = String::new(); - macro_rules! wln { - ($($text:tt)*) => { - writeln!(wat, $($text)*).unwrap(); - }; - } - let s = " "; - - wln!("(module"); - - macro_rules! group { - ($list:expr, $kind:expr) => { - (!$list.is_empty()) - .then(|| { - format!( - " ({} {})", - $kind, - $list - .iter() - .map(|x| x.to_string()) - .collect::>() - .join(" ") - ) - }) - .unwrap_or_default() - }; - } - - wln!("{s};; symbols to re-export"); - for (name, ins, outs) in HOSTIOS { - let params = group!(ins, "param"); - let result = group!(outs, "result"); - wln!(r#"{s}(import "user_host" "{name}" (func $_{name}{params}{result}))"#); - } - wln!(); - - wln!("{s};; reserved offsets for future user_host imports"); - for i in HOSTIOS.len()..512 { - wln!("{s}(func $reserved_{i} unreachable)"); - } - wln!(); - - wln!( - "{s};; allows user_host to request a trap\n\ - {s}(global $trap (mut i32) (i32.const 0))\n\ - {s}(func $check\n\ - {s}{s}global.get $trap ;; see if set\n\ - {s}{s}(global.set $trap (i32.const 0)) ;; reset the flag\n\ - {s}{s}(if (then (unreachable)))\n\ - {s})\n\ - {s}(func (export \"forward__set_trap\")\n\ - {s}{s}(global.set $trap (i32.const 1))\n\ - {s})\n" - ); - - wln!("{s};; user linkage"); - for (name, ins, outs) in HOSTIOS { - let params = group!(ins, "param"); - let result = group!(outs, "result"); - wln!("{s}(func (export \"vm_hooks__{name}\"){params}{result}"); - - for i in 0..ins.len() { - wln!("{s}{s}local.get {i}"); - } - - wln!( - "{s}{s}call $_{name}\n\ - {s}{s}call $check\n\ - {s})" - ); - } - - wln!(")"); - eprintln!("{}", &wat); - - let wasm = wasmer::wat2wasm(wat.as_bytes()).unwrap(); - - fs::write(out_path, wasm.as_ref()).unwrap(); -} - -fn main() { - let out_dir = env::var("OUT_DIR").unwrap(); - let forwarder_path = Path::new(&out_dir).join("forwarder.wasm"); - gen_forwarder(&forwarder_path); -} diff --git a/arbitrator/prover/src/binary.rs b/arbitrator/prover/src/binary.rs index b1d819deb2..2260f6bf48 100644 --- a/arbitrator/prover/src/binary.rs +++ b/arbitrator/prover/src/binary.rs @@ -9,7 +9,9 @@ use crate::{ }, value::{ArbValueType, FunctionType, IntegerValType, Value}, }; -use arbutil::{math::SaturatingSum, Bytes32, Color, DebugColor}; +use arbutil::{ + evm::ARBOS_VERSION_STYLUS_CHARGING_FIXES, math::SaturatingSum, Bytes32, Color, DebugColor, +}; use eyre::{bail, ensure, eyre, Result, WrapErr}; use fnv::{FnvHashMap as HashMap, FnvHashSet as HashSet}; use nom::{ @@ -471,14 +473,10 @@ pub fn parse<'a>(input: &'a [u8], path: &'_ Path) -> Result> { let export = export.rsplit("__").take(1); exports.extend(export); } - // forwarder is allowed to re-export the same name - // TODO: is there a real problem if import_name == export_name but module names don't? - if path != Path::new("forwarder") { - for import in &binary.imports { - let name = import.name; - if exports.contains(name) { - bail!("binary exports an import with the same name {}", name.red()); - } + for import in &binary.imports { + let name = import.name; + if exports.contains(name) { + bail!("binary exports an import with the same name {}", name.red()); } } @@ -645,6 +643,7 @@ impl<'a> WasmBinary<'a> { /// Parses and instruments a user wasm pub fn parse_user( wasm: &'a [u8], + arbos_version_for_gas: u64, page_limit: u16, compile: &CompileConfig, codehash: &Bytes32, @@ -682,6 +681,10 @@ impl<'a> WasmBinary<'a> { limit!(65536, code.expr.len(), "opcodes in func body"); } + if arbos_version_for_gas >= ARBOS_VERSION_STYLUS_CHARGING_FIXES { + limit!(513, bin.imports.len(), "imports") + } + let table_entries = bin.tables.iter().map(|x| x.initial).saturating_sum(); limit!(4096, table_entries, "table entries"); diff --git a/arbitrator/prover/src/host.rs b/arbitrator/prover/src/host.rs index dc0f0b74b2..1d0fe658ec 100644 --- a/arbitrator/prover/src/host.rs +++ b/arbitrator/prover/src/host.rs @@ -405,7 +405,7 @@ impl Hostio { } } -pub fn get_impl(module: &str, name: &str) -> Result { +pub fn get_impl(module: &str, name: &str) -> Result<(Function, bool)> { let hostio: Hostio = format!("{module}__{name}").parse()?; let append = |code: &mut Vec| { @@ -414,11 +414,8 @@ pub fn get_impl(module: &str, name: &str) -> Result { Ok(()) }; - Function::new(&[], append, hostio.ty(), &[]) -} - -pub fn hostio_module_is_debug(module: &str) -> bool { - module == "console" || module == "debug" + let debug = module == "console" || module == "debug"; + Function::new(&[], append, hostio.ty(), &[]).map(|x| (x, debug)) } /// Adds internal functions to a module. diff --git a/arbitrator/prover/src/lib.rs b/arbitrator/prover/src/lib.rs index e45d12c492..0f537478eb 100644 --- a/arbitrator/prover/src/lib.rs +++ b/arbitrator/prover/src/lib.rs @@ -70,16 +70,10 @@ pub unsafe extern "C" fn arbitrator_load_machine( library_paths: *const *const c_char, library_paths_size: isize, debug_chain: usize, - with_forwarder: bool, ) -> *mut Machine { let debug_chain = debug_chain != 0; - match arbitrator_load_machine_impl( - binary_path, - library_paths, - library_paths_size, - debug_chain, - with_forwarder, - ) { + match arbitrator_load_machine_impl(binary_path, library_paths, library_paths_size, debug_chain) + { Ok(mach) => mach, Err(err) => { eprintln!("Error loading binary: {:?}", err); @@ -93,7 +87,6 @@ unsafe fn arbitrator_load_machine_impl( library_paths: *const *const c_char, library_paths_size: isize, debug_chain: bool, - with_forwarder: bool, ) -> Result<*mut Machine> { let binary_path = cstr_to_string(binary_path); let binary_path = Path::new(&binary_path); @@ -114,7 +107,6 @@ unsafe fn arbitrator_load_machine_impl( Default::default(), Default::default(), get_empty_preimage_resolver(), - with_forwarder, )?; Ok(Box::into_raw(Box::new(mach))) } diff --git a/arbitrator/prover/src/machine.rs b/arbitrator/prover/src/machine.rs index 2418d3cc2e..4ece1f7bf2 100644 --- a/arbitrator/prover/src/machine.rs +++ b/arbitrator/prover/src/machine.rs @@ -19,7 +19,7 @@ use crate::{ IBinOpType, IRelOpType, IUnOpType, Instruction, Opcode, }, }; -use arbutil::{crypto, hostios::HOSTIOS, math, Bytes32, Color, DebugColor, PreimageType}; +use arbutil::{crypto, math, Bytes32, Color, DebugColor, PreimageType}; use brotli::Dictionary; #[cfg(feature = "native")] use c_kzg::BYTES_PER_BLOB; @@ -330,115 +330,36 @@ pub struct Module { pub(crate) extra_hash: Arc, } -#[cfg(feature = "native")] -static FORWARDER_WASM: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/forwarder.wasm")); - lazy_static! { static ref USER_IMPORTS: HashMap = { let mut imports = HashMap::default(); - // 0-513 are internal - for (index, (name, ins, outs)) in HOSTIOS.iter().enumerate() { - let import = AvailableImport::new(FunctionType::new(ins.iter().map(|x|x.into()).collect::>(), outs.iter().map(|x|x.into()).collect::>()), 1, (index + 514).try_into().unwrap()); - imports.insert(format!("vm_hooks__{name}"), import); - } - imports - }; -} + let forward = include_bytes!("../../../target/machines/latest/forward_stub.wasm"); + let forward = binary::parse(forward, Path::new("forward")).unwrap(); -trait ImportResolver: for<'a, 'b> Fn(&'a str, &'b str) -> Result + Sized { - fn compose(self, other: impl ImportResolver) -> impl ImportResolver { - move |module, name| { - self(module, name).or_else(|original_err| { - let mut result = other(module, name); - if result.is_err() { - for err_layer in original_err.chain().rev() { - result = result.wrap_err(format!("{err_layer}")); - } - } - result - }) - } - } - - fn condition(self, condition: bool) -> impl ImportResolver { - move |module, name| { - if condition { - self(module, name) - } else { - bail!("resolver disabled") + for (name, &(export, kind)) in &forward.exports { + if kind == ExportKind::Func { + let ty = match forward.get_function(FunctionIndex::from_u32(export)) { + Ok(ty) => ty, + Err(error) => panic!("failed to read export {name}: {error:?}"), + }; + let import = AvailableImport::new(ty, 1, export); + imports.insert(name.to_owned(), import); } } - } + imports + }; } -impl Fn(&'a str, &'b str) -> Result> ImportResolver for F {} - impl Module { - fn make_imports_resolver( - import_map: &HashMap, - ) -> impl ImportResolver + '_ { - move |module, name| { - let qualified_name = format!("{module}__{name}"); - - let Some(import) = import_map.get(&qualified_name) else { - bail!("func not found {module} {name}") - }; - let wavm = vec![ - Instruction::simple(Opcode::InitFrame), - Instruction::with_data( - Opcode::CrossModuleCall, - pack_cross_module_call(import.module, import.func), - ), - Instruction::simple(Opcode::Return), - ]; - Ok(Function::new_from_wavm(wavm, import.ty.clone(), vec![])) - } - } - - fn make_forward_resolver( - import_map: &HashMap, - ) -> impl ImportResolver + '_ { - move |module, name| { - let qualified_name = format!("{module}__{name}"); - - let Some(import) = import_map.get(&qualified_name) else { - bail!("func not found {module} {name}") - }; - let wavm = vec![ - Instruction::simple(Opcode::InitFrame), - Instruction::with_data( - Opcode::CrossModuleForward, - pack_cross_module_call(import.module, import.func), - ), - Instruction::simple(Opcode::Return), - ]; - Ok(Function::new_from_wavm(wavm, import.ty.clone(), vec![])) - } - } + const FORWARDING_PREFIX: &'static str = "arbitrator_forward__"; - fn hostio_resolver(module: &str, name: &str) -> Result { - if host::hostio_module_is_debug(module) { - return Self::notfound_resolver(module, name); - } - host::get_impl(module, name) - } - - fn debug_resolver(module: &str, name: &str) -> Result { - if !host::hostio_module_is_debug(module) { - return Self::notfound_resolver(module, name); - } - host::get_impl(module, name) - } - - fn notfound_resolver(module: &str, name: &str) -> Result { - bail!("import not found {module} {name}") - } - - fn from_binary( + fn from_binary( bin: &WasmBinary, - import_resolver: R, + available_imports: &HashMap, floating_point_impls: &FloatingPointImpls, + allow_hostapi: bool, + debug_funcs: bool, stylus_data: Option, ) -> Result { let mut code = Vec::new(); @@ -450,18 +371,56 @@ impl Module { for import in &bin.imports { let module = import.module; let have_ty = &bin.types[import.offset as usize]; + // allow_hostapi is only set for system modules like the + // forwarder. We restrict stripping the prefix for user modules. + let (forward, import_name) = + if allow_hostapi && import.name.starts_with(Self::FORWARDING_PREFIX) { + (true, &import.name[Self::FORWARDING_PREFIX.len()..]) + } else { + (false, import.name) + }; - let func = import_resolver(import.module, import.name)?; + let qualified_name = format!("{module}__{import_name}"); + let func = if let Some(import) = available_imports.get(&qualified_name) { + let call = match forward { + true => Opcode::CrossModuleForward, + false => Opcode::CrossModuleCall, + }; + let wavm = vec![ + Instruction::simple(Opcode::InitFrame), + Instruction::with_data( + call, + pack_cross_module_call(import.module, import.func), + ), + Instruction::simple(Opcode::Return), + ]; + Function::new_from_wavm(wavm, import.ty.clone(), vec![]) + } else if let Ok((hostio, debug)) = host::get_impl(import.module, import_name) { + ensure!( + (debug && debug_funcs) || (!debug && allow_hostapi), + "Host func {} in {} not enabled debug_funcs={debug_funcs} hostapi={allow_hostapi} debug={debug}", + import_name.red(), + import.module.red(), + ); + hostio + } else { + bail!( + "No such import {} in {} for {}", + import_name.red(), + import.module.red(), + bin_name.red() + ) + }; ensure!( &func.ty == have_ty, "Import {} for {} has different function signature than export.\nexpected {} in {}\nbut have {}", - import.name.red(), bin_name.red(), func.ty.red(), module.red(), have_ty.red(), + import_name.red(), bin_name.red(), func.ty.red(), module.red(), have_ty.red(), ); func_type_idxs.push(import.offset); code.push(func); - host_call_hooks.push(Some((import.module.into(), import.name.into()))); + host_call_hooks.push(Some((import.module.into(), import_name.into()))); } func_type_idxs.extend(bin.functions.iter()); @@ -657,9 +616,10 @@ impl Module { ) -> Result { Self::from_binary( bin, - Module::make_imports_resolver(&USER_IMPORTS) - .compose(Module::debug_resolver.condition(debug_funcs)), + &USER_IMPORTS, &HashMap::default(), + false, + debug_funcs, stylus_data, ) } @@ -1270,7 +1230,6 @@ impl Machine { global_state: GlobalState, inbox_contents: HashMap<(InboxIdentifier, u64), Vec>, preimage_resolver: PreimageResolver, - with_forwarder: bool, ) -> Result { let bin_source = file_bytes(binary_path)?; let bin = parse(&bin_source, binary_path) @@ -1296,7 +1255,6 @@ impl Machine { inbox_contents, preimage_resolver, None, - with_forwarder, ) } @@ -1326,7 +1284,6 @@ impl Machine { HashMap::default(), Arc::new(|_, _, _| panic!("tried to read preimage")), Some(stylus_data), - false, )?; let footprint: u32 = stylus_data.footprint.into(); @@ -1374,7 +1331,6 @@ impl Machine { inbox_contents: HashMap<(InboxIdentifier, u64), Vec>, preimage_resolver: PreimageResolver, stylus_data: Option, - with_forwarder: bool, ) -> Result { use ArbValueType::*; @@ -1382,19 +1338,25 @@ impl Machine { let mut modules = vec![Module::default()]; let mut available_imports = HashMap::default(); let mut floating_point_impls = HashMap::default(); - - let forwarder = if with_forwarder { - #[cfg(not(feature = "native"))] - bail!("forwarder not supported without native"); - - #[cfg(feature = "native")] - Some(parse(FORWARDER_WASM, Path::new("forwarder"))?) - } else { - None - }; + let main_module_index = u32::try_from(modules.len() + libraries.len())?; + + // make the main module's exports available to libraries + for (name, &(export, kind)) in &bin.exports { + if kind == ExportKind::Func { + let index: usize = export.try_into()?; + if let Some(index) = index.checked_sub(bin.imports.len()) { + let ty: usize = bin.functions[index].try_into()?; + let ty = bin.types[ty].clone(); + available_imports.insert( + format!("env__wavm_guest_call__{name}"), + AvailableImport::new(ty, main_module_index, export), + ); + } + } + } // collect all the library exports in advance so they can use each other's - for (index, lib) in forwarder.iter().chain(libraries.iter()).enumerate() { + for (index, lib) in libraries.iter().enumerate() { let module = 1 + index as u32; // off by one due to the entry point for (name, &(export, kind)) in &lib.exports { if kind == ExportKind::Func { @@ -1402,57 +1364,54 @@ impl Machine { Ok(ty) => ty, Err(error) => bail!("failed to read export {name}: {error}"), }; - let import = AvailableImport::new(ty.clone(), module, export); + let import = AvailableImport::new(ty, module, export); available_imports.insert(name.to_owned(), import); - if let Ok(op) = name.parse::() { - let mut sig = op.signature(); - // wavm codegen takes care of effecting this type change at callsites - for ty in sig.inputs.iter_mut().chain(sig.outputs.iter_mut()) { - if *ty == F32 { - *ty = I32; - } else if *ty == F64 { - *ty = I64; - } - } - ensure!( - ty == sig, - "Wrong type for floating point impl {} expecting {} but got {}", - name.red(), - sig.red(), - ty.red() - ); - floating_point_impls.insert(op, (module, export)); - } } } } - if let Some(lib) = forwarder { - modules.push(Module::from_binary( - &lib, - Module::make_forward_resolver(&available_imports), - &floating_point_impls, - None, - )?); - } - - for lib in libraries.iter() { - modules.push(Module::from_binary( + for lib in libraries { + let module = Module::from_binary( lib, - Module::hostio_resolver - .compose(Module::debug_resolver.condition(debug_funcs)) - .compose(Module::make_imports_resolver(&available_imports)), + &available_imports, &floating_point_impls, + true, + debug_funcs, None, - )?); + )?; + for (name, &func) in &*module.func_exports { + let ty = module.func_types[func as usize].clone(); + if let Ok(op) = name.parse::() { + let mut sig = op.signature(); + // wavm codegen takes care of effecting this type change at callsites + for ty in sig.inputs.iter_mut().chain(sig.outputs.iter_mut()) { + if *ty == F32 { + *ty = I32; + } else if *ty == F64 { + *ty = I64; + } + } + ensure!( + ty == sig, + "Wrong type for floating point impl {} expecting {} but got {}", + name.red(), + sig.red(), + ty.red() + ); + floating_point_impls.insert(op, (modules.len() as u32, func)); + } + } + modules.push(module); } + // Shouldn't be necessary, but to be safe, don't allow the main binary to import its own guest calls + available_imports.retain(|_, i| i.module as usize != modules.len()); modules.push(Module::from_binary( &bin, - Module::make_imports_resolver(&available_imports) - .compose(Module::hostio_resolver.condition(allow_hostapi_from_main)) - .compose(Module::debug_resolver.condition(debug_funcs)), + &available_imports, &floating_point_impls, + allow_hostapi_from_main, + debug_funcs, stylus_data, )?); diff --git a/arbitrator/prover/src/main.rs b/arbitrator/prover/src/main.rs index 442b35992d..dba32e0e72 100644 --- a/arbitrator/prover/src/main.rs +++ b/arbitrator/prover/src/main.rs @@ -86,8 +86,6 @@ struct Opts { skip_until_host_io: bool, #[structopt(long)] max_steps: Option, - #[structopt(short, long)] - with_forwarder: bool, } fn file_with_stub_header(path: &Path, headerlength: usize) -> Result> { @@ -213,7 +211,6 @@ fn main() -> Result<()> { global_state, inbox_contents, preimage_resolver, - opts.with_forwarder, )?; for path in &opts.stylus_modules { diff --git a/arbitrator/prover/src/programs/mod.rs b/arbitrator/prover/src/programs/mod.rs index a5df2e31a8..a35308e7ff 100644 --- a/arbitrator/prover/src/programs/mod.rs +++ b/arbitrator/prover/src/programs/mod.rs @@ -8,7 +8,7 @@ use crate::{ programs::config::CompileConfig, value::{FunctionType as ArbFunctionType, Value}, }; -use arbutil::{math::SaturatingSum, Bytes32, Color}; +use arbutil::{evm::ARBOS_VERSION_STYLUS_CHARGING_FIXES, math::SaturatingSum, Bytes32, Color}; use eyre::{bail, eyre, Report, Result, WrapErr}; use fnv::FnvHashMap as HashMap; use std::fmt::Debug; @@ -418,58 +418,64 @@ impl Module { pub fn activate( wasm: &[u8], codehash: &Bytes32, - version: u16, + stylus_version: u16, + arbos_version_for_gas: u64, // must only be used for activation gas page_limit: u16, debug: bool, gas: &mut u64, ) -> Result<(Self, StylusData)> { - // converts a number of microseconds to gas - // TODO: collapse to a single value after finalizing factors - let us_to_gas = |us: u64| { - let fudge = 2; - let sync_rate = 1_000_000 / 2; - let speed = 7_000_000; - us.saturating_mul(fudge * speed) / sync_rate - }; - - macro_rules! pay { - ($us:expr) => { - let amount = us_to_gas($us); - if *gas < amount { - *gas = 0; - bail!("out of gas"); - } - *gas -= amount; + let compile = CompileConfig::version(stylus_version, debug); + let (bin, stylus_data) = + WasmBinary::parse_user(wasm, arbos_version_for_gas, page_limit, &compile, codehash) + .wrap_err("failed to parse wasm")?; + + if arbos_version_for_gas > 0 { + // converts a number of microseconds to gas + // TODO: collapse to a single value after finalizing factors + let us_to_gas = |us: u64| { + let fudge = 2; + let sync_rate = 1_000_000 / 2; + let speed = 7_000_000; + us.saturating_mul(fudge * speed) / sync_rate }; - } - - // pay for wasm - let wasm_len = wasm.len() as u64; - pay!(wasm_len.saturating_mul(31_733 / 100_000)); - - let compile = CompileConfig::version(version, debug); - let (bin, stylus_data) = WasmBinary::parse_user(wasm, page_limit, &compile, codehash) - .wrap_err("failed to parse wasm")?; - // pay for funcs - let funcs = bin.functions.len() as u64; - pay!(funcs.saturating_mul(17_263) / 100_000); - - // pay for data - let data = bin.datas.iter().map(|x| x.data.len()).saturating_sum() as u64; - pay!(data.saturating_mul(17_376) / 100_000); - - // pay for elements - let elems = bin.elements.iter().map(|x| x.range.len()).saturating_sum() as u64; - pay!(elems.saturating_mul(17_376) / 100_000); - - // pay for memory - let mem = bin.memories.first().map(|x| x.initial).unwrap_or_default(); - pay!(mem.saturating_mul(2217)); - - // pay for code - let code = bin.codes.iter().map(|x| x.expr.len()).saturating_sum() as u64; - pay!(code.saturating_mul(535) / 1_000); + macro_rules! pay { + ($us:expr) => { + let amount = us_to_gas($us); + if *gas < amount { + *gas = 0; + bail!("out of gas"); + } + *gas -= amount; + }; + } + + // pay for wasm + if arbos_version_for_gas >= ARBOS_VERSION_STYLUS_CHARGING_FIXES { + let wasm_len = wasm.len() as u64; + pay!(wasm_len.saturating_mul(31_733) / 100_000); + } + + // pay for funcs + let funcs = bin.functions.len() as u64; + pay!(funcs.saturating_mul(17_263) / 100_000); + + // pay for data + let data = bin.datas.iter().map(|x| x.data.len()).saturating_sum() as u64; + pay!(data.saturating_mul(17_376) / 100_000); + + // pay for elements + let elems = bin.elements.iter().map(|x| x.range.len()).saturating_sum() as u64; + pay!(elems.saturating_mul(17_376) / 100_000); + + // pay for memory + let mem = bin.memories.first().map(|x| x.initial).unwrap_or_default(); + pay!(mem.saturating_mul(2217)); + + // pay for code + let code = bin.codes.iter().map(|x| x.expr.len()).saturating_sum() as u64; + pay!(code.saturating_mul(535) / 1_000); + } let module = Self::from_user_binary(&bin, compile.debug.debug_funcs, Some(stylus_data)) .wrap_err("failed to build user module")?; diff --git a/arbitrator/prover/src/test.rs b/arbitrator/prover/src/test.rs index 2f32aff012..97170441ff 100644 --- a/arbitrator/prover/src/test.rs +++ b/arbitrator/prover/src/test.rs @@ -57,7 +57,7 @@ pub fn reject_ambiguous_imports() { #[test] pub fn test_compress() -> Result<()> { - let data = include_bytes!(concat!(env!("OUT_DIR"), "/forwarder.wasm")); + let data = include_bytes!("../../../target/machines/latest/forward_stub.wasm"); let mut last = vec![]; for dict in [Dictionary::Empty, Dictionary::StylusProgram] { diff --git a/arbitrator/prover/src/value.rs b/arbitrator/prover/src/value.rs index f2f54404f0..6afffdf7a0 100644 --- a/arbitrator/prover/src/value.rs +++ b/arbitrator/prover/src/value.rs @@ -2,7 +2,7 @@ // For license information, see https://github.com/nitro/blob/master/LICENSE use crate::binary::FloatType; -use arbutil::{hostios::ParamType, Bytes32, Color}; +use arbutil::{Bytes32, Color}; use digest::Digest; use eyre::{bail, ErrReport, Result}; use serde::{Deserialize, Serialize}; @@ -77,16 +77,6 @@ impl From for ValType { } } -impl From<&ParamType> for ArbValueType { - fn from(ty: &ParamType) -> Self { - use ParamType as V; - match ty { - V::I32 => Self::I32, - V::I64 => Self::I64, - } - } -} - #[cfg(feature = "native")] pub fn parser_type(ty: &wasmer::Type) -> wasmer::wasmparser::ValType { match ty { diff --git a/arbitrator/prover/test-cases/forward-test.wat b/arbitrator/prover/test-cases/forward-test.wat new file mode 100644 index 0000000000..b9beff0d82 --- /dev/null +++ b/arbitrator/prover/test-cases/forward-test.wat @@ -0,0 +1,32 @@ + +(module + (import "forward" "add" (func $add (param i32 i32) (result i32))) + (import "forward" "sub" (func $sub (param i32 i32) (result i32))) + (import "forward" "mul" (func $mul (param i32 i32) (result i32))) + (func $start + ;; this address will update each time a forwarded call is made + i32.const 0xa4b + i32.const 805 + i32.store + + i32.const 11 + i32.const 5 + call $sub + + i32.const 3 + i32.const -2 + call $mul + + call $add + (if + (then (unreachable))) + + ;; check that the address has changed + i32.const 0xa4b + i32.load + i32.const 808 + i32.ne + (if + (then (unreachable)))) + (start $start) + (memory 1)) diff --git a/arbitrator/prover/test-cases/forward/forward.wat b/arbitrator/prover/test-cases/forward/forward.wat new file mode 100644 index 0000000000..ff55953e62 --- /dev/null +++ b/arbitrator/prover/test-cases/forward/forward.wat @@ -0,0 +1,8 @@ + +(module + (import "target" "arbitrator_forward__add" (func $add (param i32 i32) (result i32))) + (import "target" "arbitrator_forward__sub" (func $sub (param i32 i32) (result i32))) + (import "target" "arbitrator_forward__mul" (func $mul (param i32 i32) (result i32))) + (export "forward__add" (func $add)) + (export "forward__sub" (func $sub)) + (export "forward__mul" (func $mul))) diff --git a/arbitrator/prover/test-cases/forward/target.wat b/arbitrator/prover/test-cases/forward/target.wat new file mode 100644 index 0000000000..0779eb753d --- /dev/null +++ b/arbitrator/prover/test-cases/forward/target.wat @@ -0,0 +1,27 @@ + +(module + (import "env" "wavm_caller_load8" (func $load (param i32) (result i32))) + (import "env" "wavm_caller_store8" (func $store (param i32 i32))) + (func (export "target__add") (param i32 i32) (result i32) + call $write_caller + local.get 0 + local.get 1 + i32.add) + (func (export "target__sub") (param i32 i32) (result i32) + call $write_caller + local.get 0 + local.get 1 + i32.sub) + (func (export "target__mul") (param i32 i32) (result i32) + call $write_caller + local.get 0 + local.get 1 + i32.mul) + (func $write_caller (export "write_caller") + ;; increment the value at address 0xa4b in the caller + i32.const 0xa4b + i32.const 0xa4b + call $load + i32.const 1 + i32.add + call $store)) diff --git a/arbitrator/stylus/src/host.rs b/arbitrator/stylus/src/host.rs index fbe5657c5f..1afc1b4e51 100644 --- a/arbitrator/stylus/src/host.rs +++ b/arbitrator/stylus/src/host.rs @@ -13,6 +13,7 @@ use arbutil::{ }; use caller_env::GuestPtr; use eyre::Result; +use prover::value::Value; use std::{ fmt::Display, mem::{self, MaybeUninit}, @@ -440,76 +441,26 @@ pub(crate) fn pay_for_memory_grow>( hostio!(env, pay_for_memory_grow(pages)) } -pub(crate) mod console { - use super::*; - - pub(crate) fn log_txt>( - mut env: WasmEnvMut, - ptr: GuestPtr, - len: u32, - ) -> MaybeEscape { - hostio!(env, console_log_text(ptr, len)) - } - - pub(crate) fn log_i32>( - mut env: WasmEnvMut, - value: u32, - ) -> MaybeEscape { - hostio!(env, console_log(value)) - } - - pub(crate) fn tee_i32>( - mut env: WasmEnvMut, - value: u32, - ) -> Result { - hostio!(env, console_tee(value)) - } - - pub(crate) fn log_i64>( - mut env: WasmEnvMut, - value: u64, - ) -> MaybeEscape { - hostio!(env, console_log(value)) - } - - pub(crate) fn tee_i64>( - mut env: WasmEnvMut, - value: u64, - ) -> Result { - hostio!(env, console_tee(value)) - } - - pub(crate) fn log_f32>( - mut env: WasmEnvMut, - value: f32, - ) -> MaybeEscape { - hostio!(env, console_log(value)) - } - - pub(crate) fn tee_f32>( - mut env: WasmEnvMut, - value: f32, - ) -> Result { - hostio!(env, console_tee(value)) - } - - pub(crate) fn log_f64>( - mut env: WasmEnvMut, - value: f64, - ) -> MaybeEscape { - hostio!(env, console_log(value)) - } - - pub(crate) fn tee_f64>( - mut env: WasmEnvMut, - value: f64, - ) -> Result { - hostio!(env, console_tee(value)) - } +pub(crate) fn console_log_text>( + mut env: WasmEnvMut, + ptr: GuestPtr, + len: u32, +) -> MaybeEscape { + hostio!(env, console_log_text(ptr, len)) } -pub(crate) mod debug { - use super::*; +pub(crate) fn console_log, T: Into>( + mut env: WasmEnvMut, + value: T, +) -> MaybeEscape { + hostio!(env, console_log(value)) +} - pub(crate) fn null_host>(_: WasmEnvMut) {} +pub(crate) fn console_tee, T: Into + Copy>( + mut env: WasmEnvMut, + value: T, +) -> Result { + hostio!(env, console_tee(value)) } + +pub(crate) fn null_host>(_: WasmEnvMut) {} diff --git a/arbitrator/stylus/src/lib.rs b/arbitrator/stylus/src/lib.rs index a252b60a01..052bfc7229 100644 --- a/arbitrator/stylus/src/lib.rs +++ b/arbitrator/stylus/src/lib.rs @@ -139,7 +139,8 @@ impl RustBytes { pub unsafe extern "C" fn stylus_activate( wasm: GoSliceData, page_limit: u16, - version: u16, + stylus_version: u16, + arbos_version_for_gas: u64, debug: bool, output: *mut RustBytes, codehash: *const Bytes32, @@ -153,7 +154,15 @@ pub unsafe extern "C" fn stylus_activate( let codehash = &*codehash; let gas = &mut *gas; - let (module, info) = match native::activate(wasm, codehash, version, page_limit, debug, gas) { + let (module, info) = match native::activate( + wasm, + codehash, + stylus_version, + arbos_version_for_gas, + page_limit, + debug, + gas, + ) { Ok(val) => val, Err(err) => return output.write_err(err), }; diff --git a/arbitrator/stylus/src/native.rs b/arbitrator/stylus/src/native.rs index 7a82314fbc..516c6602e7 100644 --- a/arbitrator/stylus/src/native.rs +++ b/arbitrator/stylus/src/native.rs @@ -33,7 +33,7 @@ use std::{ ops::{Deref, DerefMut}, }; use wasmer::{ - AsStoreMut, Function, FunctionEnv, Imports, Instance, Memory, Module, Pages, Store, Target, + imports, AsStoreMut, Function, FunctionEnv, Instance, Memory, Module, Pages, Store, Target, TypedFunction, Value, WasmTypeList, }; use wasmer_vm::VMExtern; @@ -151,58 +151,68 @@ impl> NativeInstance { fn from_module(module: Module, mut store: Store, env: WasmEnv) -> Result { let debug_funcs = env.compile.debug.debug_funcs; let func_env = FunctionEnv::new(&mut store, env); - let mut imports = Imports::new(); macro_rules! func { - ($rust_mod:path, $func:ident) => {{ - use $rust_mod as rust_mod; - Function::new_typed_with_env(&mut store, &func_env, rust_mod::$func) - }}; - } - macro_rules! define_imports { - ($($wasm_mod:literal => $rust_mod:path { $( $import:ident ),* $(,)? },)* $(,)?) => { - $( - $( - define_imports!(@@imports $wasm_mod, func!($rust_mod, $import), $import, "arbitrator_forward__"); - )* - )* - }; - (@@imports $wasm_mod:literal, $func:expr, $import:ident, $($p:expr),*) => { - define_imports!(@imports $wasm_mod, $func, $import, $($p),*, ""); - }; - (@imports $wasm_mod:literal, $func:expr, $import:ident, $($p:expr),*) => { - $( - imports.define($wasm_mod, concat!($p, stringify!($import)), $func); - )* + ($func:expr) => { + Function::new_typed_with_env(&mut store, &func_env, $func) }; } - define_imports!( - "vm_hooks" => host { - read_args, write_result, exit_early, - storage_load_bytes32, storage_cache_bytes32, storage_flush_cache, transient_load_bytes32, transient_store_bytes32, - call_contract, delegate_call_contract, static_call_contract, create1, create2, read_return_data, return_data_size, - emit_log, - account_balance, account_code, account_codehash, account_code_size, - evm_gas_left, evm_ink_left, - block_basefee, chainid, block_coinbase, block_gas_limit, block_number, block_timestamp, - contract_address, - math_div, math_mod, math_pow, math_add_mod, math_mul_mod, - msg_reentrant, msg_sender, msg_value, - tx_gas_price, tx_ink_price, tx_origin, - pay_for_memory_grow, - native_keccak256, + let mut imports = imports! { + "vm_hooks" => { + "read_args" => func!(host::read_args), + "write_result" => func!(host::write_result), + "exit_early" => func!(host::exit_early), + "storage_load_bytes32" => func!(host::storage_load_bytes32), + "storage_cache_bytes32" => func!(host::storage_cache_bytes32), + "storage_flush_cache" => func!(host::storage_flush_cache), + "transient_load_bytes32" => func!(host::transient_load_bytes32), + "transient_store_bytes32" => func!(host::transient_store_bytes32), + "call_contract" => func!(host::call_contract), + "delegate_call_contract" => func!(host::delegate_call_contract), + "static_call_contract" => func!(host::static_call_contract), + "create1" => func!(host::create1), + "create2" => func!(host::create2), + "read_return_data" => func!(host::read_return_data), + "return_data_size" => func!(host::return_data_size), + "emit_log" => func!(host::emit_log), + "account_balance" => func!(host::account_balance), + "account_code" => func!(host::account_code), + "account_codehash" => func!(host::account_codehash), + "account_code_size" => func!(host::account_code_size), + "evm_gas_left" => func!(host::evm_gas_left), + "evm_ink_left" => func!(host::evm_ink_left), + "block_basefee" => func!(host::block_basefee), + "chainid" => func!(host::chainid), + "block_coinbase" => func!(host::block_coinbase), + "block_gas_limit" => func!(host::block_gas_limit), + "block_number" => func!(host::block_number), + "block_timestamp" => func!(host::block_timestamp), + "contract_address" => func!(host::contract_address), + "math_div" => func!(host::math_div), + "math_mod" => func!(host::math_mod), + "math_pow" => func!(host::math_pow), + "math_add_mod" => func!(host::math_add_mod), + "math_mul_mod" => func!(host::math_mul_mod), + "msg_reentrant" => func!(host::msg_reentrant), + "msg_sender" => func!(host::msg_sender), + "msg_value" => func!(host::msg_value), + "tx_gas_price" => func!(host::tx_gas_price), + "tx_ink_price" => func!(host::tx_ink_price), + "tx_origin" => func!(host::tx_origin), + "pay_for_memory_grow" => func!(host::pay_for_memory_grow), + "native_keccak256" => func!(host::native_keccak256), }, - ); + }; if debug_funcs { - define_imports!( - "console" => host::console { - log_txt, - log_i32, log_i64, log_f32, log_f64, - tee_i32, tee_i64, tee_f32, tee_f64, - }, - "debug" => host::debug { - null_host, - }, - ); + imports.define("console", "log_txt", func!(host::console_log_text)); + imports.define("console", "log_i32", func!(host::console_log::)); + imports.define("console", "log_i64", func!(host::console_log::)); + imports.define("console", "log_f32", func!(host::console_log::)); + imports.define("console", "log_f64", func!(host::console_log::)); + imports.define("console", "tee_i32", func!(host::console_tee::)); + imports.define("console", "tee_i64", func!(host::console_tee::)); + imports.define("console", "tee_f32", func!(host::console_tee::)); + imports.define("console", "tee_f64", func!(host::console_tee::)); + imports.define("debug", "null_host", func!(host::null_host)); } let instance = Instance::new(&mut store, &module, &imports)?; let exports = &instance.exports; @@ -341,8 +351,86 @@ impl> StartlessMachine for NativeInstance { } pub fn module(wasm: &[u8], compile: CompileConfig, target: Target) -> Result> { - let store = compile.store(target); + let mut store = compile.store(target); let module = Module::new(&store, wasm)?; + macro_rules! stub { + (u8 <- $($types:tt)+) => { + Function::new_typed(&mut store, $($types)+ -> u8 { panic!("incomplete import") }) + }; + (u32 <- $($types:tt)+) => { + Function::new_typed(&mut store, $($types)+ -> u32 { panic!("incomplete import") }) + }; + (u64 <- $($types:tt)+) => { + Function::new_typed(&mut store, $($types)+ -> u64 { panic!("incomplete import") }) + }; + (f32 <- $($types:tt)+) => { + Function::new_typed(&mut store, $($types)+ -> f32 { panic!("incomplete import") }) + }; + (f64 <- $($types:tt)+) => { + Function::new_typed(&mut store, $($types)+ -> f64 { panic!("incomplete import") }) + }; + ($($types:tt)+) => { + Function::new_typed(&mut store, $($types)+ -> () { panic!("incomplete import") }) + }; + } + let mut imports = imports! { + "vm_hooks" => { + "read_args" => stub!(|_: u32|), + "write_result" => stub!(|_: u32, _: u32|), + "exit_early" => stub!(|_: u32|), + "storage_load_bytes32" => stub!(|_: u32, _: u32|), + "storage_cache_bytes32" => stub!(|_: u32, _: u32|), + "storage_flush_cache" => stub!(|_: u32|), + "transient_load_bytes32" => stub!(|_: u32, _: u32|), + "transient_store_bytes32" => stub!(|_: u32, _: u32|), + "call_contract" => stub!(u8 <- |_: u32, _: u32, _: u32, _: u32, _: u64, _: u32|), + "delegate_call_contract" => stub!(u8 <- |_: u32, _: u32, _: u32, _: u64, _: u32|), + "static_call_contract" => stub!(u8 <- |_: u32, _: u32, _: u32, _: u64, _: u32|), + "create1" => stub!(|_: u32, _: u32, _: u32, _: u32, _: u32|), + "create2" => stub!(|_: u32, _: u32, _: u32, _: u32, _: u32, _: u32|), + "read_return_data" => stub!(u32 <- |_: u32, _: u32, _: u32|), + "return_data_size" => stub!(u32 <- ||), + "emit_log" => stub!(|_: u32, _: u32, _: u32|), + "account_balance" => stub!(|_: u32, _: u32|), + "account_code" => stub!(u32 <- |_: u32, _: u32, _: u32, _: u32|), + "account_codehash" => stub!(|_: u32, _: u32|), + "account_code_size" => stub!(u32 <- |_: u32|), + "evm_gas_left" => stub!(u64 <- ||), + "evm_ink_left" => stub!(u64 <- ||), + "block_basefee" => stub!(|_: u32|), + "chainid" => stub!(u64 <- ||), + "block_coinbase" => stub!(|_: u32|), + "block_gas_limit" => stub!(u64 <- ||), + "block_number" => stub!(u64 <- ||), + "block_timestamp" => stub!(u64 <- ||), + "contract_address" => stub!(|_: u32|), + "math_div" => stub!(|_: u32, _: u32|), + "math_mod" => stub!(|_: u32, _: u32|), + "math_pow" => stub!(|_: u32, _: u32|), + "math_add_mod" => stub!(|_: u32, _: u32, _: u32|), + "math_mul_mod" => stub!(|_: u32, _: u32, _: u32|), + "msg_reentrant" => stub!(u32 <- ||), + "msg_sender" => stub!(|_: u32|), + "msg_value" => stub!(|_: u32|), + "tx_gas_price" => stub!(|_: u32|), + "tx_ink_price" => stub!(u32 <- ||), + "tx_origin" => stub!(|_: u32|), + "pay_for_memory_grow" => stub!(|_: u16|), + "native_keccak256" => stub!(|_: u32, _: u32, _: u32|), + }, + }; + if compile.debug.debug_funcs { + imports.define("console", "log_txt", stub!(|_: u32, _: u32|)); + imports.define("console", "log_i32", stub!(|_: u32|)); + imports.define("console", "log_i64", stub!(|_: u64|)); + imports.define("console", "log_f32", stub!(|_: f32|)); + imports.define("console", "log_f64", stub!(|_: f64|)); + imports.define("console", "tee_i32", stub!(u32 <- |_: u32|)); + imports.define("console", "tee_i64", stub!(u64 <- |_: u64|)); + imports.define("console", "tee_f32", stub!(f32 <- |_: f32|)); + imports.define("console", "tee_f64", stub!(f64 <- |_: f64|)); + imports.define("debug", "null_host", stub!(||)); + } let module = module.serialize()?; Ok(module.to_vec()) @@ -351,13 +439,21 @@ pub fn module(wasm: &[u8], compile: CompileConfig, target: Target) -> Result Result<(ProverModule, StylusData)> { - let (module, stylus_data) = - ProverModule::activate(wasm, codehash, version, page_limit, debug, gas)?; + let (module, stylus_data) = ProverModule::activate( + wasm, + codehash, + stylus_version, + arbos_version_for_gas, + page_limit, + debug, + gas, + )?; Ok((module, stylus_data)) } diff --git a/arbitrator/stylus/src/test/api.rs b/arbitrator/stylus/src/test/api.rs index 5d9f625e5e..66d600a6f7 100644 --- a/arbitrator/stylus/src/test/api.rs +++ b/arbitrator/stylus/src/test/api.rs @@ -68,7 +68,7 @@ impl TestEvmApi { } impl EvmApi for TestEvmApi { - fn get_bytes32(&mut self, key: Bytes32) -> (Bytes32, u64) { + fn get_bytes32(&mut self, key: Bytes32, _evm_api_gas_to_use: u64) -> (Bytes32, u64) { let storage = &mut self.storage.lock(); let storage = storage.get_mut(&self.program).unwrap(); let value = storage.get(&key).cloned().unwrap_or_default(); diff --git a/arbitrator/stylus/src/test/mod.rs b/arbitrator/stylus/src/test/mod.rs index 42c9446564..00c9c62ae4 100644 --- a/arbitrator/stylus/src/test/mod.rs +++ b/arbitrator/stylus/src/test/mod.rs @@ -164,7 +164,6 @@ fn new_test_machine(path: &str, compile: &CompileConfig) -> Result { HashMap::default(), Arc::new(|_, _, _| panic!("tried to read preimage")), Some(stylus_data), - false, )?; mach.set_ink(u64::MAX); mach.set_stack(u32::MAX); diff --git a/arbitrator/stylus/src/test/native.rs b/arbitrator/stylus/src/test/native.rs index 503e5875fe..9669932a03 100644 --- a/arbitrator/stylus/src/test/native.rs +++ b/arbitrator/stylus/src/test/native.rs @@ -381,7 +381,7 @@ fn test_storage() -> Result<()> { let (mut native, mut evm) = TestInstance::new_with_evm(filename, &compile, config)?; run_native(&mut native, &store_args, ink)?; - assert_eq!(evm.get_bytes32(key.into()).0, Bytes32(value)); + assert_eq!(evm.get_bytes32(key.into(), 0).0, Bytes32(value)); assert_eq!(run_native(&mut native, &load_args, ink)?, value); let mut machine = Machine::from_user_path(Path::new(filename), &compile)?; @@ -465,7 +465,7 @@ fn test_calls() -> Result<()> { run_native(&mut native, &args, ink)?; for (key, value) in slots { - assert_eq!(evm.get_bytes32(key).0, value); + assert_eq!(evm.get_bytes32(key, 0).0, value); } Ok(()) } diff --git a/arbitrator/tools/module_roots/src/main.rs b/arbitrator/tools/module_roots/src/main.rs index fc22bed5fa..32e4764847 100644 --- a/arbitrator/tools/module_roots/src/main.rs +++ b/arbitrator/tools/module_roots/src/main.rs @@ -39,7 +39,6 @@ fn main() -> Result<()> { GlobalState::default(), HashMap::default(), Arc::new(|_, _, _| panic!("tried to read preimage")), - false, )?; let mut stylus = vec![]; diff --git a/arbitrator/wasm-libraries/Cargo.lock b/arbitrator/wasm-libraries/Cargo.lock index b234424f69..7620ff538b 100644 --- a/arbitrator/wasm-libraries/Cargo.lock +++ b/arbitrator/wasm-libraries/Cargo.lock @@ -2,21 +2,6 @@ # It is not intended for manual editing. version = 3 -[[package]] -name = "addr2line" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" -dependencies = [ - "gimli 0.29.0", -] - -[[package]] -name = "adler" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" - [[package]] name = "ahash" version = "0.7.8" @@ -106,21 +91,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" -[[package]] -name = "backtrace" -version = "0.3.73" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" -dependencies = [ - "addr2line", - "cc", - "cfg-if 1.0.0", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", -] - [[package]] name = "bincode" version = "1.3.3" @@ -217,12 +187,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - [[package]] name = "bytes" version = "1.7.0" @@ -239,15 +203,6 @@ dependencies = [ "rand_pcg", ] -[[package]] -name = "cc" -version = "1.1.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57b6a275aa2903740dc87da01c62040406b8812552e97129a63ea8850a17c6e6" -dependencies = [ - "shlex", -] - [[package]] name = "cfg-if" version = "0.1.10" @@ -281,19 +236,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" -[[package]] -name = "corosensei" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80128832c58ea9cbd041d2a759ec449224487b2c1e400453d99d244eead87a8e" -dependencies = [ - "autocfg", - "cfg-if 1.0.0", - "libc", - "scopeguard", - "windows-sys 0.33.0", -] - [[package]] name = "cpufeatures" version = "0.2.12" @@ -303,123 +245,6 @@ dependencies = [ "libc", ] -[[package]] -name = "cranelift-bforest" -version = "0.91.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a2ab4512dfd3a6f4be184403a195f76e81a8a9f9e6c898e19d2dc3ce20e0115" -dependencies = [ - "cranelift-entity", -] - -[[package]] -name = "cranelift-codegen" -version = "0.91.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98b022ed2a5913a38839dfbafe6cf135342661293b08049843362df4301261dc" -dependencies = [ - "arrayvec", - "bumpalo", - "cranelift-bforest", - "cranelift-codegen-meta", - "cranelift-codegen-shared", - "cranelift-egraph", - "cranelift-entity", - "cranelift-isle", - "gimli 0.26.2", - "log", - "regalloc2", - "smallvec", - "target-lexicon", -] - -[[package]] -name = "cranelift-codegen-meta" -version = "0.91.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "639307b45434ad112a98f8300c0f0ab085cbefcd767efcdef9ef19d4c0756e74" -dependencies = [ - "cranelift-codegen-shared", -] - -[[package]] -name = "cranelift-codegen-shared" -version = "0.91.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "278e52e29c53fcf32431ef08406c295699a70306d05a0715c5b1bf50e33a9ab7" - -[[package]] -name = "cranelift-egraph" -version = "0.91.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624b54323b06e675293939311943ba82d323bb340468ce1889be5da7932c8d73" -dependencies = [ - "cranelift-entity", - "fxhash", - "hashbrown 0.12.3", - "indexmap 1.9.3", - "log", - "smallvec", -] - -[[package]] -name = "cranelift-entity" -version = "0.91.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a59bcbca89c3f1b70b93ab3cbba5e5e0cbf3e63dadb23c7525cb142e21a9d4c" - -[[package]] -name = "cranelift-frontend" -version = "0.91.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d70abacb8cfef3dc8ff7e8836e9c1d70f7967dfdac824a4cd5e30223415aca6" -dependencies = [ - "cranelift-codegen", - "log", - "smallvec", - "target-lexicon", -] - -[[package]] -name = "cranelift-isle" -version = "0.91.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "393bc73c451830ff8dbb3a07f61843d6cb41a084f9996319917c0b291ed785bb" - -[[package]] -name = "crossbeam-deque" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-queue" -version = "0.3.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" - [[package]] name = "crunchy" version = "0.2.2" @@ -505,19 +330,6 @@ dependencies = [ "syn 2.0.72", ] -[[package]] -name = "dashmap" -version = "5.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" -dependencies = [ - "cfg-if 1.0.0", - "hashbrown 0.14.5", - "lock_api", - "once_cell", - "parking_lot_core", -] - [[package]] name = "derivative" version = "2.2.0" @@ -644,33 +456,26 @@ dependencies = [ "once_cell", ] -[[package]] -name = "fallible-iterator" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" - [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "forward" +version = "0.1.0" +dependencies = [ + "eyre", + "structopt", +] + [[package]] name = "funty" version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" -[[package]] -name = "fxhash" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" -dependencies = [ - "byteorder", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -692,23 +497,6 @@ dependencies = [ "wasi", ] -[[package]] -name = "gimli" -version = "0.26.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d" -dependencies = [ - "fallible-iterator", - "indexmap 1.9.3", - "stable_deref_trait", -] - -[[package]] -name = "gimli" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" - [[package]] name = "hashbrown" version = "0.12.3" @@ -807,15 +595,6 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" -[[package]] -name = "js-sys" -version = "0.3.70" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" -dependencies = [ - "wasm-bindgen", -] - [[package]] name = "keccak" version = "0.1.5" @@ -853,12 +632,6 @@ dependencies = [ "scopeguard", ] -[[package]] -name = "log" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" - [[package]] name = "lru" version = "0.12.4" @@ -868,57 +641,12 @@ dependencies = [ "hashbrown 0.14.5", ] -[[package]] -name = "mach" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" -dependencies = [ - "libc", -] - -[[package]] -name = "mach2" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709" -dependencies = [ - "libc", -] - [[package]] name = "memchr" version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" -[[package]] -name = "memmap2" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" -dependencies = [ - "libc", -] - -[[package]] -name = "memmap2" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d28bba84adfe6646737845bc5ebbfa2c08424eb1c37e94a1fd2a82adb56a872" -dependencies = [ - "libc", -] - -[[package]] -name = "memoffset" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" -dependencies = [ - "autocfg", -] - [[package]] name = "memory_units" version = "0.4.0" @@ -931,15 +659,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" -[[package]] -name = "miniz_oxide" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" -dependencies = [ - "adler", -] - [[package]] name = "more-asserts" version = "0.2.2" @@ -1072,15 +791,6 @@ dependencies = [ "syn 2.0.72", ] -[[package]] -name = "object" -version = "0.36.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" -dependencies = [ - "memchr", -] - [[package]] name = "once_cell" version = "1.19.0" @@ -1122,12 +832,6 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" -[[package]] -name = "pin-project-lite" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" - [[package]] name = "proc-macro-crate" version = "3.1.0" @@ -1208,7 +912,6 @@ dependencies = [ "smallvec", "static_assertions", "structopt", - "wasmer", "wasmer-types", "wasmparser", "wat", @@ -1273,26 +976,6 @@ dependencies = [ "rand_core", ] -[[package]] -name = "rayon" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - [[package]] name = "redox_syscall" version = "0.5.3" @@ -1302,30 +985,6 @@ dependencies = [ "bitflags 2.6.0", ] -[[package]] -name = "regalloc2" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "300d4fbfb40c1c66a78ba3ddd41c1110247cf52f97b87d0f2fc9209bd49b030c" -dependencies = [ - "fxhash", - "log", - "slice-group-by", - "smallvec", -] - -[[package]] -name = "region" -version = "3.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6b6ebd13bc009aef9cd476c1310d49ac354d36e240cf1bd753290f3dc7199a7" -dependencies = [ - "bitflags 1.3.2", - "libc", - "mach2", - "windows-sys 0.52.0", -] - [[package]] name = "rend" version = "0.4.2" @@ -1416,12 +1075,6 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" -[[package]] -name = "self_cell" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d369a96f978623eb3dc28807c4852d6cc617fed53da5d3c400feff1ef34a714a" - [[package]] name = "semver" version = "1.0.23" @@ -1437,17 +1090,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde-wasm-bindgen" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3b4c031cd0d9014307d82b8abf653c0290fbdaeb4c02d00c63cf52f728628bf" -dependencies = [ - "js-sys", - "serde", - "wasm-bindgen", -] - [[package]] name = "serde_derive" version = "1.0.204" @@ -1539,22 +1181,6 @@ dependencies = [ "keccak", ] -[[package]] -name = "shared-buffer" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6c99835bad52957e7aa241d3975ed17c1e5f8c92026377d117a606f36b84b16" -dependencies = [ - "bytes", - "memmap2 0.6.2", -] - -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - [[package]] name = "simdutf8" version = "0.1.4" @@ -1567,12 +1193,6 @@ version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" -[[package]] -name = "slice-group-by" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "826167069c09b99d56f31e9ae5c99049e932a98c9dc2dac47645b08dbbf76ba7" - [[package]] name = "smallvec" version = "1.13.2" @@ -1582,12 +1202,6 @@ dependencies = [ "serde", ] -[[package]] -name = "stable_deref_trait" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" - [[package]] name = "static_assertions" version = "1.1.0" @@ -1734,37 +1348,6 @@ dependencies = [ "winnow", ] -[[package]] -name = "tracing" -version = "0.1.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" -dependencies = [ - "pin-project-lite", - "tracing-attributes", - "tracing-core", -] - -[[package]] -name = "tracing-attributes" -version = "0.1.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.72", -] - -[[package]] -name = "tracing-core" -version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" -dependencies = [ - "once_cell", -] - [[package]] name = "typenum" version = "1.17.0" @@ -1862,147 +1445,13 @@ dependencies = [ "wee_alloc", ] -[[package]] -name = "wasm-bindgen" -version = "0.2.93" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" -dependencies = [ - "cfg-if 1.0.0", - "once_cell", - "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.93" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" -dependencies = [ - "bumpalo", - "log", - "once_cell", - "proc-macro2", - "quote", - "syn 2.0.72", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.93" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.93" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.72", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.93" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" - [[package]] name = "wasm-encoder" -version = "0.32.0" +version = "0.215.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ba64e81215916eaeb48fee292f29401d69235d62d8b8fd92a7b2844ec5ae5f7" -dependencies = [ - "leb128", -] - -[[package]] -name = "wasmer" -version = "4.2.8" +checksum = "4fb56df3e06b8e6b77e37d2969a50ba51281029a9aeb3855e76b7f49b6418847" dependencies = [ - "bytes", - "cfg-if 1.0.0", - "derivative", - "indexmap 1.9.3", - "js-sys", - "more-asserts", - "rustc-demangle", - "serde", - "serde-wasm-bindgen", - "shared-buffer", - "target-lexicon", - "thiserror", - "tracing", - "wasm-bindgen", - "wasmer-compiler", - "wasmer-compiler-cranelift", - "wasmer-derive", - "wasmer-types", - "wasmer-vm", - "wat", - "winapi", -] - -[[package]] -name = "wasmer-compiler" -version = "4.2.8" -dependencies = [ - "backtrace", - "bytes", - "cfg-if 1.0.0", - "enum-iterator 0.7.0", - "enumset", - "lazy_static", "leb128", - "memmap2 0.5.10", - "more-asserts", - "region", - "rkyv", - "self_cell", - "shared-buffer", - "smallvec", - "thiserror", - "wasmer-types", - "wasmer-vm", - "wasmparser", - "winapi", -] - -[[package]] -name = "wasmer-compiler-cranelift" -version = "4.2.8" -dependencies = [ - "cranelift-codegen", - "cranelift-entity", - "cranelift-frontend", - "gimli 0.26.2", - "more-asserts", - "rayon", - "smallvec", - "target-lexicon", - "tracing", - "wasmer-compiler", - "wasmer-types", -] - -[[package]] -name = "wasmer-derive" -version = "4.2.8" -dependencies = [ - "proc-macro-error", - "proc-macro2", - "quote", - "syn 1.0.109", ] [[package]] @@ -2019,32 +1468,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "wasmer-vm" -version = "4.2.8" -dependencies = [ - "backtrace", - "cc", - "cfg-if 1.0.0", - "corosensei", - "crossbeam-queue", - "dashmap", - "derivative", - "enum-iterator 0.7.0", - "fnv", - "indexmap 1.9.3", - "lazy_static", - "libc", - "mach", - "memoffset", - "more-asserts", - "region", - "scopeguard", - "thiserror", - "wasmer-types", - "winapi", -] - [[package]] name = "wasmparser" version = "0.121.2" @@ -2058,10 +1481,11 @@ dependencies = [ [[package]] name = "wast" -version = "64.0.0" +version = "215.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a259b226fd6910225aa7baeba82f9d9933b6d00f2ce1b49b80fa4214328237cc" +checksum = "1ff1d00d893593249e60720be04a7c1f42f1c4dc3806a2869f4e66ab61eb54cb" dependencies = [ + "bumpalo", "leb128", "memchr", "unicode-width", @@ -2070,9 +1494,9 @@ dependencies = [ [[package]] name = "wat" -version = "1.0.71" +version = "1.215.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53253d920ab413fca1c7dc2161d601c79b4fdf631d0ba51dd4343bf9b556c3f6" +checksum = "670bf4d9c8cf76ae242d70ded47c546525b6dafaa6871f9bcb065344bf2b4e3d" dependencies = [ "wast", ] @@ -2111,28 +1535,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows-sys" -version = "0.33.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43dbb096663629518eb1dfa72d80243ca5a6aca764cae62a2df70af760a9be75" -dependencies = [ - "windows_aarch64_msvc 0.33.0", - "windows_i686_gnu 0.33.0", - "windows_i686_msvc 0.33.0", - "windows_x86_64_gnu 0.33.0", - "windows_x86_64_msvc 0.33.0", -] - -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets", -] - [[package]] name = "windows-targets" version = "0.52.6" @@ -2140,13 +1542,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ "windows_aarch64_gnullvm", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", + "windows_aarch64_msvc", + "windows_i686_gnu", "windows_i686_gnullvm", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", + "windows_i686_msvc", + "windows_x86_64_gnu", "windows_x86_64_gnullvm", - "windows_x86_64_msvc 0.52.6", + "windows_x86_64_msvc", ] [[package]] @@ -2155,24 +1557,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_msvc" -version = "0.33.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd761fd3eb9ab8cc1ed81e56e567f02dd82c4c837e48ac3b2181b9ffc5060807" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_i686_gnu" -version = "0.33.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cab0cf703a96bab2dc0c02c0fa748491294bf9b7feb27e1f4f96340f208ada0e" - [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -2185,24 +1575,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_msvc" -version = "0.33.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cfdbe89cc9ad7ce618ba34abc34bbb6c36d99e96cae2245b7943cd75ee773d0" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_x86_64_gnu" -version = "0.33.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4dd9b0c0e9ece7bb22e84d70d01b71c6d6248b81a3c60d11869451b4cb24784" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -2215,12 +1593,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_msvc" -version = "0.33.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff1e4aa646495048ec7f3ffddc411e1d829c026a2ec62b39da15c1055e406eaa" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" diff --git a/arbitrator/wasm-libraries/Cargo.toml b/arbitrator/wasm-libraries/Cargo.toml index 2fd585ee7d..837df8f4da 100644 --- a/arbitrator/wasm-libraries/Cargo.toml +++ b/arbitrator/wasm-libraries/Cargo.toml @@ -7,5 +7,6 @@ members = [ "user-host-trait", "user-test", "program-exec", + "forward", ] resolver = "2" diff --git a/arbitrator/wasm-libraries/forward/.gitignore b/arbitrator/wasm-libraries/forward/.gitignore new file mode 100644 index 0000000000..40da2042b7 --- /dev/null +++ b/arbitrator/wasm-libraries/forward/.gitignore @@ -0,0 +1 @@ +**.wat diff --git a/arbitrator/wasm-libraries/forward/Cargo.toml b/arbitrator/wasm-libraries/forward/Cargo.toml new file mode 100644 index 0000000000..73ed9d8827 --- /dev/null +++ b/arbitrator/wasm-libraries/forward/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "forward" +version = "0.1.0" +edition = "2021" + +[dependencies] +eyre = "0.6.5" +structopt = "0.3.26" diff --git a/arbitrator/wasm-libraries/forward/src/main.rs b/arbitrator/wasm-libraries/forward/src/main.rs new file mode 100644 index 0000000000..f978a8723b --- /dev/null +++ b/arbitrator/wasm-libraries/forward/src/main.rs @@ -0,0 +1,207 @@ +// Copyright 2022-2024, Offchain Labs, Inc. +// For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE + +use eyre::Result; +use std::{fs::File, io::Write, path::PathBuf}; +use structopt::StructOpt; + +/// order matters! +const HOSTIOS: [[&str; 3]; 42] = [ + ["read_args", "i32", ""], + ["write_result", "i32 i32", ""], + ["exit_early", "i32", ""], + ["storage_load_bytes32", "i32 i32", ""], + ["storage_cache_bytes32", "i32 i32", ""], + ["storage_flush_cache", "i32", ""], + ["transient_load_bytes32", "i32 i32", ""], + ["transient_store_bytes32", "i32 i32", ""], + ["call_contract", "i32 i32 i32 i32 i64 i32", "i32"], + ["delegate_call_contract", "i32 i32 i32 i64 i32", "i32"], + ["static_call_contract", "i32 i32 i32 i64 i32", "i32"], + ["create1", "i32 i32 i32 i32 i32", ""], + ["create2", "i32 i32 i32 i32 i32 i32", ""], + ["read_return_data", "i32 i32 i32", "i32"], + ["return_data_size", "", "i32"], + ["emit_log", "i32 i32 i32", ""], + ["account_balance", "i32 i32", ""], + ["account_code", "i32 i32 i32 i32", "i32"], + ["account_code_size", "i32", "i32"], + ["account_codehash", "i32 i32", ""], + ["evm_gas_left", "", "i64"], + ["evm_ink_left", "", "i64"], + ["block_basefee", "i32", ""], + ["chainid", "", "i64"], + ["block_coinbase", "i32", ""], + ["block_gas_limit", "", "i64"], + ["block_number", "", "i64"], + ["block_timestamp", "", "i64"], + ["contract_address", "i32", ""], + ["math_div", "i32 i32", ""], + ["math_mod", "i32 i32", ""], + ["math_pow", "i32 i32", ""], + ["math_add_mod", "i32 i32 i32", ""], + ["math_mul_mod", "i32 i32 i32", ""], + ["msg_reentrant", "", "i32"], + ["msg_sender", "i32", ""], + ["msg_value", "i32", ""], + ["native_keccak256", "i32 i32 i32", ""], + ["tx_gas_price", "i32", ""], + ["tx_ink_price", "", "i32"], + ["tx_origin", "i32", ""], + ["pay_for_memory_grow", "i32", ""], +]; + +#[derive(StructOpt)] +#[structopt(name = "arbitrator-prover")] +struct Opts { + #[structopt(long)] + path: PathBuf, + #[structopt(long)] + stub: bool, +} + +fn main() -> Result<()> { + let opts = Opts::from_args(); + let file = &mut File::options() + .create(true) + .write(true) + .truncate(true) + .open(opts.path)?; + + match opts.stub { + true => forward_stub(file), + false => forward(file), + } +} + +fn forward(file: &mut File) -> Result<()> { + macro_rules! wln { + ($($text:tt)*) => { + writeln!(file, $($text)*)?; + }; + } + let s = " "; + + wln!( + ";; Copyright 2022-2023, Offchain Labs, Inc.\n\ + ;; For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE\n\ + ;; This file is auto-generated.\n\ + \n\ + (module" + ); + + macro_rules! group { + ($list:expr, $kind:expr) => { + (!$list.is_empty()) + .then(|| format!(" ({} {})", $kind, $list)) + .unwrap_or_default() + }; + } + + wln!("{s};; symbols to re-export"); + for [name, ins, outs] in HOSTIOS { + let params = group!(ins, "param"); + let result = group!(outs, "result"); + wln!( + r#"{s}(import "user_host" "arbitrator_forward__{name}" (func ${name}{params}{result}))"# + ); + } + wln!(); + + wln!("{s};; reserved offsets for future user_host imports"); + for i in HOSTIOS.len()..512 { + wln!("{s}(func $reserved_{i} unreachable)"); + } + wln!(); + + wln!( + "{s};; allows user_host to request a trap\n\ + {s}(global $trap (mut i32) (i32.const 0))\n\ + {s}(func $check\n\ + {s}{s}global.get $trap ;; see if set\n\ + {s}{s}(global.set $trap (i32.const 0)) ;; reset the flag\n\ + {s}{s}(if (then (unreachable)))\n\ + {s})\n\ + {s}(func (export \"forward__set_trap\")\n\ + {s}{s}(global.set $trap (i32.const 1))\n\ + {s})\n" + ); + + wln!("{s};; user linkage"); + for [name, ins, outs] in HOSTIOS { + let params = group!(ins, "param"); + let result = group!(outs, "result"); + wln!("{s}(func (export \"vm_hooks__{name}\"){params}{result}"); + + let gets = (1 + ins.len()) / 4; + for i in 0..gets { + wln!("{s}{s}local.get {i}"); + } + + wln!( + "{s}{s}call ${name}\n\ + {s}{s}call $check\n\ + {s})" + ); + } + + wln!(")"); + Ok(()) +} + +fn forward_stub(file: &mut File) -> Result<()> { + macro_rules! wln { + ($($text:tt)*) => { + writeln!(file, $($text)*)?; + }; + } + let s = " "; + + wln!( + ";; Copyright 2022-2023, Offchain Labs, Inc.\n\ + ;; For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE\n\ + ;; This file is auto-generated.\n\ + \n\ + (module" + ); + + macro_rules! group { + ($list:expr, $kind:expr) => { + (!$list.is_empty()) + .then(|| format!(" ({} {})", $kind, $list)) + .unwrap_or_default() + }; + } + + wln!("{s};; stubs for the symbols we re-export"); + for [name, ins, outs] in HOSTIOS { + let params = group!(ins, "param"); + let result = group!(outs, "result"); + wln!("{s}(func ${name}{params}{result} unreachable)"); + } + wln!(); + + wln!("{s};; reserved offsets for future user_host imports"); + for i in HOSTIOS.len()..512 { + wln!("{s}(func $reserved_{i} unreachable)"); + } + wln!(); + + wln!( + "{s};; allows user_host to request a trap\n\ + {s}(global $trap (mut i32) (i32.const 0))\n\ + {s}(func $check unreachable)\n\ + {s};; stub for the forward__set_trap function\n\ + {s}(func $forward__set_trap unreachable)" + ); + + wln!("{s};; user linkage"); + for [name, ins, outs] in HOSTIOS { + let params = group!(ins, "param"); + let result = group!(outs, "result"); + wln!("{s}(func (export \"vm_hooks__{name}\"){params}{result} unreachable)"); + } + + wln!(")"); + Ok(()) +} diff --git a/arbitrator/wasm-libraries/user-host-trait/src/lib.rs b/arbitrator/wasm-libraries/user-host-trait/src/lib.rs index 37af85c382..12a6bdbed2 100644 --- a/arbitrator/wasm-libraries/user-host-trait/src/lib.rs +++ b/arbitrator/wasm-libraries/user-host-trait/src/lib.rs @@ -8,7 +8,7 @@ use arbutil::{ api::{DataReader, EvmApi}, storage::StorageCache, user::UserOutcomeKind, - EvmData, + EvmData, ARBOS_VERSION_STYLUS_CHARGING_FIXES, }, pricing::{self, EVM_API_INK, HOSTIO_INK, PTR_INK}, Bytes20, Bytes32, @@ -143,11 +143,20 @@ pub trait UserHost: GasMeteredMachine { /// [`SLOAD`]: https://www.evm.codes/#54 fn storage_load_bytes32(&mut self, key: GuestPtr, dest: GuestPtr) -> Result<(), Self::Err> { self.buy_ink(HOSTIO_INK + 2 * PTR_INK)?; - self.require_gas(evm::COLD_SLOAD_GAS + EVM_API_INK + StorageCache::REQUIRED_ACCESS_GAS)?; // cache-miss case + let arbos_version = self.evm_data().arbos_version; + // require for cache-miss case, preserve wrong behavior for old arbos + let evm_api_gas_to_use = if arbos_version < ARBOS_VERSION_STYLUS_CHARGING_FIXES { + EVM_API_INK + } else { + self.pricing().ink_to_gas(EVM_API_INK) + }; + self.require_gas( + evm::COLD_SLOAD_GAS + StorageCache::REQUIRED_ACCESS_GAS + evm_api_gas_to_use, + )?; let key = self.read_bytes32(key)?; - let (value, gas_cost) = self.evm_api().get_bytes32(key); + let (value, gas_cost) = self.evm_api().get_bytes32(key, evm_api_gas_to_use); self.buy_gas(gas_cost)?; self.write_bytes32(dest, value)?; trace!("storage_load_bytes32", self, key, value) @@ -185,7 +194,10 @@ pub trait UserHost: GasMeteredMachine { self.require_gas(evm::SSTORE_SENTRY_GAS)?; // see operations_acl_arbitrum.go let gas_left = self.gas_left()?; - self.evm_api().flush_storage_cache(clear, gas_left)?; + let gas_cost = self.evm_api().flush_storage_cache(clear, gas_left)?; + if self.evm_data().arbos_version >= ARBOS_VERSION_STYLUS_CHARGING_FIXES { + self.buy_gas(gas_cost)?; + } trace!("storage_flush_cache", self, [be!(clear as u8)], &[]) } diff --git a/arbitrator/wasm-libraries/user-host/src/link.rs b/arbitrator/wasm-libraries/user-host/src/link.rs index 428611167d..f4c402fd97 100644 --- a/arbitrator/wasm-libraries/user-host/src/link.rs +++ b/arbitrator/wasm-libraries/user-host/src/link.rs @@ -37,14 +37,15 @@ struct MemoryLeaf([u8; 32]); /// /// pages_ptr: starts pointing to max allowed pages, returns number of pages used #[no_mangle] -pub unsafe extern "C" fn programs__activate( +pub unsafe extern "C" fn programs__activate_v2( wasm_ptr: GuestPtr, wasm_size: usize, pages_ptr: GuestPtr, asm_estimate_ptr: GuestPtr, init_cost_ptr: GuestPtr, cached_init_cost_ptr: GuestPtr, - version: u16, + stylus_version: u16, + arbos_version_for_gas: u64, debug: u32, codehash: GuestPtr, module_hash_ptr: GuestPtr, @@ -58,7 +59,15 @@ pub unsafe extern "C" fn programs__activate( let page_limit = STATIC_MEM.read_u16(pages_ptr); let gas_left = &mut STATIC_MEM.read_u64(gas_ptr); - match Module::activate(&wasm, codehash, version, page_limit, debug, gas_left) { + match Module::activate( + &wasm, + codehash, + stylus_version, + arbos_version_for_gas, + page_limit, + debug, + gas_left, + ) { Ok((module, data)) => { STATIC_MEM.write_u64(gas_ptr, *gas_left); STATIC_MEM.write_u16(pages_ptr, data.footprint); @@ -242,7 +251,8 @@ pub unsafe extern "C" fn programs__create_stylus_config( /// Creates an `EvmData` handler from its component parts. /// #[no_mangle] -pub unsafe extern "C" fn programs__create_evm_data( +pub unsafe extern "C" fn programs__create_evm_data_v2( + arbos_version: u64, block_basefee_ptr: GuestPtr, chainid: u64, block_coinbase_ptr: GuestPtr, @@ -259,6 +269,7 @@ pub unsafe extern "C" fn programs__create_evm_data( reentrant: u32, ) -> u64 { let evm_data = EvmData { + arbos_version, block_basefee: read_bytes32(block_basefee_ptr), cached: cached != 0, chainid, diff --git a/arbitrator/wasm-libraries/user-test/src/program.rs b/arbitrator/wasm-libraries/user-test/src/program.rs index c56ea52ad0..85b522ee74 100644 --- a/arbitrator/wasm-libraries/user-test/src/program.rs +++ b/arbitrator/wasm-libraries/user-test/src/program.rs @@ -102,7 +102,7 @@ impl Program { pub struct MockEvmApi; impl EvmApi for MockEvmApi { - fn get_bytes32(&mut self, key: Bytes32) -> (Bytes32, u64) { + fn get_bytes32(&mut self, key: Bytes32, _evm_api_gas_to_use: u64) -> (Bytes32, u64) { let value = KEYS.lock().get(&key).cloned().unwrap_or_default(); (value, 2100) // pretend worst case } diff --git a/arbitrator/wasm-testsuite/src/main.rs b/arbitrator/wasm-testsuite/src/main.rs index cb861e3a35..2144dcf992 100644 --- a/arbitrator/wasm-testsuite/src/main.rs +++ b/arbitrator/wasm-testsuite/src/main.rs @@ -343,7 +343,6 @@ fn main() -> eyre::Result<()> { GlobalState::default(), HashMap::default(), machine::get_empty_preimage_resolver(), - false, ); if let Err(error) = &mech { @@ -430,7 +429,6 @@ fn main() -> eyre::Result<()> { GlobalState::default(), HashMap::default(), machine::get_empty_preimage_resolver(), - false, ) .expect_err(&format!("failed to reject invalid module {}", filename)); } diff --git a/arbos/arbosState/arbosstate.go b/arbos/arbosState/arbosstate.go index 91c2207aae..f53d9c892a 100644 --- a/arbos/arbosState/arbosstate.go +++ b/arbos/arbosState/arbosstate.go @@ -41,28 +41,29 @@ import ( // persisted beyond the end of the test.) type ArbosState struct { - arbosVersion uint64 // version of the ArbOS storage format and semantics - maxArbosVersionSupported uint64 // maximum ArbOS version supported by this code - maxDebugArbosVersionSupported uint64 // maximum ArbOS version supported by this code in debug mode - upgradeVersion storage.StorageBackedUint64 // version we're planning to upgrade to, or 0 if not planning to upgrade - upgradeTimestamp storage.StorageBackedUint64 // when to do the planned upgrade - networkFeeAccount storage.StorageBackedAddress - l1PricingState *l1pricing.L1PricingState - l2PricingState *l2pricing.L2PricingState - retryableState *retryables.RetryableState - addressTable *addressTable.AddressTable - chainOwners *addressSet.AddressSet - sendMerkle *merkleAccumulator.MerkleAccumulator - programs *programs.Programs - blockhashes *blockhash.Blockhashes - chainId storage.StorageBackedBigInt - chainConfig storage.StorageBackedBytes - genesisBlockNum storage.StorageBackedUint64 - infraFeeAccount storage.StorageBackedAddress - brotliCompressionLevel storage.StorageBackedUint64 // brotli compression level used for pricing - backingStorage *storage.Storage - Burner burn.Burner -} + arbosVersion uint64 // version of the ArbOS storage format and semantics + upgradeVersion storage.StorageBackedUint64 // version we're planning to upgrade to, or 0 if not planning to upgrade + upgradeTimestamp storage.StorageBackedUint64 // when to do the planned upgrade + networkFeeAccount storage.StorageBackedAddress + l1PricingState *l1pricing.L1PricingState + l2PricingState *l2pricing.L2PricingState + retryableState *retryables.RetryableState + addressTable *addressTable.AddressTable + chainOwners *addressSet.AddressSet + sendMerkle *merkleAccumulator.MerkleAccumulator + programs *programs.Programs + blockhashes *blockhash.Blockhashes + chainId storage.StorageBackedBigInt + chainConfig storage.StorageBackedBytes + genesisBlockNum storage.StorageBackedUint64 + infraFeeAccount storage.StorageBackedAddress + brotliCompressionLevel storage.StorageBackedUint64 // brotli compression level used for pricing + backingStorage *storage.Storage + Burner burn.Burner +} + +const MaxArbosVersionSupported uint64 = params.ArbosVersion_StylusChargingFixes +const MaxDebugArbosVersionSupported uint64 = params.ArbosVersion_StylusChargingFixes var ErrUninitializedArbOS = errors.New("ArbOS uninitialized") var ErrAlreadyInitialized = errors.New("ArbOS is already initialized") @@ -78,8 +79,6 @@ func OpenArbosState(stateDB vm.StateDB, burner burn.Burner) (*ArbosState, error) } return &ArbosState{ arbosVersion, - 31, - 31, backingStorage.OpenStorageBackedUint64(uint64(upgradeVersionOffset)), backingStorage.OpenStorageBackedUint64(uint64(upgradeTimestampOffset)), backingStorage.OpenStorageBackedAddress(uint64(networkFeeAccountOffset)), @@ -332,6 +331,9 @@ func (state *ArbosState) UpgradeArbosVersion( ensure(params.UpgradeToVersion(2)) ensure(params.Save()) + case 32: + // no change state needed + default: return fmt.Errorf( "the chain is upgrading to unsupported ArbOS version %v, %w", @@ -416,14 +418,6 @@ func (state *ArbosState) RetryableState() *retryables.RetryableState { return state.retryableState } -func (state *ArbosState) MaxArbosVersionSupported() uint64 { - return state.maxArbosVersionSupported -} - -func (state *ArbosState) MaxDebugArbosVersionSupported() uint64 { - return state.maxDebugArbosVersionSupported -} - func (state *ArbosState) L1PricingState() *l1pricing.L1PricingState { return state.l1PricingState } diff --git a/arbos/programs/native.go b/arbos/programs/native.go index e834888b47..b13cfef2cf 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -51,11 +51,12 @@ func activateProgram( codehash common.Hash, wasm []byte, page_limit uint16, - version uint16, + stylusVersion uint16, + arbosVersionForGas uint64, debug bool, burner burn.Burner, ) (*activationInfo, error) { - info, asmMap, err := activateProgramInternal(db, program, codehash, wasm, page_limit, version, debug, burner.GasLeft()) + info, asmMap, err := activateProgramInternal(db, program, codehash, wasm, page_limit, stylusVersion, arbosVersionForGas, debug, burner.GasLeft()) if err != nil { return nil, err } @@ -69,7 +70,8 @@ func activateProgramInternal( codehash common.Hash, wasm []byte, page_limit uint16, - version uint16, + stylusVersion uint16, + arbosVersionForGas uint64, debug bool, gasLeft *uint64, ) (*activationInfo, map[ethdb.WasmTarget][]byte, error) { @@ -81,7 +83,8 @@ func activateProgramInternal( status_mod := userStatus(C.stylus_activate( goSlice(wasm), u16(page_limit), - u16(version), + u16(stylusVersion), + u64(arbosVersionForGas), cbool(debug), output, &codeHash, @@ -100,6 +103,7 @@ func activateProgramInternal( } return nil, nil, err } + hash := moduleHash.toHash() targets := db.Database().WasmTargets() type result struct { target ethdb.WasmTarget @@ -116,7 +120,7 @@ func activateProgramInternal( output := &rustBytes{} status_asm := C.stylus_compile( goSlice(wasm), - u16(version), + u16(stylusVersion), cbool(debug), goSlice([]byte(target)), output, @@ -140,10 +144,17 @@ func activateProgramInternal( } } if err != nil { - return nil, nil, fmt.Errorf("compilation failed for one or more targets: %w", err) + log.Error( + "Compilation failed for one or more targets despite activation succeeding", + "address", addressForLogging, + "codeHash", codeHash, + "moduleHash", hash, + "targets", targets, + "err", err, + ) + panic(fmt.Sprintf("Compilation of %v failed for one or more targets despite activation succeeding: %v", addressForLogging, err)) } - hash := moduleHash.toHash() info := &activationInfo{ moduleHash: hash, initGas: uint16(stylusData.init_cost), @@ -168,9 +179,12 @@ func getLocalAsm(statedb vm.StateDB, moduleHash common.Hash, addressForLogging c return nil, fmt.Errorf("failed to reactivate program address: %v err: %w", addressForLogging, err) } - unlimitedGas := uint64(0xffffffffffff) + // don't charge gas + zeroArbosVersion := uint64(0) + zeroGas := uint64(0) + // we know program is activated, so it must be in correct version and not use too much memory - info, asmMap, err := activateProgramInternal(statedb, addressForLogging, codeHash, wasm, pagelimit, program.version, debugMode, &unlimitedGas) + info, asmMap, err := activateProgramInternal(statedb, addressForLogging, codeHash, wasm, pagelimit, program.version, zeroArbosVersion, debugMode, &zeroGas) if err != nil { log.Error("failed to reactivate program", "address", addressForLogging, "expected moduleHash", moduleHash, "err", err) return nil, fmt.Errorf("failed to reactivate program address: %v err: %w", addressForLogging, err) @@ -391,6 +405,7 @@ func (params *ProgParams) encode() C.StylusConfig { func (data *EvmData) encode() C.EvmData { return C.EvmData{ + arbos_version: u64(data.arbosVersion), block_basefee: hashToBytes32(data.blockBasefee), chainid: u64(data.chainId), block_coinbase: addressToBytes20(data.blockCoinbase), diff --git a/arbos/programs/programs.go b/arbos/programs/programs.go index 12102bac84..80fc139a9d 100644 --- a/arbos/programs/programs.go +++ b/arbos/programs/programs.go @@ -82,7 +82,7 @@ func (p Programs) CacheManagers() *addressSet.AddressSet { return p.cacheManagers } -func (p Programs) ActivateProgram(evm *vm.EVM, address common.Address, runMode core.MessageRunMode, debugMode bool) ( +func (p Programs) ActivateProgram(evm *vm.EVM, address common.Address, arbosVersion uint64, runMode core.MessageRunMode, debugMode bool) ( uint16, common.Hash, common.Hash, *big.Int, bool, error, ) { statedb := evm.StateDB @@ -116,7 +116,7 @@ func (p Programs) ActivateProgram(evm *vm.EVM, address common.Address, runMode c // require the program's footprint not exceed the remaining memory budget pageLimit := am.SaturatingUSub(params.PageLimit, statedb.GetStylusPagesOpen()) - info, err := activateProgram(statedb, address, codeHash, wasm, pageLimit, stylusVersion, debugMode, burner) + info, err := activateProgram(statedb, address, codeHash, wasm, pageLimit, stylusVersion, arbosVersion, debugMode, burner) if err != nil { return 0, codeHash, common.Hash{}, nil, true, err } @@ -222,6 +222,7 @@ func (p Programs) CallProgram( } evmData := &EvmData{ + arbosVersion: evm.Context.ArbOSVersion, blockBasefee: common.BigToHash(evm.Context.BaseFee), chainId: evm.ChainConfig().ChainID.Uint64(), blockCoinbase: evm.Context.Coinbase, @@ -517,6 +518,7 @@ func (p Programs) progParams(version uint16, debug bool, params *StylusParams) * } type EvmData struct { + arbosVersion uint64 blockBasefee common.Hash chainId uint64 blockCoinbase common.Address diff --git a/arbos/programs/wasm.go b/arbos/programs/wasm.go index f7191dca8f..8f79944fbe 100644 --- a/arbos/programs/wasm.go +++ b/arbos/programs/wasm.go @@ -36,7 +36,7 @@ type rustConfig byte type rustModule byte type rustEvmData byte -//go:wasmimport programs activate +//go:wasmimport programs activate_v2 func programActivate( wasm_ptr unsafe.Pointer, wasm_size uint32, @@ -44,7 +44,8 @@ func programActivate( asm_estimation_ptr unsafe.Pointer, init_gas_ptr unsafe.Pointer, cached_init_gas_ptr unsafe.Pointer, - version uint32, + stylusVersion uint32, + arbosVersion uint64, debug uint32, codehash unsafe.Pointer, module_hash_ptr unsafe.Pointer, @@ -59,7 +60,8 @@ func activateProgram( codehash common.Hash, wasm []byte, pageLimit u16, - version u16, + stylusVersion u16, + arbosVersion uint64, debug bool, burner burn.Burner, ) (*activationInfo, error) { @@ -79,7 +81,8 @@ func activateProgram( unsafe.Pointer(&asmEstimate), unsafe.Pointer(&initGas), unsafe.Pointer(&cachedInitGas), - uint32(version), + uint32(stylusVersion), + arbosVersion, debugMode, arbutil.SliceToUnsafePointer(codehash[:]), arbutil.SliceToUnsafePointer(moduleHash[:]), diff --git a/arbos/programs/wasm_api.go b/arbos/programs/wasm_api.go index d7bac056c0..a4ebc1f778 100644 --- a/arbos/programs/wasm_api.go +++ b/arbos/programs/wasm_api.go @@ -20,8 +20,9 @@ func createStylusConfig(version uint32, max_depth uint32, ink_price uint32, debu type evmDataHandler uint64 -//go:wasmimport programs create_evm_data +//go:wasmimport programs create_evm_data_v2 func createEvmData( + arbosVersion uint64, blockBaseFee unsafe.Pointer, chainid uint64, blockCoinbase unsafe.Pointer, @@ -45,6 +46,7 @@ func (params *ProgParams) createHandler() stylusConfigHandler { func (data *EvmData) createHandler() evmDataHandler { return createEvmData( + data.arbosVersion, arbutil.SliceToUnsafePointer(data.blockBasefee[:]), data.chainId, arbutil.SliceToUnsafePointer(data.blockCoinbase[:]), diff --git a/arbos/programs/wasmstorehelper.go b/arbos/programs/wasmstorehelper.go index 434820dd9c..c2d1aa65b0 100644 --- a/arbos/programs/wasmstorehelper.go +++ b/arbos/programs/wasmstorehelper.go @@ -17,12 +17,12 @@ import ( // SaveActiveProgramToWasmStore is used to save active stylus programs to wasm store during rebuilding func (p Programs) SaveActiveProgramToWasmStore(statedb *state.StateDB, codeHash common.Hash, code []byte, time uint64, debugMode bool, rebuildingStartBlockTime uint64) error { - params, err := p.Params() + progParams, err := p.Params() if err != nil { return err } - program, err := p.getActiveProgram(codeHash, time, params) + program, err := p.getActiveProgram(codeHash, time, progParams) if err != nil { // The program is not active so return early log.Info("program is not active, getActiveProgram returned error, hence do not include in rebuilding", "err", err) @@ -56,10 +56,13 @@ func (p Programs) SaveActiveProgramToWasmStore(statedb *state.StateDB, codeHash return fmt.Errorf("failed to reactivate program while rebuilding wasm store: %w", err) } - unlimitedGas := uint64(0xffffffffffff) + // don't charge gas + zeroArbosVersion := uint64(0) + zeroGas := uint64(0) + // We know program is activated, so it must be in correct version and not use too much memory // Empty program address is supplied because we dont have access to this during rebuilding of wasm store - info, asmMap, err := activateProgramInternal(statedb, common.Address{}, codeHash, wasm, params.PageLimit, program.version, debugMode, &unlimitedGas) + info, asmMap, err := activateProgramInternal(statedb, common.Address{}, codeHash, wasm, progParams.PageLimit, program.version, zeroArbosVersion, debugMode, &zeroGas) if err != nil { log.Error("failed to reactivate program while rebuilding wasm store", "expected moduleHash", moduleHash, "err", err) return fmt.Errorf("failed to reactivate program while rebuilding wasm store: %w", err) diff --git a/cmd/chaininfo/arbitrum_chain_info.json b/cmd/chaininfo/arbitrum_chain_info.json index 524433a7b5..f862c6dfbf 100644 --- a/cmd/chaininfo/arbitrum_chain_info.json +++ b/cmd/chaininfo/arbitrum_chain_info.json @@ -164,7 +164,7 @@ "EnableArbOS": true, "AllowDebugPrecompiles": true, "DataAvailabilityCommittee": false, - "InitialArbOSVersion": 31, + "InitialArbOSVersion": 32, "InitialChainOwner": "0x0000000000000000000000000000000000000000", "GenesisBlockNum": 0 } @@ -196,7 +196,7 @@ "EnableArbOS": true, "AllowDebugPrecompiles": true, "DataAvailabilityCommittee": true, - "InitialArbOSVersion": 31, + "InitialArbOSVersion": 32, "InitialChainOwner": "0x0000000000000000000000000000000000000000", "GenesisBlockNum": 0 } diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index fc59f2d231..07c74cb802 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -354,12 +354,12 @@ func validateBlockChain(blockChain *core.BlockChain, chainConfig *params.ChainCo } // Make sure we don't allow accidentally downgrading ArbOS if chainConfig.DebugMode() { - if currentArbosState.ArbOSVersion() > currentArbosState.MaxDebugArbosVersionSupported() { - return fmt.Errorf("attempted to launch node in debug mode with ArbOS version %v on ArbOS state with version %v", currentArbosState.MaxDebugArbosVersionSupported(), currentArbosState.ArbOSVersion()) + if currentArbosState.ArbOSVersion() > arbosState.MaxDebugArbosVersionSupported { + return fmt.Errorf("attempted to launch node in debug mode with ArbOS version %v on ArbOS state with version %v", arbosState.MaxDebugArbosVersionSupported, currentArbosState.ArbOSVersion()) } } else { - if currentArbosState.ArbOSVersion() > currentArbosState.MaxArbosVersionSupported() { - return fmt.Errorf("attempted to launch node with ArbOS version %v on ArbOS state with version %v", currentArbosState.MaxArbosVersionSupported(), currentArbosState.ArbOSVersion()) + if currentArbosState.ArbOSVersion() > arbosState.MaxArbosVersionSupported { + return fmt.Errorf("attempted to launch node with ArbOS version %v on ArbOS state with version %v", arbosState.MaxArbosVersionSupported, currentArbosState.ArbOSVersion()) } } diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index a13a92fdc1..1078f44808 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -472,6 +472,10 @@ func mainImpl() int { if nodeConfig.BlocksReExecutor.Enable && l2BlockChain != nil { blocksReExecutor = blocksreexecutor.New(&nodeConfig.BlocksReExecutor, l2BlockChain, fatalErrChan) if nodeConfig.Init.ThenQuit { + if err := gethexec.PopulateStylusTargetCache(&nodeConfig.Execution.StylusTarget); err != nil { + log.Error("error populating stylus target cache", "err", err) + return 1 + } success := make(chan struct{}) blocksReExecutor.Start(ctx, success) deferFuncs = append(deferFuncs, func() { blocksReExecutor.StopAndWait() }) diff --git a/contracts b/contracts index 23fc796282..7396313311 160000 --- a/contracts +++ b/contracts @@ -1 +1 @@ -Subproject commit 23fc79628292aa5d604d449fed48937ae7faeb2f +Subproject commit 7396313311ab17cb30e2eef27cccf96f0a9e8f7f diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index 8594d5867d..8d6484e3c9 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -150,7 +150,7 @@ func (s *ExecutionEngine) MarkFeedStart(to arbutil.MessageIndex) { } } -func populateStylusTargetCache(targetConfig *StylusTargetConfig) error { +func PopulateStylusTargetCache(targetConfig *StylusTargetConfig) error { localTarget := rawdb.LocalTarget() targets := targetConfig.WasmTargets() var nativeSet bool @@ -186,7 +186,7 @@ func (s *ExecutionEngine) Initialize(rustCacheSize uint32, targetConfig *StylusT if rustCacheSize != 0 { programs.ResizeWasmLruCache(rustCacheSize) } - if err := populateStylusTargetCache(targetConfig); err != nil { + if err := PopulateStylusTargetCache(targetConfig); err != nil { return fmt.Errorf("error populating stylus target cache: %w", err) } return nil diff --git a/execution/gethexec/wasmstorerebuilder.go b/execution/gethexec/wasmstorerebuilder.go index 698ba3ec8a..e3eb8e9268 100644 --- a/execution/gethexec/wasmstorerebuilder.go +++ b/execution/gethexec/wasmstorerebuilder.go @@ -63,7 +63,7 @@ func RebuildWasmStore(ctx context.Context, wasmStore ethdb.KeyValueStore, chainD var err error var stateDb *state.StateDB - if err := populateStylusTargetCache(targetConfig); err != nil { + if err := PopulateStylusTargetCache(targetConfig); err != nil { return fmt.Errorf("error populating stylus target cache: %w", err) } diff --git a/go-ethereum b/go-ethereum index 9ca65b8610..17cd001675 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 9ca65b86101b4c87df188923d6eeff4db992520f +Subproject commit 17cd00167543a5a2b0b083e32820051100154c2f diff --git a/precompiles/ArbWasm.go b/precompiles/ArbWasm.go index 9f42cacb5a..bc24c8a6e8 100644 --- a/precompiles/ArbWasm.go +++ b/precompiles/ArbWasm.go @@ -5,6 +5,8 @@ package precompiles import ( "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/vm" + gethparams "github.com/ethereum/go-ethereum/params" "github.com/offchainlabs/nitro/arbos/programs" "github.com/offchainlabs/nitro/arbos/util" "github.com/offchainlabs/nitro/util/arbmath" @@ -32,12 +34,13 @@ func (con ArbWasm) ActivateProgram(c ctx, evm mech, value huge, program addr) (u debug := evm.ChainConfig().DebugMode() runMode := c.txProcessor.RunMode() programs := c.State.Programs() + arbosVersion := c.State.ArbOSVersion() // charge a fixed cost up front to begin activation if err := c.Burn(1659168); err != nil { return 0, nil, err } - version, codeHash, moduleHash, dataFee, takeAllGas, err := programs.ActivateProgram(evm, program, runMode, debug) + version, codeHash, moduleHash, dataFee, takeAllGas, err := programs.ActivateProgram(evm, program, arbosVersion, runMode, debug) if takeAllGas { _ = c.BurnOut() } @@ -133,6 +136,9 @@ func (con ArbWasm) MinInitGas(c ctx, _ mech) (uint64, uint64, error) { params, err := c.State.Programs().Params() init := uint64(params.MinInitGas) * programs.MinInitGasUnits cached := uint64(params.MinCachedInitGas) * programs.MinCachedGasUnits + if c.State.ArbOSVersion() < gethparams.ArbosVersion_StylusChargingFixes { + return 0, 0, vm.ErrExecutionReverted + } return init, cached, err } diff --git a/system_tests/block_validator_test.go b/system_tests/block_validator_test.go index cca5572f4e..9125c3921e 100644 --- a/system_tests/block_validator_test.go +++ b/system_tests/block_validator_test.go @@ -284,6 +284,20 @@ func TestBlockValidatorSimpleOnchain(t *testing.T) { testBlockValidatorSimple(t, opts) } +func TestBlockValidatorSimpleJITOnchainWithPublishedMachine(t *testing.T) { + cr, err := github.LatestConsensusRelease(context.Background()) + Require(t, err) + machPath := populateMachineDir(t, cr) + opts := Options{ + dasModeString: "onchain", + workloadLoops: 1, + workload: ethSend, + arbitrator: false, + wasmRootDir: machPath, + } + testBlockValidatorSimple(t, opts) +} + func TestBlockValidatorSimpleOnchainWithPublishedMachine(t *testing.T) { cr, err := github.LatestConsensusRelease(context.Background()) Require(t, err) diff --git a/validator/server_arb/machine.go b/validator/server_arb/machine.go index e77e071d70..adca9695e2 100644 --- a/validator/server_arb/machine.go +++ b/validator/server_arb/machine.go @@ -92,7 +92,7 @@ func LoadSimpleMachine(wasm string, libraries []string, debugChain bool) (*Arbit cWasm := C.CString(wasm) cLibraries := CreateCStringList(libraries) debug := usize(arbmath.BoolToUint32(debugChain)) - mach := C.arbitrator_load_machine(cWasm, cLibraries, C.long(len(libraries)), debug, false) + mach := C.arbitrator_load_machine(cWasm, cLibraries, C.long(len(libraries)), debug) C.free(unsafe.Pointer(cWasm)) FreeCStringList(cLibraries, len(libraries)) if mach == nil { From 4fa04c0636da911cc702f91b63a9ba2dc7bc1963 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Tue, 24 Sep 2024 09:41:12 -0500 Subject: [PATCH 254/489] Re-enable G115 linter --- .golangci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index 8b10d7116c..0594670137 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -44,7 +44,6 @@ linters-settings: gosec: excludes: - G404 # checks that random numbers are securely generated - - G115 govet: enable-all: true From 556b2dfb7785479ca018de57b493ff335cdb5208 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Tue, 24 Sep 2024 20:14:43 +0530 Subject: [PATCH 255/489] fix rust cache --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 453bd00617..ef9ea9e721 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,6 +92,7 @@ jobs: arbitrator/wasm-libraries/target/ arbitrator/wasm-libraries/soft-float/ target/etc/initial-machine-cache/ + /home/runner/.rustup/toolchains/ key: ${{ runner.os }}-cargo-${{ steps.install-rust.outputs.rustc_hash }}-min-${{ hashFiles('arbitrator/Cargo.lock') }}-${{ matrix.test-mode }} restore-keys: ${{ runner.os }}-cargo-${{ steps.install-rust.outputs.rustc_hash }}- From eb9af0d7f4e30c24d887e481d576986d03d8730a Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Tue, 24 Sep 2024 20:32:17 +0530 Subject: [PATCH 256/489] Trigger Build From d81324daee7340281ee61c0f5db5231ec1d88eea Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Tue, 24 Sep 2024 10:12:36 -0500 Subject: [PATCH 257/489] Add support for consensus v32 to Docker --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 37226c397c..9138ed30ad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -218,6 +218,7 @@ COPY ./scripts/download-machine.sh . #RUN ./download-machine.sh consensus-v20 0x8b104a2e80ac6165dc58b9048de12f301d70b02a0ab51396c22b4b4b802a16a4 RUN ./download-machine.sh consensus-v30 0xb0de9cb89e4d944ae6023a3b62276e54804c242fd8c4c2d8e6cc4450f5fa8b1b && true RUN ./download-machine.sh consensus-v31 0x260f5fa5c3176a856893642e149cf128b5a8de9f828afec8d11184415dd8dc69 +RUN ./download-machine.sh consensus-v32 0x184884e1eb9fefdc158f6c8ac912bb183bf3cf83f0090317e0bc4ac5860baa39 FROM golang:1.21.10-bookworm AS node-builder WORKDIR /workspace From 67a66f50bce25619703ea2a93e7c499694dd6324 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Tue, 24 Sep 2024 20:59:46 +0530 Subject: [PATCH 258/489] add back test --- .github/workflows/ci.yml | 150 +++++++++++++++++++-------------------- 1 file changed, 75 insertions(+), 75 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ef9ea9e721..b439fe4aec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -140,81 +140,81 @@ jobs: echo "GOGC=80" >> "$GITHUB_ENV" echo "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" >> "$GITHUB_ENV" -# - name: run tests without race detection and path state scheme -# if: matrix.test-mode == 'defaults' -# env: -# TEST_STATE_SCHEME: path -# run: | -# packages=`go list ./...` -# for package in $packages; do -# echo running tests for $package -# if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -timeout 20m -tags=cionly > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then -# exit 1 -# fi -# done - -# - name: run tests without race detection and hash state scheme -# if: matrix.test-mode == 'defaults' -# env: -# TEST_STATE_SCHEME: hash -# run: | -# packages=`go list ./...` -# for package in $packages; do -# echo running tests for $package -# if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -timeout 20m -tags=cionly; then -# exit 1 -# fi -# done - -# - name: run tests with race detection and hash state scheme -# if: matrix.test-mode == 'race' -# env: -# TEST_STATE_SCHEME: hash -# run: | -# packages=`go list ./...` -# for package in $packages; do -# echo running tests for $package -# if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -race -timeout 30m; then -# exit 1 -# fi -# done - -# - name: run redis tests -# if: matrix.test-mode == 'defaults' -# run: TEST_REDIS=redis://localhost:6379/0 gotestsum --format short-verbose -- -p 1 -run TestRedis ./arbnode/... ./system_tests/... -coverprofile=coverage-redis.txt -covermode=atomic -coverpkg=./... - -# - name: run challenge tests -# if: matrix.test-mode == 'challenge' -# run: | -# packages=`go list ./...` -# for package in $packages; do -# echo running tests for $package -# if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -tags=challengetest -run=TestChallenge > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then -# exit 1 -# fi -# done - -# - name: run stylus tests -# if: matrix.test-mode == 'stylus' -# run: | -# packages=`go list ./...` -# for package in $packages; do -# echo running tests for $package -# if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -timeout 60m -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -tags=stylustest -run="TestProgramArbitrator" > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then -# exit 1 -# fi -# done - -# - name: run long stylus tests -# if: matrix.test-mode == 'long' -# run: | -# packages=`go list ./...` -# for package in $packages; do -# echo running tests for $package -# if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -timeout 60m -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -tags=stylustest -run="TestProgramLong" > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then -# exit 1 -# fi -# done + - name: run tests without race detection and path state scheme + if: matrix.test-mode == 'defaults' + env: + TEST_STATE_SCHEME: path + run: | + packages=`go list ./...` + for package in $packages; do + echo running tests for $package + if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -timeout 20m -tags=cionly > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then + exit 1 + fi + done + + - name: run tests without race detection and hash state scheme + if: matrix.test-mode == 'defaults' + env: + TEST_STATE_SCHEME: hash + run: | + packages=`go list ./...` + for package in $packages; do + echo running tests for $package + if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -timeout 20m -tags=cionly; then + exit 1 + fi + done + + - name: run tests with race detection and hash state scheme + if: matrix.test-mode == 'race' + env: + TEST_STATE_SCHEME: hash + run: | + packages=`go list ./...` + for package in $packages; do + echo running tests for $package + if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -race -timeout 30m; then + exit 1 + fi + done + + - name: run redis tests + if: matrix.test-mode == 'defaults' + run: TEST_REDIS=redis://localhost:6379/0 gotestsum --format short-verbose -- -p 1 -run TestRedis ./arbnode/... ./system_tests/... -coverprofile=coverage-redis.txt -covermode=atomic -coverpkg=./... + + - name: run challenge tests + if: matrix.test-mode == 'challenge' + run: | + packages=`go list ./...` + for package in $packages; do + echo running tests for $package + if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -tags=challengetest -run=TestChallenge > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then + exit 1 + fi + done + + - name: run stylus tests + if: matrix.test-mode == 'stylus' + run: | + packages=`go list ./...` + for package in $packages; do + echo running tests for $package + if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -timeout 60m -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -tags=stylustest -run="TestProgramArbitrator" > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then + exit 1 + fi + done + + - name: run long stylus tests + if: matrix.test-mode == 'long' + run: | + packages=`go list ./...` + for package in $packages; do + echo running tests for $package + if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -timeout 60m -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -tags=stylustest -run="TestProgramLong" > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then + exit 1 + fi + done - name: Archive detailed run log uses: actions/upload-artifact@v3 From 3fb2b1914cdec259ab4127490de380ac50ced3ff Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Tue, 24 Sep 2024 21:29:59 +0530 Subject: [PATCH 259/489] Trigger Build From 01b54da7d01ed495d2160acb635c8faff5aff6a4 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Tue, 24 Sep 2024 15:16:33 -0500 Subject: [PATCH 260/489] Update nitro-testnode pin --- nitro-testnode | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nitro-testnode b/nitro-testnode index f328006579..20b5d14a0d 160000 --- a/nitro-testnode +++ b/nitro-testnode @@ -1 +1 @@ -Subproject commit f328006579cbefe22c6c57de3d6b86397fde4438 +Subproject commit 20b5d14a0dd6d3ec1132b86a9522559311f1bb78 From affee2eba34813a5a4c447744e58d3d66a3fe31a Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Wed, 25 Sep 2024 04:14:21 +0800 Subject: [PATCH 261/489] refactor: make timeout configurable for jit execution --- validator/server_jit/jit_machine.go | 22 ++++++++++++---------- validator/server_jit/machine_loader.go | 16 +++++++++------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/validator/server_jit/jit_machine.go b/validator/server_jit/jit_machine.go index 06c451bda1..77112f0635 100644 --- a/validator/server_jit/jit_machine.go +++ b/validator/server_jit/jit_machine.go @@ -26,13 +26,14 @@ import ( var jitWasmMemoryUsage = metrics.NewRegisteredHistogram("jit/wasm/memoryusage", nil, metrics.NewBoundedHistogramSample()) type JitMachine struct { - binary string - process *exec.Cmd - stdin io.WriteCloser - wasmMemoryUsageLimit int + binary string + process *exec.Cmd + stdin io.WriteCloser + wasmMemoryUsageLimit int + maxExecutionTimeInSeconds int } -func createJitMachine(jitBinary string, binaryPath string, cranelift bool, wasmMemoryUsageLimit int, moduleRoot common.Hash, fatalErrChan chan error) (*JitMachine, error) { +func createJitMachine(jitBinary string, binaryPath string, cranelift bool, wasmMemoryUsageLimit int, maxExecutionTimeInSeconds int, moduleRoot common.Hash, fatalErrChan chan error) (*JitMachine, error) { invocation := []string{"--binary", binaryPath, "--forks"} if cranelift { invocation = append(invocation, "--cranelift") @@ -51,10 +52,11 @@ func createJitMachine(jitBinary string, binaryPath string, cranelift bool, wasmM }() machine := &JitMachine{ - binary: binaryPath, - process: process, - stdin: stdin, - wasmMemoryUsageLimit: wasmMemoryUsageLimit, + binary: binaryPath, + process: process, + stdin: stdin, + wasmMemoryUsageLimit: wasmMemoryUsageLimit, + maxExecutionTimeInSeconds: maxExecutionTimeInSeconds, } return machine, nil } @@ -73,7 +75,7 @@ func (machine *JitMachine) prove( defer cancel() // ensure our cleanup functions run when we're done state := validator.GoGlobalState{} - timeout := time.Now().Add(60 * time.Second) + timeout := time.Now().Add(time.Duration(machine.maxExecutionTimeInSeconds) * time.Second) tcp, err := net.ListenTCP("tcp4", &net.TCPAddr{ IP: []byte{127, 0, 0, 1}, }) diff --git a/validator/server_jit/machine_loader.go b/validator/server_jit/machine_loader.go index cfa475370c..342f5c37de 100644 --- a/validator/server_jit/machine_loader.go +++ b/validator/server_jit/machine_loader.go @@ -13,15 +13,17 @@ import ( ) type JitMachineConfig struct { - ProverBinPath string - JitCranelift bool - WasmMemoryUsageLimit int + ProverBinPath string + JitCranelift bool + WasmMemoryUsageLimit int + MaxExecutionTimeInSeconds int } var DefaultJitMachineConfig = JitMachineConfig{ - JitCranelift: true, - ProverBinPath: "replay.wasm", - WasmMemoryUsageLimit: 4294967296, + JitCranelift: true, + ProverBinPath: "replay.wasm", + WasmMemoryUsageLimit: 4294967296, + MaxExecutionTimeInSeconds: 60, } func getJitPath() (string, error) { @@ -59,7 +61,7 @@ func NewJitMachineLoader(config *JitMachineConfig, locator *server_common.Machin } createMachineThreadFunc := func(ctx context.Context, moduleRoot common.Hash) (*JitMachine, error) { binPath := filepath.Join(locator.GetMachinePath(moduleRoot), config.ProverBinPath) - return createJitMachine(jitPath, binPath, config.JitCranelift, config.WasmMemoryUsageLimit, moduleRoot, fatalErrChan) + return createJitMachine(jitPath, binPath, config.JitCranelift, config.WasmMemoryUsageLimit, config.MaxExecutionTimeInSeconds, moduleRoot, fatalErrChan) } return &JitMachineLoader{ MachineLoader: *server_common.NewMachineLoader[JitMachine](locator, createMachineThreadFunc), From c2d720cea420b03e77903ed0103e2e5cba45d86f Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Wed, 25 Sep 2024 04:51:11 +0800 Subject: [PATCH 262/489] refactor: move MaxExecutionTime to JitSpawner config --- validator/server_jit/jit_machine.go | 24 ++++++++++++------------ validator/server_jit/machine_loader.go | 19 +++++++++---------- validator/server_jit/spawner.go | 14 +++++++++----- 3 files changed, 30 insertions(+), 27 deletions(-) diff --git a/validator/server_jit/jit_machine.go b/validator/server_jit/jit_machine.go index 77112f0635..2bea75fbe9 100644 --- a/validator/server_jit/jit_machine.go +++ b/validator/server_jit/jit_machine.go @@ -26,14 +26,14 @@ import ( var jitWasmMemoryUsage = metrics.NewRegisteredHistogram("jit/wasm/memoryusage", nil, metrics.NewBoundedHistogramSample()) type JitMachine struct { - binary string - process *exec.Cmd - stdin io.WriteCloser - wasmMemoryUsageLimit int - maxExecutionTimeInSeconds int + binary string + process *exec.Cmd + stdin io.WriteCloser + wasmMemoryUsageLimit int + maxExecutionTime time.Duration } -func createJitMachine(jitBinary string, binaryPath string, cranelift bool, wasmMemoryUsageLimit int, maxExecutionTimeInSeconds int, moduleRoot common.Hash, fatalErrChan chan error) (*JitMachine, error) { +func createJitMachine(jitBinary string, binaryPath string, cranelift bool, wasmMemoryUsageLimit int, maxExecutionTime time.Duration, moduleRoot common.Hash, fatalErrChan chan error) (*JitMachine, error) { invocation := []string{"--binary", binaryPath, "--forks"} if cranelift { invocation = append(invocation, "--cranelift") @@ -52,11 +52,11 @@ func createJitMachine(jitBinary string, binaryPath string, cranelift bool, wasmM }() machine := &JitMachine{ - binary: binaryPath, - process: process, - stdin: stdin, - wasmMemoryUsageLimit: wasmMemoryUsageLimit, - maxExecutionTimeInSeconds: maxExecutionTimeInSeconds, + binary: binaryPath, + process: process, + stdin: stdin, + wasmMemoryUsageLimit: wasmMemoryUsageLimit, + maxExecutionTime: maxExecutionTime, } return machine, nil } @@ -75,7 +75,7 @@ func (machine *JitMachine) prove( defer cancel() // ensure our cleanup functions run when we're done state := validator.GoGlobalState{} - timeout := time.Now().Add(time.Duration(machine.maxExecutionTimeInSeconds) * time.Second) + timeout := time.Now().Add(machine.maxExecutionTime) tcp, err := net.ListenTCP("tcp4", &net.TCPAddr{ IP: []byte{127, 0, 0, 1}, }) diff --git a/validator/server_jit/machine_loader.go b/validator/server_jit/machine_loader.go index 342f5c37de..b2393120d5 100644 --- a/validator/server_jit/machine_loader.go +++ b/validator/server_jit/machine_loader.go @@ -7,23 +7,22 @@ import ( "path/filepath" "runtime" "strings" + "time" "github.com/ethereum/go-ethereum/common" "github.com/offchainlabs/nitro/validator/server_common" ) type JitMachineConfig struct { - ProverBinPath string - JitCranelift bool - WasmMemoryUsageLimit int - MaxExecutionTimeInSeconds int + ProverBinPath string + JitCranelift bool + WasmMemoryUsageLimit int } var DefaultJitMachineConfig = JitMachineConfig{ - JitCranelift: true, - ProverBinPath: "replay.wasm", - WasmMemoryUsageLimit: 4294967296, - MaxExecutionTimeInSeconds: 60, + JitCranelift: true, + ProverBinPath: "replay.wasm", + WasmMemoryUsageLimit: 4294967296, } func getJitPath() (string, error) { @@ -54,14 +53,14 @@ type JitMachineLoader struct { stopped bool } -func NewJitMachineLoader(config *JitMachineConfig, locator *server_common.MachineLocator, fatalErrChan chan error) (*JitMachineLoader, error) { +func NewJitMachineLoader(config *JitMachineConfig, locator *server_common.MachineLocator, MaxExecutionTime time.Duration, fatalErrChan chan error) (*JitMachineLoader, error) { jitPath, err := getJitPath() if err != nil { return nil, err } createMachineThreadFunc := func(ctx context.Context, moduleRoot common.Hash) (*JitMachine, error) { binPath := filepath.Join(locator.GetMachinePath(moduleRoot), config.ProverBinPath) - return createJitMachine(jitPath, binPath, config.JitCranelift, config.WasmMemoryUsageLimit, config.MaxExecutionTimeInSeconds, moduleRoot, fatalErrChan) + return createJitMachine(jitPath, binPath, config.JitCranelift, config.WasmMemoryUsageLimit, MaxExecutionTime, moduleRoot, fatalErrChan) } return &JitMachineLoader{ MachineLoader: *server_common.NewMachineLoader[JitMachine](locator, createMachineThreadFunc), diff --git a/validator/server_jit/spawner.go b/validator/server_jit/spawner.go index d77317d218..094943fd1d 100644 --- a/validator/server_jit/spawner.go +++ b/validator/server_jit/spawner.go @@ -3,10 +3,10 @@ package server_jit import ( "context" "fmt" + flag "github.com/spf13/pflag" "runtime" "sync/atomic" - - flag "github.com/spf13/pflag" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" @@ -18,8 +18,9 @@ import ( ) type JitSpawnerConfig struct { - Workers int `koanf:"workers" reload:"hot"` - Cranelift bool `koanf:"cranelift"` + Workers int `koanf:"workers" reload:"hot"` + Cranelift bool `koanf:"cranelift"` + MaxExecutionTime time.Duration `koanf:"max-execution-time" reload:"hot"` // TODO: change WasmMemoryUsageLimit to a string and use resourcemanager.ParseMemLimit WasmMemoryUsageLimit int `koanf:"wasm-memory-usage-limit"` @@ -30,6 +31,7 @@ type JitSpawnerConfigFecher func() *JitSpawnerConfig var DefaultJitSpawnerConfig = JitSpawnerConfig{ Workers: 0, Cranelift: true, + MaxExecutionTime: time.Minute * 10, WasmMemoryUsageLimit: 4294967296, // 2^32 WASM memeory limit } @@ -37,6 +39,7 @@ func JitSpawnerConfigAddOptions(prefix string, f *flag.FlagSet) { f.Int(prefix+".workers", DefaultJitSpawnerConfig.Workers, "number of concurrent validation threads") f.Bool(prefix+".cranelift", DefaultJitSpawnerConfig.Cranelift, "use Cranelift instead of LLVM when validating blocks using the jit-accelerated block validator") f.Int(prefix+".wasm-memory-usage-limit", DefaultJitSpawnerConfig.WasmMemoryUsageLimit, "if memory used by a jit wasm exceeds this limit, a warning is logged") + f.Duration(prefix+"max_execution_time", DefaultJitSpawnerConfig.MaxExecutionTime, "if execution time used by a jit wasm exceeds this limit, a rpc error is returned") } type JitSpawner struct { @@ -52,7 +55,8 @@ func NewJitSpawner(locator *server_common.MachineLocator, config JitSpawnerConfi machineConfig := DefaultJitMachineConfig machineConfig.JitCranelift = config().Cranelift machineConfig.WasmMemoryUsageLimit = config().WasmMemoryUsageLimit - loader, err := NewJitMachineLoader(&machineConfig, locator, fatalErrChan) + maxExecutionTime := config().MaxExecutionTime + loader, err := NewJitMachineLoader(&machineConfig, locator, maxExecutionTime, fatalErrChan) if err != nil { return nil, err } From 43645ce69c95fffc37decef9c4bccbcb46d431ef Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Wed, 25 Sep 2024 04:59:26 +0800 Subject: [PATCH 263/489] chore: rename some variable --- validator/server_jit/machine_loader.go | 4 ++-- validator/server_jit/spawner.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/validator/server_jit/machine_loader.go b/validator/server_jit/machine_loader.go index b2393120d5..3d8b01367f 100644 --- a/validator/server_jit/machine_loader.go +++ b/validator/server_jit/machine_loader.go @@ -53,14 +53,14 @@ type JitMachineLoader struct { stopped bool } -func NewJitMachineLoader(config *JitMachineConfig, locator *server_common.MachineLocator, MaxExecutionTime time.Duration, fatalErrChan chan error) (*JitMachineLoader, error) { +func NewJitMachineLoader(config *JitMachineConfig, locator *server_common.MachineLocator, maxExecutionTime time.Duration, fatalErrChan chan error) (*JitMachineLoader, error) { jitPath, err := getJitPath() if err != nil { return nil, err } createMachineThreadFunc := func(ctx context.Context, moduleRoot common.Hash) (*JitMachine, error) { binPath := filepath.Join(locator.GetMachinePath(moduleRoot), config.ProverBinPath) - return createJitMachine(jitPath, binPath, config.JitCranelift, config.WasmMemoryUsageLimit, MaxExecutionTime, moduleRoot, fatalErrChan) + return createJitMachine(jitPath, binPath, config.JitCranelift, config.WasmMemoryUsageLimit, maxExecutionTime, moduleRoot, fatalErrChan) } return &JitMachineLoader{ MachineLoader: *server_common.NewMachineLoader[JitMachine](locator, createMachineThreadFunc), diff --git a/validator/server_jit/spawner.go b/validator/server_jit/spawner.go index 094943fd1d..29acff246d 100644 --- a/validator/server_jit/spawner.go +++ b/validator/server_jit/spawner.go @@ -39,7 +39,7 @@ func JitSpawnerConfigAddOptions(prefix string, f *flag.FlagSet) { f.Int(prefix+".workers", DefaultJitSpawnerConfig.Workers, "number of concurrent validation threads") f.Bool(prefix+".cranelift", DefaultJitSpawnerConfig.Cranelift, "use Cranelift instead of LLVM when validating blocks using the jit-accelerated block validator") f.Int(prefix+".wasm-memory-usage-limit", DefaultJitSpawnerConfig.WasmMemoryUsageLimit, "if memory used by a jit wasm exceeds this limit, a warning is logged") - f.Duration(prefix+"max_execution_time", DefaultJitSpawnerConfig.MaxExecutionTime, "if execution time used by a jit wasm exceeds this limit, a rpc error is returned") + f.Duration(prefix+"max-execution-time", DefaultJitSpawnerConfig.MaxExecutionTime, "if execution time used by a jit wasm exceeds this limit, a rpc error is returned") } type JitSpawner struct { From 3ea5870c6127fa2db36a83ce5eccc4f7f0786219 Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Wed, 25 Sep 2024 05:03:52 +0800 Subject: [PATCH 264/489] fix: add . for max-execution-time --- validator/server_jit/spawner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/server_jit/spawner.go b/validator/server_jit/spawner.go index 29acff246d..f30b6e181a 100644 --- a/validator/server_jit/spawner.go +++ b/validator/server_jit/spawner.go @@ -39,7 +39,7 @@ func JitSpawnerConfigAddOptions(prefix string, f *flag.FlagSet) { f.Int(prefix+".workers", DefaultJitSpawnerConfig.Workers, "number of concurrent validation threads") f.Bool(prefix+".cranelift", DefaultJitSpawnerConfig.Cranelift, "use Cranelift instead of LLVM when validating blocks using the jit-accelerated block validator") f.Int(prefix+".wasm-memory-usage-limit", DefaultJitSpawnerConfig.WasmMemoryUsageLimit, "if memory used by a jit wasm exceeds this limit, a warning is logged") - f.Duration(prefix+"max-execution-time", DefaultJitSpawnerConfig.MaxExecutionTime, "if execution time used by a jit wasm exceeds this limit, a rpc error is returned") + f.Duration(prefix+".max-execution-time", DefaultJitSpawnerConfig.MaxExecutionTime, "if execution time used by a jit wasm exceeds this limit, a rpc error is returned") } type JitSpawner struct { From 52cac0574f46144f910b4ec0b563e0af5927bcfd Mon Sep 17 00:00:00 2001 From: Joshua Colvin Date: Tue, 24 Sep 2024 15:10:48 -0700 Subject: [PATCH 265/489] Fix typo in license --- LICENSE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE.md b/LICENSE.md index ea9a53da75..25768b3010 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -22,7 +22,7 @@ Additional Use Grant: You may use the Licensed Work in a production environment Expansion Program Term of Use](https://docs.arbitrum.foundation/assets/files/Arbitrum%20Expansion%20Program%20Jan182024-4f08b0c2cb476a55dc153380fa3e64b0.pdf). For purposes of this Additional Use Grant, the "Covered Arbitrum Chains" are (a) Arbitrum One (chainid:42161), Arbitrum Nova (chainid:42170), - rbitrum Rinkeby testnet/Rinkarby (chainid:421611),Arbitrum Nitro + Arbitrum Rinkeby testnet/Rinkarby (chainid:421611),Arbitrum Nitro Goerli testnet (chainid:421613), and Arbitrum Sepolia Testnet (chainid:421614); (b) any future blockchains authorized to be designated as Covered Arbitrum Chains by the decentralized autonomous From 68b6ba6521ee9792c13825540ae00e50ee0d475b Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Tue, 24 Sep 2024 17:29:32 -0500 Subject: [PATCH 266/489] Update nitro-testnode pin again --- nitro-testnode | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nitro-testnode b/nitro-testnode index 20b5d14a0d..72141dd495 160000 --- a/nitro-testnode +++ b/nitro-testnode @@ -1 +1 @@ -Subproject commit 20b5d14a0dd6d3ec1132b86a9522559311f1bb78 +Subproject commit 72141dd495ad965aa2a23723ea3e755037903ad7 From 6765ba608286c5c6473c3c0b7cb3c85759befb93 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Wed, 25 Sep 2024 15:39:26 -0300 Subject: [PATCH 267/489] Remove brotli build dir in make clean --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 0a71d64f12..c3cf1a5144 100644 --- a/Makefile +++ b/Makefile @@ -283,6 +283,7 @@ clean: rm -f arbitrator/wasm-libraries/soft-float/SoftFloat/build/Wasm-Clang/*.a rm -f arbitrator/wasm-libraries/forward/*.wat rm -rf arbitrator/stylus/tests/*/target/ arbitrator/stylus/tests/*/*.wasm + rm -rf brotli/buildfiles @rm -rf contracts/build contracts/cache solgen/go/ @rm -f .make/* From 12253bd6f4c9308707c95175188714f74ca4f3ae Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Wed, 25 Sep 2024 17:30:01 -0300 Subject: [PATCH 268/489] Fix CaptureHostIO when slices are bigger than 2^16 Support slices bigger than 2^16 when capturing a HostIO for tracing. This could be reproduced with the test case added in the commit. --- arbitrator/arbutil/src/evm/req.rs | 7 +++--- arbitrator/stylus/tests/write-result-len.wat | 24 ++++++++++++++++++++ arbos/programs/api.go | 6 ++--- system_tests/stylus_trace_test.go | 15 ++++++++++++ 4 files changed, 46 insertions(+), 6 deletions(-) create mode 100644 arbitrator/stylus/tests/write-result-len.wat diff --git a/arbitrator/arbutil/src/evm/req.rs b/arbitrator/arbutil/src/evm/req.rs index 1cfceda6b7..0304f2d378 100644 --- a/arbitrator/arbutil/src/evm/req.rs +++ b/arbitrator/arbutil/src/evm/req.rs @@ -298,9 +298,10 @@ impl> EvmApi for EvmApiRequestor { let mut request = Vec::with_capacity(2 * 8 + 3 * 2 + name.len() + args.len() + outs.len()); request.extend(start_ink.to_be_bytes()); request.extend(end_ink.to_be_bytes()); - request.extend((name.len() as u16).to_be_bytes()); - request.extend((args.len() as u16).to_be_bytes()); - request.extend((outs.len() as u16).to_be_bytes()); + // u32 is enough to represent the slices lengths because the WASM environment runs in 32 bits. + request.extend((name.len() as u32).to_be_bytes()); + request.extend((args.len() as u32).to_be_bytes()); + request.extend((outs.len() as u32).to_be_bytes()); request.extend(name.as_bytes()); request.extend(args); request.extend(outs); diff --git a/arbitrator/stylus/tests/write-result-len.wat b/arbitrator/stylus/tests/write-result-len.wat new file mode 100644 index 0000000000..4c9ad35087 --- /dev/null +++ b/arbitrator/stylus/tests/write-result-len.wat @@ -0,0 +1,24 @@ +;; Copyright 2024, Offchain Labs, Inc. +;; For license information, see https://github.com/nitro/blob/master/LICENSE + +(module + (import "vm_hooks" "read_args" (func $read_args (param i32))) + (import "vm_hooks" "write_result" (func $write_result (param i32 i32))) + (memory (export "memory") 2 2) + (func $main (export "user_entrypoint") (param $args_len i32) (result i32) + (local $len i32) + + ;; write args to 0x0 + (call $read_args (i32.const 0)) + + ;; treat first 4 bytes as size to write + (i32.load (i32.const 0)) + local.set $len + + ;; call write + (call $write_result (i32.const 0) (local.get $len)) + + ;; return success + i32.const 0 + ) +) diff --git a/arbos/programs/api.go b/arbos/programs/api.go index 504289322f..3e59031b2d 100644 --- a/arbos/programs/api.go +++ b/arbos/programs/api.go @@ -400,9 +400,9 @@ func newApiClosures( } startInk := takeU64() endInk := takeU64() - nameLen := takeU16() - argsLen := takeU16() - outsLen := takeU16() + nameLen := takeU32() + argsLen := takeU32() + outsLen := takeU32() name := string(takeFixed(int(nameLen))) args := takeFixed(int(argsLen)) outs := takeFixed(int(outsLen)) diff --git a/system_tests/stylus_trace_test.go b/system_tests/stylus_trace_test.go index 5c4463d9f7..52039df460 100644 --- a/system_tests/stylus_trace_test.go +++ b/system_tests/stylus_trace_test.go @@ -6,6 +6,7 @@ package arbtest import ( "bytes" "encoding/binary" + "math" "math/big" "testing" @@ -478,3 +479,17 @@ func TestStylusOpcodeTraceEquivalence(t *testing.T) { checkOpcode(t, wasmResult, 12, vm.RETURN, offset, returnLen) checkOpcode(t, evmResult, 5078, vm.RETURN, offset, returnLen) } + +func TestStylusHugeWriteResultTrace(t *testing.T) { + const jit = false + builder, auth, cleanup := setupProgramTest(t, jit) + ctx := builder.ctx + l2client := builder.L2.Client + defer cleanup() + + program := deployWasm(t, ctx, auth, l2client, watFile("write-result-len")) + const returnLen = math.MaxUint16 + 1 + args := binary.LittleEndian.AppendUint32(nil, returnLen) + result := sendAndTraceTransaction(t, builder, program, nil, args) + checkOpcode(t, result, 3, vm.RETURN, nil, intToBe32(returnLen)) +} From 910666f7bd2db9575751ead1f6a9136da34e47b5 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Thu, 26 Sep 2024 14:37:51 +0200 Subject: [PATCH 269/489] add MessageRunMode to ProduceBlock parameters --- arbos/block_processor.go | 5 ++++- cmd/replay/main.go | 3 ++- execution/gethexec/block_recorder.go | 2 ++ execution/gethexec/executionengine.go | 6 ++++++ system_tests/state_fuzz_test.go | 7 ++++--- 5 files changed, 18 insertions(+), 5 deletions(-) diff --git a/arbos/block_processor.go b/arbos/block_processor.go index b180405c43..19fc36b351 100644 --- a/arbos/block_processor.go +++ b/arbos/block_processor.go @@ -144,6 +144,7 @@ func ProduceBlock( chainContext core.ChainContext, chainConfig *params.ChainConfig, isMsgForPrefetch bool, + runMode core.MessageRunMode, ) (*types.Block, types.Receipts, error) { txes, err := ParseL2Transactions(message, chainConfig.ChainID) if err != nil { @@ -153,7 +154,7 @@ func ProduceBlock( hooks := NoopSequencingHooks() return ProduceBlockAdvanced( - message.Header, txes, delayedMessagesRead, lastBlockHeader, statedb, chainContext, chainConfig, hooks, isMsgForPrefetch, + message.Header, txes, delayedMessagesRead, lastBlockHeader, statedb, chainContext, chainConfig, hooks, isMsgForPrefetch, runMode, ) } @@ -168,6 +169,7 @@ func ProduceBlockAdvanced( chainConfig *params.ChainConfig, sequencingHooks *SequencingHooks, isMsgForPrefetch bool, + runMode core.MessageRunMode, ) (*types.Block, types.Receipts, error) { state, err := arbosState.OpenSystemArbosState(statedb, nil, true) @@ -318,6 +320,7 @@ func ProduceBlockAdvanced( tx, &header.GasUsed, vm.Config{}, + runMode, func(result *core.ExecutionResult) error { return hooks.PostTxFilter(header, state, tx, sender, dataGas, result) }, diff --git a/cmd/replay/main.go b/cmd/replay/main.go index 0fe56eb4c9..d10d57a9c7 100644 --- a/cmd/replay/main.go +++ b/cmd/replay/main.go @@ -14,6 +14,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" @@ -291,7 +292,7 @@ func main() { message := readMessage(chainConfig.ArbitrumChainParams.DataAvailabilityCommittee) chainContext := WavmChainContext{} - newBlock, _, err = arbos.ProduceBlock(message.Message, message.DelayedMessagesRead, lastBlockHeader, statedb, chainContext, chainConfig, false) + newBlock, _, err = arbos.ProduceBlock(message.Message, message.DelayedMessagesRead, lastBlockHeader, statedb, chainContext, chainConfig, false, core.MessageReplayMode) // TODO verify runMode if err != nil { panic(err) } diff --git a/execution/gethexec/block_recorder.go b/execution/gethexec/block_recorder.go index a31b6b3736..a3af7876a8 100644 --- a/execution/gethexec/block_recorder.go +++ b/execution/gethexec/block_recorder.go @@ -8,6 +8,7 @@ import ( "github.com/ethereum/go-ethereum/arbitrum" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" @@ -154,6 +155,7 @@ func (r *BlockRecorder) RecordBlockCreation( chaincontext, chainConfig, false, + core.MessageReplayMode, ) if err != nil { return nil, err diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index 8d6484e3c9..d21522d2d4 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -505,6 +505,7 @@ func (s *ExecutionEngine) sequenceTransactionsWithBlockMutex(header *arbostypes. s.bc.Config(), hooks, false, + core.MessageCommitMode, ) if err != nil { return nil, err @@ -661,6 +662,10 @@ func (s *ExecutionEngine) createBlockFromNextMessage(msg *arbostypes.MessageWith statedb.StartPrefetcher("TransactionStreamer") defer statedb.StopPrefetcher() + runMode := core.MessageCommitMode + if isMsgForPrefetch { + runMode = core.MessageReplayMode + } block, receipts, err := arbos.ProduceBlock( msg.Message, msg.DelayedMessagesRead, @@ -669,6 +674,7 @@ func (s *ExecutionEngine) createBlockFromNextMessage(msg *arbostypes.MessageWith s.bc, s.bc.Config(), isMsgForPrefetch, + runMode, ) return block, statedb, receipts, err diff --git a/system_tests/state_fuzz_test.go b/system_tests/state_fuzz_test.go index 24140e480d..8fdfa3a098 100644 --- a/system_tests/state_fuzz_test.go +++ b/system_tests/state_fuzz_test.go @@ -38,6 +38,7 @@ func BuildBlock( chainConfig *params.ChainConfig, inbox arbstate.InboxBackend, seqBatch []byte, + runMode core.MessageRunMode, // TODO do we need to fuzz runMode? ) (*types.Block, error) { var delayedMessagesRead uint64 if lastBlockHeader != nil { @@ -63,7 +64,7 @@ func BuildBlock( } block, _, err := arbos.ProduceBlock( - l1Message, delayedMessagesRead, lastBlockHeader, statedb, chainContext, chainConfig, false, + l1Message, delayedMessagesRead, lastBlockHeader, statedb, chainContext, chainConfig, false, runMode, ) return block, err } @@ -127,7 +128,7 @@ func (c noopChainContext) GetHeader(common.Hash, uint64) *types.Header { } func FuzzStateTransition(f *testing.F) { - f.Fuzz(func(t *testing.T, compressSeqMsg bool, seqMsg []byte, delayedMsg []byte) { + f.Fuzz(func(t *testing.T, compressSeqMsg bool, seqMsg []byte, delayedMsg []byte, runMode uint8) { if len(seqMsg) > 0 && daprovider.IsL1AuthenticatedMessageHeaderByte(seqMsg[0]) { return } @@ -201,7 +202,7 @@ func FuzzStateTransition(f *testing.F) { positionWithinMessage: 0, delayedMessages: delayedMessages, } - _, err = BuildBlock(statedb, genesis, noopChainContext{}, params.ArbitrumOneChainConfig(), inbox, seqBatch) + _, err = BuildBlock(statedb, genesis, noopChainContext{}, params.ArbitrumOneChainConfig(), inbox, seqBatch, core.MessageRunMode(runMode)) if err != nil { // With the fixed header it shouldn't be possible to read a delayed message, // and no other type of error should be possible. From 05133258c92380005e1492304fbf08e137ef1c5d Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Thu, 26 Sep 2024 17:09:49 +0200 Subject: [PATCH 270/489] update geth pin --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 17cd001675..0c3f6eba21 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 17cd00167543a5a2b0b083e32820051100154c2f +Subproject commit 0c3f6eba21cbe0196b298dfbd3fa7d51dffd627e From ed680537f60ce8af3b6caa660b41698e77f2e007 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Thu, 26 Sep 2024 18:16:51 +0200 Subject: [PATCH 271/489] remove outdated todo comments --- cmd/replay/main.go | 2 +- system_tests/state_fuzz_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/replay/main.go b/cmd/replay/main.go index d10d57a9c7..661040ea10 100644 --- a/cmd/replay/main.go +++ b/cmd/replay/main.go @@ -292,7 +292,7 @@ func main() { message := readMessage(chainConfig.ArbitrumChainParams.DataAvailabilityCommittee) chainContext := WavmChainContext{} - newBlock, _, err = arbos.ProduceBlock(message.Message, message.DelayedMessagesRead, lastBlockHeader, statedb, chainContext, chainConfig, false, core.MessageReplayMode) // TODO verify runMode + newBlock, _, err = arbos.ProduceBlock(message.Message, message.DelayedMessagesRead, lastBlockHeader, statedb, chainContext, chainConfig, false, core.MessageReplayMode) if err != nil { panic(err) } diff --git a/system_tests/state_fuzz_test.go b/system_tests/state_fuzz_test.go index 8fdfa3a098..c0477060ed 100644 --- a/system_tests/state_fuzz_test.go +++ b/system_tests/state_fuzz_test.go @@ -38,7 +38,7 @@ func BuildBlock( chainConfig *params.ChainConfig, inbox arbstate.InboxBackend, seqBatch []byte, - runMode core.MessageRunMode, // TODO do we need to fuzz runMode? + runMode core.MessageRunMode, ) (*types.Block, error) { var delayedMessagesRead uint64 if lastBlockHeader != nil { From 39e94812606ba64d8d719be9729c955a0907d4d7 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 27 Sep 2024 00:51:36 +0200 Subject: [PATCH 272/489] fuzz state transition: skip malformed batch posting report --- system_tests/state_fuzz_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/system_tests/state_fuzz_test.go b/system_tests/state_fuzz_test.go index c0477060ed..2287870cb5 100644 --- a/system_tests/state_fuzz_test.go +++ b/system_tests/state_fuzz_test.go @@ -60,7 +60,8 @@ func BuildBlock( } err = l1Message.FillInBatchGasCost(batchFetcher) if err != nil { - return nil, err + // skip malformed batch posting report + return nil, nil } block, _, err := arbos.ProduceBlock( From e4d4b971d3ae60b124b220f3e8a6f745faed29cc Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 27 Sep 2024 00:55:45 +0200 Subject: [PATCH 273/489] fuzz state transition: test only existing message run modes --- system_tests/state_fuzz_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/system_tests/state_fuzz_test.go b/system_tests/state_fuzz_test.go index 2287870cb5..d722aa4212 100644 --- a/system_tests/state_fuzz_test.go +++ b/system_tests/state_fuzz_test.go @@ -129,7 +129,7 @@ func (c noopChainContext) GetHeader(common.Hash, uint64) *types.Header { } func FuzzStateTransition(f *testing.F) { - f.Fuzz(func(t *testing.T, compressSeqMsg bool, seqMsg []byte, delayedMsg []byte, runMode uint8) { + f.Fuzz(func(t *testing.T, compressSeqMsg bool, seqMsg []byte, delayedMsg []byte, runModeSeed uint8) { if len(seqMsg) > 0 && daprovider.IsL1AuthenticatedMessageHeaderByte(seqMsg[0]) { return } @@ -203,7 +203,9 @@ func FuzzStateTransition(f *testing.F) { positionWithinMessage: 0, delayedMessages: delayedMessages, } - _, err = BuildBlock(statedb, genesis, noopChainContext{}, params.ArbitrumOneChainConfig(), inbox, seqBatch, core.MessageRunMode(runMode)) + numberOfMessageRunModes := uint8(core.MessageReplayMode) + 1 // TODO update number of run modes when new mode is added + runMode := core.MessageRunMode(runModeSeed % numberOfMessageRunModes) + _, err = BuildBlock(statedb, genesis, noopChainContext{}, params.ArbitrumOneChainConfig(), inbox, seqBatch, runMode) if err != nil { // With the fixed header it shouldn't be possible to read a delayed message, // and no other type of error should be possible. From 760081d9d0a34cc9acb6a509a953f689a47f36b3 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 27 Sep 2024 01:06:23 +0200 Subject: [PATCH 274/489] make lint happy --- system_tests/state_fuzz_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/system_tests/state_fuzz_test.go b/system_tests/state_fuzz_test.go index d722aa4212..c8312350e6 100644 --- a/system_tests/state_fuzz_test.go +++ b/system_tests/state_fuzz_test.go @@ -61,6 +61,7 @@ func BuildBlock( err = l1Message.FillInBatchGasCost(batchFetcher) if err != nil { // skip malformed batch posting report + // nolint:nilerr return nil, nil } From 61554ac54681d0c4560829c03e992594fd02a1b4 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 26 Aug 2024 12:07:57 -0300 Subject: [PATCH 275/489] Moves package iteration from ci.yml to bash script --- .github/workflows/ci.yml | 54 +++----------------- .github/workflows/gotestsum.sh | 92 ++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 48 deletions(-) create mode 100755 .github/workflows/gotestsum.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b439fe4aec..e190b62809 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -144,40 +144,19 @@ jobs: if: matrix.test-mode == 'defaults' env: TEST_STATE_SCHEME: path - run: | - packages=`go list ./...` - for package in $packages; do - echo running tests for $package - if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -timeout 20m -tags=cionly > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then - exit 1 - fi - done + run: ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags cionly --timeout 20m --cover --write-full-log - name: run tests without race detection and hash state scheme if: matrix.test-mode == 'defaults' env: TEST_STATE_SCHEME: hash - run: | - packages=`go list ./...` - for package in $packages; do - echo running tests for $package - if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -timeout 20m -tags=cionly; then - exit 1 - fi - done + run: ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags cionly --timeout 20m - name: run tests with race detection and hash state scheme if: matrix.test-mode == 'race' env: TEST_STATE_SCHEME: hash - run: | - packages=`go list ./...` - for package in $packages; do - echo running tests for $package - if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -race -timeout 30m; then - exit 1 - fi - done + run: ${{ github.workspace }}/.github/workflows/gotestsum.sh --race --timeout 30m - name: run redis tests if: matrix.test-mode == 'defaults' @@ -185,36 +164,15 @@ jobs: - name: run challenge tests if: matrix.test-mode == 'challenge' - run: | - packages=`go list ./...` - for package in $packages; do - echo running tests for $package - if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -tags=challengetest -run=TestChallenge > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then - exit 1 - fi - done + run: ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags challengetest --run TestChallenge --cover --write-full-log - name: run stylus tests if: matrix.test-mode == 'stylus' - run: | - packages=`go list ./...` - for package in $packages; do - echo running tests for $package - if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -timeout 60m -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -tags=stylustest -run="TestProgramArbitrator" > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then - exit 1 - fi - done + run: ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags stylustest --run TestProgramArbitrator --timeout 60m --cover --write-full-log - name: run long stylus tests if: matrix.test-mode == 'long' - run: | - packages=`go list ./...` - for package in $packages; do - echo running tests for $package - if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -timeout 60m -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -tags=stylustest -run="TestProgramLong" > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then - exit 1 - fi - done + run: ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags stylustest --run TestProgramLong --timeout 60m --cover --write-full-log - name: Archive detailed run log uses: actions/upload-artifact@v3 diff --git a/.github/workflows/gotestsum.sh b/.github/workflows/gotestsum.sh new file mode 100755 index 0000000000..3c89538507 --- /dev/null +++ b/.github/workflows/gotestsum.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +check_missing_value() { + if [[ $1 -eq 0 || $2 == -* ]]; then + echo "missing $3 argument value" + exit 1 + fi +} + +timeout="" +tags="" +run="" +race=false +cover=false +write_full_log=false +while [[ $# -gt 0 ]]; do + case $1 in + --timeout) + shift + check_missing_value $# "$1" "--timeout" + timeout=$1 + shift + ;; + --tags) + shift + check_missing_value $# "$1" "--tags" + tags=$1 + shift + ;; + --run) + shift + check_missing_value $# "$1" "--run" + run=$1 + shift + ;; + --race) + race=true + shift + ;; + --cover) + cover=true + shift + ;; + --write-full-log) + write_full_log=true + shift + ;; + *) + echo "Invalid argument: $1" + exit 1 + ;; + esac +done + +packages=$(go list ./...) +for package in $packages; do + cmd="stdbuf -oL gotestsum --format short-verbose --packages=\"$package\" --rerun-fails=2 --no-color=false --" + + if [ "$timeout" != "" ]; then + cmd="$cmd -timeout $timeout" + else + cmd="$cmd -timeout 20m" + fi + + if [ "$tags" != "" ]; then + cmd="$cmd -tags=$tags" + fi + + if [ "$run" != "" ]; then + cmd="$cmd -run=$run" + fi + + if [ "$race" == true ]; then + cmd="$cmd -race" + fi + + if [ "$cover" == true ]; then + cmd="$cmd -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/..." + fi + + if [ "$write_full_log" == true ]; then + cmd="$cmd > >(stdbuf -oL tee -a full.log | grep -vE \"INFO|seal\")" + fi + + echo "" + echo running tests for "$package" + echo "$cmd" + + if ! eval "$cmd"; then + exit 1 + fi +done From a77653ed3412da048240cc3c88e60eed2f2364aa Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 29 Aug 2024 10:20:00 -0300 Subject: [PATCH 276/489] Removes default timeout in gotestsum.sh --- .github/workflows/gotestsum.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/gotestsum.sh b/.github/workflows/gotestsum.sh index 3c89538507..5da16cea24 100755 --- a/.github/workflows/gotestsum.sh +++ b/.github/workflows/gotestsum.sh @@ -58,8 +58,6 @@ for package in $packages; do if [ "$timeout" != "" ]; then cmd="$cmd -timeout $timeout" - else - cmd="$cmd -timeout 20m" fi if [ "$tags" != "" ]; then From 3c38c4098bbf4d95a70db0c853226eb4baf214c4 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 23 Sep 2024 16:45:59 -0300 Subject: [PATCH 277/489] Removes --write-full-log from .github/workflows/gotestsum.sh --- .github/workflows/ci.yml | 22 +++++++++++++++------- .github/workflows/gotestsum.sh | 4 +--- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e190b62809..5794580f03 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -144,35 +144,43 @@ jobs: if: matrix.test-mode == 'defaults' env: TEST_STATE_SCHEME: path - run: ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags cionly --timeout 20m --cover --write-full-log + run: | + echo "Running tests with Path Scheme" >> full.log + ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags cionly --timeout 20m --cover - name: run tests without race detection and hash state scheme if: matrix.test-mode == 'defaults' env: TEST_STATE_SCHEME: hash - run: ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags cionly --timeout 20m + run: | + echo "Running tests with Hash Scheme" >> full.log + ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags cionly --timeout 20m - name: run tests with race detection and hash state scheme if: matrix.test-mode == 'race' env: TEST_STATE_SCHEME: hash - run: ${{ github.workspace }}/.github/workflows/gotestsum.sh --race --timeout 30m + run: | + echo "Running tests with Hash Scheme" >> full.log + ${{ github.workspace }}/.github/workflows/gotestsum.sh --race --timeout 30m - name: run redis tests if: matrix.test-mode == 'defaults' - run: TEST_REDIS=redis://localhost:6379/0 gotestsum --format short-verbose -- -p 1 -run TestRedis ./arbnode/... ./system_tests/... -coverprofile=coverage-redis.txt -covermode=atomic -coverpkg=./... + run: | + echo "Running redis tests" >> full.log + TEST_REDIS=redis://localhost:6379/0 gotestsum --format short-verbose -- -p 1 -run TestRedis ./arbnode/... ./system_tests/... -coverprofile=coverage-redis.txt -covermode=atomic -coverpkg=./... - name: run challenge tests if: matrix.test-mode == 'challenge' - run: ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags challengetest --run TestChallenge --cover --write-full-log + run: ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags challengetest --run TestChallenge --cover - name: run stylus tests if: matrix.test-mode == 'stylus' - run: ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags stylustest --run TestProgramArbitrator --timeout 60m --cover --write-full-log + run: ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags stylustest --run TestProgramArbitrator --timeout 60m --cover - name: run long stylus tests if: matrix.test-mode == 'long' - run: ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags stylustest --run TestProgramLong --timeout 60m --cover --write-full-log + run: ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags stylustest --run TestProgramLong --timeout 60m --cover - name: Archive detailed run log uses: actions/upload-artifact@v3 diff --git a/.github/workflows/gotestsum.sh b/.github/workflows/gotestsum.sh index 5da16cea24..13cfc62fa9 100755 --- a/.github/workflows/gotestsum.sh +++ b/.github/workflows/gotestsum.sh @@ -76,9 +76,7 @@ for package in $packages; do cmd="$cmd -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/..." fi - if [ "$write_full_log" == true ]; then - cmd="$cmd > >(stdbuf -oL tee -a full.log | grep -vE \"INFO|seal\")" - fi + cmd="$cmd > >(stdbuf -oL tee -a full.log | grep -vE \"INFO|seal\")" echo "" echo running tests for "$package" From 73da786ab496fdb06a155af15840931861084c07 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 23 Sep 2024 16:48:47 -0300 Subject: [PATCH 278/489] Removes --write-full-log flag from .github/workflows/gotestsum.sh --- .github/workflows/gotestsum.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/gotestsum.sh b/.github/workflows/gotestsum.sh index 13cfc62fa9..ed631847b7 100755 --- a/.github/workflows/gotestsum.sh +++ b/.github/workflows/gotestsum.sh @@ -12,7 +12,6 @@ tags="" run="" race=false cover=false -write_full_log=false while [[ $# -gt 0 ]]; do case $1 in --timeout) @@ -41,10 +40,6 @@ while [[ $# -gt 0 ]]; do cover=true shift ;; - --write-full-log) - write_full_log=true - shift - ;; *) echo "Invalid argument: $1" exit 1 From 9a96fbf4fc58c598236cb486f22eb4967508c66f Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Fri, 27 Sep 2024 16:56:02 +0530 Subject: [PATCH 279/489] address PR comments --- pubsub/common.go | 4 +- pubsub/consumer.go | 20 +++++----- pubsub/producer.go | 62 ++++++++++++++--------------- pubsub/pubsub_test.go | 7 ++-- validator/valnode/redis/consumer.go | 4 +- 5 files changed, 45 insertions(+), 52 deletions(-) diff --git a/pubsub/common.go b/pubsub/common.go index 4b5778b9ba..ad36b6e622 100644 --- a/pubsub/common.go +++ b/pubsub/common.go @@ -9,9 +9,7 @@ import ( "github.com/redis/go-redis/v9" ) -const UNIQUEID_MSGID_MAP_KEY string = ".msgId" // Is used to map unique identifier to msgId of the message consisting request in the stream - -func MessageKeyFor(streamName, id string) string { return fmt.Sprintf("%s.%s", streamName, id) } +func ResultKeyFor(streamName, id string) string { return fmt.Sprintf("%s.%s", streamName, id) } // CreateStream tries to create stream with given name, if it already exists // does not return an error. diff --git a/pubsub/consumer.go b/pubsub/consumer.go index fa1f5894b4..3265744218 100644 --- a/pubsub/consumer.go +++ b/pubsub/consumer.go @@ -51,9 +51,9 @@ type Consumer[Request any, Response any] struct { } type Message[Request any] struct { - ID string - Value Request - AckNotifier chan struct{} + ID string + Value Request + Ack func() } func NewConsumer[Request any, Response any](client redis.UniversalClient, streamName string, cfg *ConsumerConfig) (*Consumer[Request, Response], error) { @@ -101,7 +101,7 @@ func decrementMsgIdByOne(msgId string) string { } else if id[0] > 0 { return strconv.FormatUint(id[0]-1, 10) + "-" + strconv.FormatUint(math.MaxUint64, 10) } else { - log.Error("Error decrementing start of XAutoClaim by one, defaulting to 0", "err", err) + log.Error("Error decrementing start of XAutoClaim by one, defaulting to 0") return "0" } } @@ -207,9 +207,9 @@ func (c *Consumer[Request, Response]) Consume(ctx context.Context) (*Message[Req }) log.Debug("Redis stream consuming", "consumer_id", c.id, "message_id", messages[0].ID) return &Message[Request]{ - ID: messages[0].ID, - Value: req, - AckNotifier: ackNotifier, + ID: messages[0].ID, + Value: req, + Ack: func() { close(ackNotifier) }, }, nil } @@ -218,9 +218,9 @@ func (c *Consumer[Request, Response]) SetResult(ctx context.Context, messageID s if err != nil { return fmt.Errorf("marshaling result: %w", err) } - msgKey := MessageKeyFor(c.StreamName(), messageID) - log.Debug("consumer: setting result", "cid", c.id, "msgIdInStream", messageID, "msgKeyInRedis", msgKey) - acquired, err := c.client.SetNX(ctx, msgKey, resp, c.cfg.ResponseEntryTimeout).Result() + resultKey := ResultKeyFor(c.StreamName(), messageID) + log.Debug("consumer: setting result", "cid", c.id, "msgIdInStream", messageID, "resultKeyInRedis", resultKey) + acquired, err := c.client.SetNX(ctx, resultKey, resp, c.cfg.ResponseEntryTimeout).Result() if err != nil || !acquired { return fmt.Errorf("setting result for message with message-id in stream: %v, error: %w", messageID, err) } diff --git a/pubsub/producer.go b/pubsub/producer.go index 932b455d6a..5c87f4f722 100644 --- a/pubsub/producer.go +++ b/pubsub/producer.go @@ -50,27 +50,22 @@ type Producer[Request any, Response any] struct { type ProducerConfig struct { // Interval duration for checking the result set by consumers. CheckResultInterval time.Duration `koanf:"check-result-interval"` - // Timeout of entry's written to redis by producer - ResponseEntryTimeout time.Duration `koanf:"response-entry-timeout"` // RequestTimeout is a TTL for any message sent to the redis stream RequestTimeout time.Duration `koanf:"request-timeout"` } var DefaultProducerConfig = ProducerConfig{ - CheckResultInterval: 5 * time.Second, - ResponseEntryTimeout: time.Hour, - RequestTimeout: time.Hour, // should we increase this? + CheckResultInterval: 5 * time.Second, + RequestTimeout: 3 * time.Hour, } var TestProducerConfig = ProducerConfig{ - CheckResultInterval: 5 * time.Millisecond, - ResponseEntryTimeout: time.Minute, - RequestTimeout: time.Minute, + CheckResultInterval: 5 * time.Millisecond, + RequestTimeout: time.Minute, } func ProducerAddConfigAddOptions(prefix string, f *pflag.FlagSet) { f.Duration(prefix+".check-result-interval", DefaultProducerConfig.CheckResultInterval, "interval in which producer checks pending messages whether consumer processing them is inactive") - f.Duration(prefix+".response-entry-timeout", DefaultProducerConfig.ResponseEntryTimeout, "timeout after which responses written from producer to the redis are cleared. Currently used for the key mapping unique request id to redis stream message id") f.Duration(prefix+".request-timeout", DefaultProducerConfig.RequestTimeout, "timeout after which the message in redis stream is considered as errored, this prevents workers from working on wrong requests indefinitely") } @@ -133,39 +128,30 @@ func cmpMsgId(msgId1, msgId2 string) int { // checkResponses checks iteratively whether response for the promise is ready. func (p *Producer[Request, Response]) checkResponses(ctx context.Context) time.Duration { - pelData, err := p.client.XPending(ctx, p.redisStream, p.redisGroup).Result() - if err != nil { - log.Error("error getting PEL data from xpending, xtrimming is disabled", "err", err) - } log.Debug("redis producer: check responses starting") p.promisesLock.Lock() defer p.promisesLock.Unlock() responded := 0 errored := 0 checked := 0 + allowedOldestID := fmt.Sprintf("%d-0", time.Now().Add(-p.cfg.RequestTimeout).UnixMilli()) for id, promise := range p.promises { if ctx.Err() != nil { return 0 } checked++ - msgKey := MessageKeyFor(p.redisStream, id) - res, err := p.client.Get(ctx, msgKey).Result() + resultKey := ResultKeyFor(p.redisStream, id) + res, err := p.client.Get(ctx, resultKey).Result() if err != nil { if !errors.Is(err, redis.Nil) { - log.Error("Error reading value in redis", "key", msgKey, "error", err) - } else { + log.Error("Error reading value in redis", "key", resultKey, "error", err) + } else if cmpMsgId(id, allowedOldestID) == -1 { // The request this producer is waiting for has been past its TTL or is older than current PEL's lower, // so safe to error and stop tracking this promise - allowedOldestID := fmt.Sprintf("%d-0", time.Now().Add(-p.cfg.RequestTimeout).UnixMilli()) - if pelData != nil && pelData.Lower != "" { - allowedOldestID = pelData.Lower - } - if cmpMsgId(id, allowedOldestID) == -1 { - promise.ProduceError(errors.New("error getting response, request has been waiting for too long")) - log.Error("error getting response, request has been waiting past its TTL") - errored++ - delete(p.promises, id) - } + promise.ProduceError(errors.New("error getting response, request has been waiting for too long")) + log.Error("error getting response, request has been waiting past its TTL") + errored++ + delete(p.promises, id) } continue } @@ -178,16 +164,25 @@ func (p *Producer[Request, Response]) checkResponses(ctx context.Context) time.D promise.Produce(resp) responded++ } - p.client.Del(ctx, msgKey) + p.client.Del(ctx, resultKey) delete(p.promises, id) } + log.Debug("checkResponses", "responded", responded, "errored", errored, "checked", checked) + return p.cfg.CheckResultInterval +} + +func (p *Producer[Request, Response]) clearMessages(ctx context.Context) time.Duration { + pelData, err := p.client.XPending(ctx, p.redisStream, p.redisGroup).Result() + if err != nil { + log.Error("error getting PEL data from xpending, xtrimming is disabled", "err", err) + } // XDEL on consumer side already deletes acked messages (mark as deleted) but doesnt claim the memory back, XTRIM helps in claiming this memory in normal conditions // pelData might be outdated when we do the xtrim, but thats ok as the messages are also being trimmed by other producers if pelData != nil && pelData.Lower != "" { trimmed, trimErr := p.client.XTrimMinID(ctx, p.redisStream, pelData.Lower).Result() - log.Debug("trimming", "xTrimMinID", pelData.Lower, "trimmed", trimmed, "responded", responded, "errored", errored, "trim-err", trimErr, "checked", checked) + log.Debug("trimming", "xTrimMinID", pelData.Lower, "trimmed", trimmed, "trim-err", trimErr) // Check if pelData.Lower has been past its TTL and if it is then ack it to remove from PEL and delete it, once - // its taken out from PEL the producer that sent this request will handle the corresponding promise accordingly (if PEL is non-empty) + // its taken out from PEL the producer that sent this request will handle the corresponding promise accordingly (as its past TTL) allowedOldestID := fmt.Sprintf("%d-0", time.Now().Add(-p.cfg.RequestTimeout).UnixMilli()) if cmpMsgId(pelData.Lower, allowedOldestID) == -1 { if err := p.client.XClaim(ctx, &redis.XClaimArgs{ @@ -198,18 +193,18 @@ func (p *Producer[Request, Response]) checkResponses(ctx context.Context) time.D Messages: []string{pelData.Lower}, }).Err(); err != nil { log.Error("error claiming PEL's lower message thats past its TTL", "msgID", pelData.Lower, "err", err) - return p.cfg.CheckResultInterval + return 5 * p.cfg.CheckResultInterval } if _, err := p.client.XAck(ctx, p.redisStream, p.redisGroup, pelData.Lower).Result(); err != nil { log.Error("error acking PEL's lower message thats past its TTL", "msgID", pelData.Lower, "err", err) - return p.cfg.CheckResultInterval + return 5 * p.cfg.CheckResultInterval } if _, err := p.client.XDel(ctx, p.redisStream, pelData.Lower).Result(); err != nil { log.Error("error deleting PEL's lower message thats past its TTL", "msgID", pelData.Lower, "err", err) } } } - return p.cfg.CheckResultInterval + return 5 * p.cfg.CheckResultInterval } func (p *Producer[Request, Response]) Start(ctx context.Context) { @@ -246,6 +241,7 @@ func (p *Producer[Request, Response]) Produce(ctx context.Context, value Request log.Debug("Redis stream producing", "value", value) p.once.Do(func() { p.StopWaiter.CallIteratively(p.checkResponses) + p.StopWaiter.CallIteratively(p.clearMessages) }) return p.produce(ctx, value) } diff --git a/pubsub/pubsub_test.go b/pubsub/pubsub_test.go index 391bd7555c..8bd1aed25d 100644 --- a/pubsub/pubsub_test.go +++ b/pubsub/pubsub_test.go @@ -48,9 +48,8 @@ func destroyRedisGroup(ctx context.Context, t *testing.T, streamName string, cli func producerCfg() *ProducerConfig { return &ProducerConfig{ - CheckResultInterval: TestProducerConfig.CheckResultInterval, - ResponseEntryTimeout: TestProducerConfig.ResponseEntryTimeout, - RequestTimeout: 2 * time.Second, + CheckResultInterval: TestProducerConfig.CheckResultInterval, + RequestTimeout: 2 * time.Second, } } @@ -186,7 +185,7 @@ func consume(ctx context.Context, t *testing.T, consumers []*Consumer[testReques } wantResponses[idx] = append(wantResponses[idx], resp) } - close(res.AckNotifier) + res.Ack() } }) } diff --git a/validator/valnode/redis/consumer.go b/validator/valnode/redis/consumer.go index c87191e444..4392a3c91e 100644 --- a/validator/valnode/redis/consumer.go +++ b/validator/valnode/redis/consumer.go @@ -161,12 +161,12 @@ func (s *ValidationServer) Start(ctx_in context.Context) { res, err := valRun.Await(ctx) if err != nil { log.Error("Error validating", "request value", work.req.Value, "error", err) - close(work.req.AckNotifier) + work.req.Ack() } else { log.Debug("done work", "thread", i, "workid", work.req.ID) err := s.consumers[work.moduleRoot].SetResult(ctx, work.req.ID, res) // Even in error we close ackNotifier as there's no retry mechanism here and closing it will alow other consumers to autoclaim - close(work.req.AckNotifier) + work.req.Ack() if err != nil { log.Error("Error setting result for request", "id", work.req.ID, "result", res, "error", err) } From 43a54e7cef089510869deb3afa23c79f4c9db2c5 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Fri, 27 Sep 2024 16:58:47 +0530 Subject: [PATCH 280/489] remove unnecessary error log in decrementMsgIdByOne --- pubsub/consumer.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pubsub/consumer.go b/pubsub/consumer.go index 3265744218..391042bd7e 100644 --- a/pubsub/consumer.go +++ b/pubsub/consumer.go @@ -100,10 +100,8 @@ func decrementMsgIdByOne(msgId string) string { return strconv.FormatUint(id[0], 10) + "-" + strconv.FormatUint(id[1]-1, 10) } else if id[0] > 0 { return strconv.FormatUint(id[0]-1, 10) + "-" + strconv.FormatUint(math.MaxUint64, 10) - } else { - log.Error("Error decrementing start of XAutoClaim by one, defaulting to 0") - return "0" } + return "0" } // Consumer first checks it there exists pending message that is claimed by From b30eb33252b68d4e6c4cd433fc12098c4cb36b90 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Fri, 27 Sep 2024 18:31:24 +0530 Subject: [PATCH 281/489] Changes based on PR comments --- arbos/util/storage_cache.go | 3 +-- arbos/util/storage_cache_test.go | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/arbos/util/storage_cache.go b/arbos/util/storage_cache.go index dfbdc0892b..9573d1ffc7 100644 --- a/arbos/util/storage_cache.go +++ b/arbos/util/storage_cache.go @@ -4,7 +4,6 @@ package util import ( - "bytes" "github.com/ethereum/go-ethereum/common" "slices" ) @@ -70,7 +69,7 @@ func (s *storageCache) Flush() []storageCacheStores { } } sortFunc := func(a, b storageCacheStores) int { - return bytes.Compare(a.Key.Bytes(), b.Key.Bytes()) + return a.Key.Cmp(b.Key) } slices.SortFunc(stores, sortFunc) return stores diff --git a/arbos/util/storage_cache_test.go b/arbos/util/storage_cache_test.go index 1cc4ea14ec..9fd452851d 100644 --- a/arbos/util/storage_cache_test.go +++ b/arbos/util/storage_cache_test.go @@ -4,7 +4,6 @@ package util import ( - "bytes" "slices" "testing" @@ -76,7 +75,7 @@ func TestStorageCache(t *testing.T) { {Key: keys[2], Value: values[2]}, } sortFunc := func(a, b storageCacheStores) int { - return bytes.Compare(a.Key.Bytes(), b.Key.Bytes()) + return a.Key.Cmp(b.Key) } slices.SortFunc(stores, sortFunc) slices.SortFunc(expected, sortFunc) From 43e852f5b081a9c9355bf89013db573df7b0d810 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Fri, 27 Sep 2024 11:49:59 -0300 Subject: [PATCH 282/489] Test Stylus gas usage These tests compare the gas usage of EVM opcodes and Stylus HostIOs. Co-authored-by: Aman Sanghi Co-authored-by: Tsahi Zidenberg --- Makefile | 8 +- .../stylus/tests/hostio-test/Cargo.lock | 636 ++++++++++++++++++ .../stylus/tests/hostio-test/Cargo.toml | 17 + .../stylus/tests/hostio-test/src/main.rs | 201 ++++++ contracts | 2 +- system_tests/program_gas_test.go | 437 ++++++++++++ 6 files changed, 1299 insertions(+), 2 deletions(-) create mode 100644 arbitrator/stylus/tests/hostio-test/Cargo.lock create mode 100644 arbitrator/stylus/tests/hostio-test/Cargo.toml create mode 100644 arbitrator/stylus/tests/hostio-test/src/main.rs create mode 100644 system_tests/program_gas_test.go diff --git a/Makefile b/Makefile index c3cf1a5144..88bbd8dabe 100644 --- a/Makefile +++ b/Makefile @@ -149,8 +149,10 @@ stylus_test_erc20_wasm = $(call get_stylus_test_wasm,erc20) stylus_test_erc20_src = $(call get_stylus_test_rust,erc20) stylus_test_read-return-data_wasm = $(call get_stylus_test_wasm,read-return-data) stylus_test_read-return-data_src = $(call get_stylus_test_rust,read-return-data) +stylus_test_hostio-test_wasm = $(call get_stylus_test_wasm,hostio-test) +stylus_test_hostio-test_src = $(call get_stylus_test_rust,hostio-test) -stylus_test_wasms = $(stylus_test_keccak_wasm) $(stylus_test_keccak-100_wasm) $(stylus_test_fallible_wasm) $(stylus_test_storage_wasm) $(stylus_test_multicall_wasm) $(stylus_test_log_wasm) $(stylus_test_create_wasm) $(stylus_test_math_wasm) $(stylus_test_sdk-storage_wasm) $(stylus_test_erc20_wasm) $(stylus_test_read-return-data_wasm) $(stylus_test_evm-data_wasm) $(stylus_test_bfs:.b=.wasm) +stylus_test_wasms = $(stylus_test_keccak_wasm) $(stylus_test_keccak-100_wasm) $(stylus_test_fallible_wasm) $(stylus_test_storage_wasm) $(stylus_test_multicall_wasm) $(stylus_test_log_wasm) $(stylus_test_create_wasm) $(stylus_test_math_wasm) $(stylus_test_sdk-storage_wasm) $(stylus_test_erc20_wasm) $(stylus_test_read-return-data_wasm) $(stylus_test_evm-data_wasm) $(stylus_test_hostio-test_wasm) $(stylus_test_bfs:.b=.wasm) stylus_benchmarks = $(wildcard $(stylus_dir)/*.toml $(stylus_dir)/src/*.rs) $(stylus_test_wasms) # user targets @@ -482,6 +484,10 @@ $(stylus_test_erc20_wasm): $(stylus_test_erc20_src) $(cargo_nightly) --manifest-path $< --release --config $(stylus_cargo) @touch -c $@ # cargo might decide to not rebuild the binary +$(stylus_test_hostio-test_wasm): $(stylus_test_hostio-test_src) + $(cargo_nightly) --manifest-path $< --release --config $(stylus_cargo) + @touch -c $@ # cargo might decide to not rebuild the binary + contracts/test/prover/proofs/float%.json: $(arbitrator_cases)/float%.wasm $(prover_bin) $(output_latest)/soft-float.wasm $(prover_bin) $< -l $(output_latest)/soft-float.wasm -o $@ -b --allow-hostapi --require-success diff --git a/arbitrator/stylus/tests/hostio-test/Cargo.lock b/arbitrator/stylus/tests/hostio-test/Cargo.lock new file mode 100644 index 0000000000..1e726910b1 --- /dev/null +++ b/arbitrator/stylus/tests/hostio-test/Cargo.lock @@ -0,0 +1,636 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloy-primitives" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e416903084d3392ebd32d94735c395d6709415b76c7728e594d3f996f2b03e65" +dependencies = [ + "bytes", + "cfg-if 1.0.0", + "const-hex", + "derive_more", + "hex-literal", + "itoa", + "ruint", + "tiny-keccak", +] + +[[package]] +name = "alloy-sol-macro" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a74ceeffdacf9dd0910404d743d07273776fd17b85f9cb17b49a97e5c6055ce9" +dependencies = [ + "dunce", + "heck", + "proc-macro2", + "quote", + "syn 2.0.77", + "syn-solidity", + "tiny-keccak", +] + +[[package]] +name = "alloy-sol-types" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f347cb6bb307b3802ec455ef43ce00f5e590e0ceca3d2f3b070f5ee367e235" +dependencies = [ + "alloy-primitives", + "alloy-sol-macro", + "const-hex", +] + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "bitflags" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "const-hex" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8a24a26d37e1ffd45343323dc9fe6654ceea44c12f2fcb3d7ac29e610bc6" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "hex", + "proptest", + "serde", +] + +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "cpufeatures" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +dependencies = [ + "libc", +] + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "derivative" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "derive_more" +version = "0.99.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce" +dependencies = [ + "convert_case 0.4.0", + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.77", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hex-literal" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fe2267d4ed49bc07b63801559be28c718ea06c4738b7a03c94df7386d2cde46" + +[[package]] +name = "hostio-test" +version = "0.1.0" +dependencies = [ + "mini-alloc", + "stylus-sdk", +] + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "keccak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" +dependencies = [ + "cpufeatures", +] + +[[package]] +name = "keccak-const" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d8d8ce877200136358e0bbff3a77965875db3af755a11e1fa6b1b3e2df13ea" + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.159" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" + +[[package]] +name = "libm" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "memory_units" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8452105ba047068f40ff7093dd1d9da90898e63dd61736462e9cdda6a90ad3c3" + +[[package]] +name = "mini-alloc" +version = "0.4.2" +dependencies = [ + "cfg-if 1.0.0", + "wee_alloc", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proptest" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c2511913b88df1637da85cc8d96ec8e43a3f8bb8ccb71ee1ac240d6f3df58d" +dependencies = [ + "bitflags", + "num-traits", + "rand", + "rand_chacha", + "rand_xorshift", + "unarray", +] + +[[package]] +name = "quote" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" + +[[package]] +name = "rand_xorshift" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" +dependencies = [ + "rand_core", +] + +[[package]] +name = "regex" +version = "1.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" + +[[package]] +name = "ruint" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c3cc4c2511671f327125da14133d0c5c5d137f006a1017a16f557bc85b16286" +dependencies = [ + "proptest", + "rand", + "ruint-macro", + "serde", + "valuable", + "zeroize", +] + +[[package]] +name = "ruint-macro" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48fd7bd8a6377e15ad9d42a8ec25371b94ddc67abe7c8b9127bec79bebaaae18" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "semver" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" + +[[package]] +name = "serde" +version = "1.0.210" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.210" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] + +[[package]] +name = "sha3" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" +dependencies = [ + "digest", + "keccak", +] + +[[package]] +name = "stylus-proc" +version = "0.4.2" +dependencies = [ + "alloy-primitives", + "alloy-sol-types", + "cfg-if 1.0.0", + "convert_case 0.6.0", + "lazy_static", + "proc-macro2", + "quote", + "regex", + "sha3", + "syn 1.0.109", + "syn-solidity", +] + +[[package]] +name = "stylus-sdk" +version = "0.4.2" +dependencies = [ + "alloy-primitives", + "alloy-sol-types", + "cfg-if 1.0.0", + "derivative", + "hex", + "keccak-const", + "lazy_static", + "stylus-proc", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn-solidity" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5f995d2140b0f751dbe94365be2591edbf3d1b75dcfaeac14183abbd2ff07bd" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + +[[package]] +name = "unicode-ident" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" + +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wee_alloc" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbb3b5a6b2bb17cb6ad44a2e68a43e8d2722c997da10e928665c72ec6c0a0b8e" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "memory_units", + "winapi", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] + +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" diff --git a/arbitrator/stylus/tests/hostio-test/Cargo.toml b/arbitrator/stylus/tests/hostio-test/Cargo.toml new file mode 100644 index 0000000000..da7bbce7a3 --- /dev/null +++ b/arbitrator/stylus/tests/hostio-test/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "hostio-test" +version = "0.1.0" +edition = "2021" + +[dependencies] +stylus-sdk = { path = "../../../langs/rust/stylus-sdk", features = ["debug", "hostio"] } +mini-alloc.path = "../../../langs/rust/mini-alloc" + +[profile.release] +codegen-units = 1 +strip = true +lto = true +panic = "abort" +opt-level = "s" + +[workspace] diff --git a/arbitrator/stylus/tests/hostio-test/src/main.rs b/arbitrator/stylus/tests/hostio-test/src/main.rs new file mode 100644 index 0000000000..5aa465598e --- /dev/null +++ b/arbitrator/stylus/tests/hostio-test/src/main.rs @@ -0,0 +1,201 @@ +// Copyright 2024, Offchain Labs, Inc. +// For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE + +#![no_main] + +use stylus_sdk::{ + abi::Bytes, + alloy_primitives::{Address, B256, U256}, + block, console, contract, evm, hostio, msg, + prelude::*, + stylus_proc::entrypoint, + tx, + types::AddressVM, +}; +extern crate alloc; + +#[cfg(target_arch = "wasm32")] +#[global_allocator] +static ALLOC: mini_alloc::MiniAlloc = mini_alloc::MiniAlloc::INIT; + +sol_storage! { + #[entrypoint] + pub struct HostioTest { + } +} + +type Result = std::result::Result>; + +// These are not available as hostios in the sdk, so we import them directly. +#[link(wasm_import_module = "vm_hooks")] +extern "C" { + fn math_div(value: *mut u8, divisor: *const u8); + fn math_mod(value: *mut u8, modulus: *const u8); + fn math_pow(value: *mut u8, exponent: *const u8); + fn math_add_mod(value: *mut u8, addend: *const u8, modulus: *const u8); + fn math_mul_mod(value: *mut u8, multiplier: *const u8, modulus: *const u8); + fn transient_load_bytes32(key: *const u8, dest: *mut u8); + fn transient_store_bytes32(key: *const u8, value: *const u8); + fn exit_early(status: u32); +} + +#[external] +impl HostioTest { + fn exit_early() -> Result<()> { + unsafe { + exit_early(0); + } + Ok(()) + } + + fn transient_load_bytes32(key: B256) -> Result { + let mut result = B256::ZERO; + unsafe { + transient_load_bytes32(key.as_ptr(), result.as_mut_ptr()); + } + Ok(result) + } + + fn transient_store_bytes32(key: B256, value: B256) { + unsafe { + transient_store_bytes32(key.as_ptr(), value.as_ptr()); + } + } + + fn return_data_size() -> Result { + unsafe { Ok(hostio::return_data_size().try_into().unwrap()) } + } + + fn emit_log(data: Bytes, n: i8, t1: B256, t2: B256, t3: B256, t4: B256) -> Result<()> { + let topics = &[t1, t2, t3, t4]; + evm::raw_log(&topics[0..n as usize], data.as_slice())?; + Ok(()) + } + + fn account_balance(account: Address) -> Result { + Ok(account.balance()) + } + + fn account_code(account: Address) -> Result> { + Ok(account.code()) + } + + fn account_code_size(account: Address) -> Result { + Ok(account.code_size().try_into().unwrap()) + } + + fn account_codehash(account: Address) -> Result { + Ok(account.codehash()) + } + + fn evm_gas_left() -> Result { + Ok(evm::gas_left().try_into().unwrap()) + } + + fn evm_ink_left() -> Result { + Ok(tx::ink_to_gas(evm::ink_left()).try_into().unwrap()) + } + + fn block_basefee() -> Result { + Ok(block::basefee()) + } + + fn chainid() -> Result { + Ok(block::chainid().try_into().unwrap()) + } + + fn block_coinbase() -> Result
{ + Ok(block::coinbase()) + } + + fn block_gas_limit() -> Result { + Ok(block::gas_limit().try_into().unwrap()) + } + + fn block_number() -> Result { + Ok(block::number().try_into().unwrap()) + } + + fn block_timestamp() -> Result { + Ok(block::timestamp().try_into().unwrap()) + } + + fn contract_address() -> Result
{ + Ok(contract::address()) + } + + fn math_div(a: U256, b: U256) -> Result { + let mut a_bytes: B256 = a.into(); + let b_bytes: B256 = b.into(); + unsafe { + math_div(a_bytes.as_mut_ptr(), b_bytes.as_ptr()); + } + Ok(a_bytes.into()) + } + + fn math_mod(a: U256, b: U256) -> Result { + let mut a_bytes: B256 = a.into(); + let b_bytes: B256 = b.into(); + unsafe { + math_mod(a_bytes.as_mut_ptr(), b_bytes.as_ptr()); + } + Ok(a_bytes.into()) + } + + fn math_pow(a: U256, b: U256) -> Result { + let mut a_bytes: B256 = a.into(); + let b_bytes: B256 = b.into(); + unsafe { + math_pow(a_bytes.as_mut_ptr(), b_bytes.as_ptr()); + } + Ok(a_bytes.into()) + } + + fn math_add_mod(a: U256, b: U256, c: U256) -> Result { + let mut a_bytes: B256 = a.into(); + let b_bytes: B256 = b.into(); + let c_bytes: B256 = c.into(); + unsafe { + math_add_mod(a_bytes.as_mut_ptr(), b_bytes.as_ptr(), c_bytes.as_ptr()); + } + Ok(a_bytes.into()) + } + + fn math_mul_mod(a: U256, b: U256, c: U256) -> Result { + let mut a_bytes: B256 = a.into(); + let b_bytes: B256 = b.into(); + let c_bytes: B256 = c.into(); + unsafe { + math_mul_mod(a_bytes.as_mut_ptr(), b_bytes.as_ptr(), c_bytes.as_ptr()); + } + Ok(a_bytes.into()) + } + + fn msg_sender() -> Result
{ + Ok(msg::sender()) + } + + fn msg_value() -> Result { + Ok(msg::value()) + } + + fn keccak(preimage: Bytes) -> Result { + let mut result = B256::ZERO; + unsafe { + hostio::native_keccak256(preimage.as_ptr(), preimage.len(), result.as_mut_ptr()); + } + Ok(result) + } + + fn tx_gas_price() -> Result { + Ok(tx::gas_price()) + } + + fn tx_ink_price() -> Result { + Ok(tx::ink_to_gas(tx::ink_price().into()).try_into().unwrap()) + } + + fn tx_origin() -> Result
{ + Ok(tx::origin()) + } +} diff --git a/contracts b/contracts index 7396313311..cde7f96486 160000 --- a/contracts +++ b/contracts @@ -1 +1 @@ -Subproject commit 7396313311ab17cb30e2eef27cccf96f0a9e8f7f +Subproject commit cde7f96486cc21befc9a05fc3b27a7d6ebb5d5f8 diff --git a/system_tests/program_gas_test.go b/system_tests/program_gas_test.go new file mode 100644 index 0000000000..48db268907 --- /dev/null +++ b/system_tests/program_gas_test.go @@ -0,0 +1,437 @@ +package arbtest + +import ( + "context" + "fmt" + "math" + "math/big" + "regexp" + "strings" + "testing" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/eth/tracers/logger" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/rpc" + "github.com/offchainlabs/nitro/arbos/util" + "github.com/offchainlabs/nitro/execution/gethexec" + "github.com/offchainlabs/nitro/solgen/go/mocksgen" + "github.com/offchainlabs/nitro/solgen/go/precompilesgen" + "github.com/offchainlabs/nitro/util/testhelpers" +) + +func TestProgramSimpleCost(t *testing.T) { + ctx, l2info, l2client, auth := setupGasCostTest(t) + stylusProgram := deployWasm(t, ctx, auth, l2client, rustFile("hostio-test")) + evmProgram := deployEvmContract(t, ctx, auth, l2client, mocksgen.HostioTestMetaData) + otherProgram := deployWasm(t, ctx, auth, l2client, rustFile("storage")) + matchSnake := regexp.MustCompile("_[a-z]") + + for _, tc := range []struct { + hostio string + opcode vm.OpCode + params []any + maxDiff float64 + }{ + {hostio: "exit_early", opcode: vm.STOP}, + {hostio: "transient_load_bytes32", opcode: vm.TLOAD, params: []any{common.HexToHash("dead")}}, + {hostio: "transient_store_bytes32", opcode: vm.TSTORE, params: []any{common.HexToHash("dead"), common.HexToHash("beef")}}, + {hostio: "return_data_size", opcode: vm.RETURNDATASIZE, maxDiff: 1.0}, + {hostio: "account_balance", opcode: vm.BALANCE, params: []any{l2info.GetAddress("Owner")}}, + {hostio: "account_code", opcode: vm.EXTCODECOPY, params: []any{otherProgram}, maxDiff: 2}, + {hostio: "account_code_size", opcode: vm.EXTCODESIZE, params: []any{otherProgram}}, + {hostio: "account_codehash", opcode: vm.EXTCODEHASH, params: []any{otherProgram}}, + {hostio: "evm_gas_left", opcode: vm.GAS, maxDiff: 1.0}, + {hostio: "evm_ink_left", opcode: vm.GAS, maxDiff: 1.0}, + {hostio: "block_basefee", opcode: vm.BASEFEE, maxDiff: 0.5}, + {hostio: "chainid", opcode: vm.CHAINID, maxDiff: 1.0}, + {hostio: "block_coinbase", opcode: vm.COINBASE, maxDiff: 0.5}, + {hostio: "block_gas_limit", opcode: vm.GASLIMIT, maxDiff: 1.0}, + {hostio: "block_number", opcode: vm.NUMBER, maxDiff: 1.0}, + {hostio: "block_timestamp", opcode: vm.TIMESTAMP, maxDiff: 1.0}, + {hostio: "contract_address", opcode: vm.ADDRESS, maxDiff: 0.5}, + {hostio: "math_div", opcode: vm.DIV, params: []any{big.NewInt(1), big.NewInt(3)}}, + {hostio: "math_mod", opcode: vm.MOD, params: []any{big.NewInt(1), big.NewInt(3)}}, + {hostio: "math_pow", opcode: vm.EXP, params: []any{big.NewInt(1), new(big.Int).Lsh(big.NewInt(1), 255)}, maxDiff: 2}, // worst case + {hostio: "math_add_mod", opcode: vm.ADDMOD, params: []any{big.NewInt(1), big.NewInt(3), big.NewInt(5)}, maxDiff: 0.5}, + {hostio: "math_mul_mod", opcode: vm.MULMOD, params: []any{big.NewInt(1), big.NewInt(3), big.NewInt(5)}, maxDiff: 0.5}, + {hostio: "msg_sender", opcode: vm.CALLER, maxDiff: 0.5}, + {hostio: "msg_value", opcode: vm.CALLVALUE, maxDiff: 0.5}, + {hostio: "tx_gas_price", opcode: vm.GASPRICE, maxDiff: 0.5}, + {hostio: "tx_ink_price", opcode: vm.GASPRICE, maxDiff: 1.0}, + {hostio: "tx_origin", opcode: vm.ORIGIN, maxDiff: 0.5}, + } { + t.Run(tc.hostio, func(t *testing.T) { + solFunc := matchSnake.ReplaceAllStringFunc(tc.hostio, func(s string) string { + return strings.ToUpper(strings.TrimPrefix(s, "_")) + }) + packer, _ := util.NewCallParser(mocksgen.HostioTestABI, solFunc) + data, err := packer(tc.params...) + Require(t, err) + compareGasUsage(t, ctx, l2client, l2info, evmProgram, stylusProgram, data, nil, + compareGasForEach, tc.maxDiff, compareGasPair{tc.opcode, tc.hostio}) + }) + } +} + +func TestProgramStorageCost(t *testing.T) { + ctx, l2info, l2client, auth := setupGasCostTest(t) + + stylusMulticall := deployWasm(t, ctx, auth, l2client, rustFile("multicall")) + evmMulticall := deployEvmContract(t, ctx, auth, l2client, mocksgen.MultiCallTestMetaData) + + const numSlots = 42 + rander := testhelpers.NewPseudoRandomDataSource(t, 0) + readData := multicallEmptyArgs() + writeRandAData := multicallEmptyArgs() + writeRandBData := multicallEmptyArgs() + writeZeroData := multicallEmptyArgs() + for i := 0; i < numSlots; i++ { + slot := rander.GetHash() + readData = multicallAppendLoad(readData, slot, false) + writeRandAData = multicallAppendStore(writeRandAData, slot, rander.GetHash(), false) + writeRandBData = multicallAppendStore(writeRandBData, slot, rander.GetHash(), false) + writeZeroData = multicallAppendStore(writeZeroData, slot, common.Hash{}, false) + } + + for _, tc := range []struct { + name string + data []byte + }{ + {"initialWrite", writeRandAData}, + {"read", readData}, + {"writeAgain", writeRandBData}, + {"delete", writeZeroData}, + {"readZeros", readData}, + {"writeAgainAgain", writeRandAData}, + } { + t.Run(tc.name, func(t *testing.T) { + compareGasUsage(t, ctx, l2client, l2info, evmMulticall, stylusMulticall, tc.data, nil, + compareGasSum, 0, compareGasPair{vm.SSTORE, "storage_flush_cache"}, + compareGasPair{vm.SLOAD, "storage_load_bytes32"}) + }) + } +} + +func TestProgramLogCost(t *testing.T) { + ctx, l2info, l2client, auth := setupGasCostTest(t) + + stylusProgram := deployWasm(t, ctx, auth, l2client, rustFile("hostio-test")) + evmProgram := deployEvmContract(t, ctx, auth, l2client, mocksgen.HostioTestMetaData) + packer, _ := util.NewCallParser(mocksgen.HostioTestABI, "emitLog") + + for ntopics := int8(0); ntopics < 5; ntopics++ { + for _, dataSize := range []uint64{10, 100, 1000} { + name := fmt.Sprintf("emitLog%dData%d", ntopics, dataSize) + t.Run(name, func(t *testing.T) { + args := []any{ + testhelpers.RandomSlice(dataSize), + ntopics, + } + for t := 0; t < 4; t++ { + args = append(args, testhelpers.RandomHash()) + } + data, err := packer(args...) + Require(t, err) + opcode := vm.LOG0 + vm.OpCode(ntopics) + compareGasUsage(t, ctx, l2client, l2info, evmProgram, stylusProgram, data, nil, + compareGasForEach, 0, compareGasPair{opcode, "emit_log"}) + }) + } + } + +} + +func TestProgramCallCost(t *testing.T) { + ctx, l2info, l2client, auth := setupGasCostTest(t) + + stylusMulticall := deployWasm(t, ctx, auth, l2client, rustFile("multicall")) + evmMulticall := deployEvmContract(t, ctx, auth, l2client, mocksgen.MultiCallTestMetaData) + otherStylusProgram := deployWasm(t, ctx, auth, l2client, rustFile("hostio-test")) + otherEvmProgram := deployEvmContract(t, ctx, auth, l2client, mocksgen.HostioTestMetaData) + packer, _ := util.NewCallParser(mocksgen.HostioTestABI, "msgValue") + otherData, err := packer() + Require(t, err) + + for _, pair := range []compareGasPair{ + {vm.CALL, "call_contract"}, + {vm.DELEGATECALL, "delegate_call_contract"}, + {vm.STATICCALL, "static_call_contract"}, + } { + t.Run(pair.hostio+"/burnGas", func(t *testing.T) { + arbTest := common.HexToAddress("0x0000000000000000000000000000000000000069") + burnArbGas, _ := util.NewCallParser(precompilesgen.ArbosTestABI, "burnArbGas") + burnData, err := burnArbGas(big.NewInt(0)) + Require(t, err) + data := argsForMulticall(pair.opcode, arbTest, nil, burnData) + compareGasUsage(t, ctx, l2client, l2info, evmMulticall, stylusMulticall, data, nil, compareGasForEach, 0, pair) + }) + + t.Run(pair.hostio+"/evmContract", func(t *testing.T) { + data := argsForMulticall(pair.opcode, otherEvmProgram, nil, otherData) + compareGasUsage(t, ctx, l2client, l2info, evmMulticall, stylusMulticall, data, nil, compareGasForEach, 0, pair, + compareGasPair{vm.RETURNDATACOPY, "read_return_data"}) // also test read_return_data + }) + + t.Run(pair.hostio+"/stylusContract", func(t *testing.T) { + data := argsForMulticall(pair.opcode, otherStylusProgram, nil, otherData) + compareGasUsage(t, ctx, l2client, l2info, evmMulticall, stylusMulticall, data, nil, compareGasForEach, 0, pair, + compareGasPair{vm.RETURNDATACOPY, "read_return_data"}) // also test read_return_data + }) + + t.Run(pair.hostio+"/multipleTimes", func(t *testing.T) { + data := multicallEmptyArgs() + for i := 0; i < 9; i++ { + data = multicallAppend(data, pair.opcode, otherEvmProgram, otherData) + } + compareGasUsage(t, ctx, l2client, l2info, evmMulticall, stylusMulticall, data, nil, compareGasForEach, 0, pair) + }) + } + + t.Run("call_contract/evmContractWithValue", func(t *testing.T) { + value := big.NewInt(1000) + data := argsForMulticall(vm.CALL, otherEvmProgram, value, otherData) + compareGasUsage(t, ctx, l2client, l2info, evmMulticall, stylusMulticall, data, value, compareGasForEach, 0, compareGasPair{vm.CALL, "call_contract"}) + }) +} + +func TestProgramCreateCost(t *testing.T) { + ctx, l2info, l2client, auth := setupGasCostTest(t) + + stylusCreate := deployWasm(t, ctx, auth, l2client, rustFile("create")) + evmCreate := deployEvmContract(t, ctx, auth, l2client, mocksgen.CreateTestMetaData) + deployCode := common.FromHex(mocksgen.ProgramTestMetaData.Bin) + + t.Run("create1", func(t *testing.T) { + data := []byte{0x01} + data = append(data, (common.Hash{}).Bytes()...) // endowment + data = append(data, deployCode...) + compareGasUsage(t, ctx, l2client, l2info, evmCreate, stylusCreate, data, nil, + compareGasForEach, 0, compareGasPair{vm.CREATE, "create1"}) + }) + + t.Run("create2", func(t *testing.T) { + data := []byte{0x02} + data = append(data, (common.Hash{}).Bytes()...) // endowment + data = append(data, (common.HexToHash("beef")).Bytes()...) // salt + data = append(data, deployCode...) + compareGasUsage(t, ctx, l2client, l2info, evmCreate, stylusCreate, data, nil, + compareGasForEach, 0, compareGasPair{vm.CREATE2, "create2"}) + }) +} + +func TestProgramKeccakCost(t *testing.T) { + ctx, l2info, l2client, auth := setupGasCostTest(t) + + stylusProgram := deployWasm(t, ctx, auth, l2client, rustFile("hostio-test")) + evmProgram := deployEvmContract(t, ctx, auth, l2client, mocksgen.HostioTestMetaData) + packer, _ := util.NewCallParser(mocksgen.HostioTestABI, "keccak") + + for i := 1; i < 5; i++ { + size := uint64(math.Pow10(i)) + name := fmt.Sprintf("keccak%d", size) + t.Run(name, func(t *testing.T) { + preImage := testhelpers.RandomSlice(size) + preImage[len(preImage)-1] = 0 + data, err := packer(preImage) + Require(t, err) + const maxDiff = 1.1 + compareGasUsage(t, ctx, l2client, l2info, evmProgram, stylusProgram, data, nil, + compareGasForEach, maxDiff, compareGasPair{vm.KECCAK256, "native_keccak256"}) + }) + } +} + +func setupGasCostTest(t *testing.T) (ctx context.Context, l2info *BlockchainTestInfo, l2client *ethclient.Client, auth bind.TransactOpts) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + builder := NewNodeBuilder(ctx).DefaultConfig(t, true) + cleanup := builder.Build(t) + t.Cleanup(cleanup) + l2info = builder.L2Info + l2client = builder.L2.Client + auth = builder.L2Info.GetDefaultTransactOpts("Owner", ctx) + return +} + +// deployEvmContract deploys an Evm contract and return its address. +func deployEvmContract(t *testing.T, ctx context.Context, auth bind.TransactOpts, client *ethclient.Client, metadata *bind.MetaData) common.Address { + t.Helper() + parsed, err := metadata.GetAbi() + Require(t, err) + address, tx, _, err := bind.DeployContract(&auth, *parsed, common.FromHex(metadata.Bin), client) + Require(t, err) + _, err = EnsureTxSucceeded(ctx, client, tx) + Require(t, err) + return address +} + +type compareGasPair struct { + opcode vm.OpCode + hostio string +} + +type compareGasMode int + +const ( + compareGasForEach compareGasMode = iota + compareGasSum +) + +// compareGasUsage calls an EVM and a Wasm contract passing the same data and the same value. +// Then, it ensures the given opcodes and hostios cost roughly the same amount of gas. +func compareGasUsage( + t *testing.T, + ctx context.Context, + client *ethclient.Client, + info *BlockchainTestInfo, + evmContract common.Address, + stylusContract common.Address, + txData []byte, + txValue *big.Int, + mode compareGasMode, + maxAllowedDifference float64, + pairs ...compareGasPair, +) { + if evmContract == stylusContract { + Fatal(t, "evm and stylus contract are the same") + } + + const txGas uint64 = 32_000_000 + tx := info.PrepareTxTo("Owner", &evmContract, txGas, txValue, txData) + evmGas := sendAndEnsureTransaction(t, ctx, client, tx) + evmGasUsage, err := evmOpcodesGasUsage(ctx, client.Client(), tx) + Require(t, err) + + tx = info.PrepareTxTo("Owner", &stylusContract, txGas, txValue, txData) + stylusGas := sendAndEnsureTransaction(t, ctx, client, tx) + stylusGasUsage, err := stylusHostiosGasUsage(ctx, client.Client(), tx) + Require(t, err) + + t.Logf("evm total usage: %v - stylus total usage: %v", evmGas, stylusGas) + + for i := range pairs { + opcode := pairs[i].opcode + hostio := pairs[i].hostio + switch mode { + case compareGasForEach: + if len(evmGasUsage[opcode]) != len(stylusGasUsage[hostio]) { + Fatal(t, "mismatch between hostios and opcodes", evmGasUsage, stylusGasUsage) + } + for i := range evmGasUsage[opcode] { + opcodeGas := evmGasUsage[opcode][i] + hostioGas := stylusGasUsage[hostio][i] + t.Logf("evm %v usage: %v - stylus %v usage: %v", opcode, opcodeGas, hostio, hostioGas) + checkPercentDiff(t, float64(opcodeGas), hostioGas, maxAllowedDifference) + } + case compareGasSum: + evmSum := float64(0) + stylusSum := float64(0) + for i := range evmGasUsage[opcode] { + evmSum += float64(evmGasUsage[opcode][i]) + stylusSum += stylusGasUsage[hostio][i] + } + t.Logf("evm %v usage: %v - stylus %v usage: %v", opcode, evmSum, hostio, stylusSum) + checkPercentDiff(t, evmSum, stylusSum, maxAllowedDifference) + } + } +} + +func evmOpcodesGasUsage(ctx context.Context, rpcClient rpc.ClientInterface, tx *types.Transaction) ( + map[vm.OpCode][]uint64, error) { + + var result logger.ExecutionResult + err := rpcClient.CallContext(ctx, &result, "debug_traceTransaction", tx.Hash(), nil) + if err != nil { + return nil, fmt.Errorf("failed to trace evm call: %w", err) + } + + gasUsage := map[vm.OpCode][]uint64{} + for i := range result.StructLogs { + op := vm.StringToOp(result.StructLogs[i].Op) + gasUsed := uint64(0) + if op == vm.CALL || op == vm.STATICCALL || op == vm.DELEGATECALL || op == vm.CREATE || op == vm.CREATE2 { + // For the CALL* opcodes, the GasCost in the tracer represents the gas sent + // to the callee contract, which is 63/64 of the remaining gas. This happens + // because the tracer is evaluated before the call is executed, so the EVM + // doesn't know how much gas will being used. + // + // In the case of the Stylus tracer, the trace is emitted after the + // execution, so the EndInk field is set to the ink after the call returned. + // Hence, it also includes the ink spent by the callee contract. + // + // To make a precise comparison between the EVM and Stylus, we modify the + // EVM measurement to include the gas spent by the callee contract. To do + // so, we go through the opcodes after CALL until we find the first opcode + // in the caller's depth. Then, we subtract the gas before the call by the + // gas after the call returned. + var gasAfterCall uint64 + for j := i + 1; j < len(result.StructLogs); j++ { + if result.StructLogs[j].Depth == result.StructLogs[i].Depth { + // back to the original call + gasAfterCall = result.StructLogs[j].Gas + result.StructLogs[j].GasCost + break + } + } + if gasAfterCall == 0 { + return nil, fmt.Errorf("malformed log: didn't get back to call original depth") + } + if i == 0 { + return nil, fmt.Errorf("malformed log: call is first opcode") + } + gasUsed = result.StructLogs[i-1].Gas - gasAfterCall + } else { + gasUsed = result.StructLogs[i].GasCost + } + gasUsage[op] = append(gasUsage[op], gasUsed) + } + return gasUsage, nil +} + +func stylusHostiosGasUsage(ctx context.Context, rpcClient rpc.ClientInterface, tx *types.Transaction) ( + map[string][]float64, error) { + + traceOpts := struct { + Tracer string `json:"tracer"` + }{ + Tracer: "stylusTracer", + } + var result []gethexec.HostioTraceInfo + err := rpcClient.CallContext(ctx, &result, "debug_traceTransaction", tx.Hash(), traceOpts) + if err != nil { + return nil, fmt.Errorf("failed to trace stylus call: %w", err) + } + + const InkPerGas = 10000 + gasUsage := map[string][]float64{} + for _, hostioLog := range result { + gasCost := float64(hostioLog.StartInk-hostioLog.EndInk) / InkPerGas + gasUsage[hostioLog.Name] = append(gasUsage[hostioLog.Name], gasCost) + } + return gasUsage, nil +} + +// checkPercentDiff checks whether the two values are close enough. +func checkPercentDiff(t *testing.T, a, b float64, maxAllowedDifference float64) { + t.Helper() + if maxAllowedDifference == 0 { + maxAllowedDifference = 0.25 + } + percentageDifference := math.Abs(a-b) / ((a + b) / 2) + if percentageDifference > maxAllowedDifference { + Fatal(t, fmt.Sprintf("gas usages are too different; got %v, max allowed is %v", percentageDifference, maxAllowedDifference)) + } +} + +// sendAndEnsureTransaction sends a transaction, ensures it succeed, and returns the total gas cost. +func sendAndEnsureTransaction(t *testing.T, ctx context.Context, client *ethclient.Client, tx *types.Transaction) uint64 { + t.Helper() + err := client.SendTransaction(ctx, tx) + Require(t, err) + receipt, err := EnsureTxSucceeded(ctx, client, tx) + Require(t, err) + return receipt.GasUsedForL2() +} From a528c6e025984fb14dfebf5e6b68eeb7416c89ed Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Fri, 27 Sep 2024 15:47:50 -0300 Subject: [PATCH 283/489] Improve account_code hostio gas test --- arbitrator/stylus/tests/hostio-test/src/main.rs | 8 +++++++- contracts | 2 +- system_tests/program_gas_test.go | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/arbitrator/stylus/tests/hostio-test/src/main.rs b/arbitrator/stylus/tests/hostio-test/src/main.rs index 5aa465598e..17a5d10266 100644 --- a/arbitrator/stylus/tests/hostio-test/src/main.rs +++ b/arbitrator/stylus/tests/hostio-test/src/main.rs @@ -77,7 +77,13 @@ impl HostioTest { } fn account_code(account: Address) -> Result> { - Ok(account.code()) + let mut size = 10000; + let mut code = vec![0; size]; + unsafe { + size = hostio::account_code(account.as_ptr(), 0, size, code.as_mut_ptr()); + } + code.resize(size, 0); + Ok(code) } fn account_code_size(account: Address) -> Result { diff --git a/contracts b/contracts index cde7f96486..b5fd6ecbaa 160000 --- a/contracts +++ b/contracts @@ -1 +1 @@ -Subproject commit cde7f96486cc21befc9a05fc3b27a7d6ebb5d5f8 +Subproject commit b5fd6ecbaa0bc37f946f07278c4c543d992b095b diff --git a/system_tests/program_gas_test.go b/system_tests/program_gas_test.go index 48db268907..cdc75345cb 100644 --- a/system_tests/program_gas_test.go +++ b/system_tests/program_gas_test.go @@ -41,7 +41,7 @@ func TestProgramSimpleCost(t *testing.T) { {hostio: "transient_store_bytes32", opcode: vm.TSTORE, params: []any{common.HexToHash("dead"), common.HexToHash("beef")}}, {hostio: "return_data_size", opcode: vm.RETURNDATASIZE, maxDiff: 1.0}, {hostio: "account_balance", opcode: vm.BALANCE, params: []any{l2info.GetAddress("Owner")}}, - {hostio: "account_code", opcode: vm.EXTCODECOPY, params: []any{otherProgram}, maxDiff: 2}, + {hostio: "account_code", opcode: vm.EXTCODECOPY, params: []any{otherProgram}}, {hostio: "account_code_size", opcode: vm.EXTCODESIZE, params: []any{otherProgram}}, {hostio: "account_codehash", opcode: vm.EXTCODEHASH, params: []any{otherProgram}}, {hostio: "evm_gas_left", opcode: vm.GAS, maxDiff: 1.0}, From 91a974f18b98eb943946475496f06c71fd9ce71d Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Fri, 27 Sep 2024 01:09:16 +0800 Subject: [PATCH 284/489] chore: remove unused variable --- arbnode/transaction_streamer.go | 4 ++-- system_tests/eth_sync_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arbnode/transaction_streamer.go b/arbnode/transaction_streamer.go index 24a0564b97..38b1c003db 100644 --- a/arbnode/transaction_streamer.go +++ b/arbnode/transaction_streamer.go @@ -1140,7 +1140,7 @@ func (s *TransactionStreamer) storeResult( // exposed for testing // return value: true if should be called again immediately -func (s *TransactionStreamer) ExecuteNextMsg(ctx context.Context, exec execution.ExecutionSequencer) bool { +func (s *TransactionStreamer) ExecuteNextMsg(ctx context.Context) bool { if ctx.Err() != nil { return false } @@ -1212,7 +1212,7 @@ func (s *TransactionStreamer) ExecuteNextMsg(ctx context.Context, exec execution } func (s *TransactionStreamer) executeMessages(ctx context.Context, ignored struct{}) time.Duration { - if s.ExecuteNextMsg(ctx, s.exec) { + if s.ExecuteNextMsg(ctx) { return 0 } return s.config().ExecuteMessageLoopDelay diff --git a/system_tests/eth_sync_test.go b/system_tests/eth_sync_test.go index 1f07f7c45f..ce9994fb1e 100644 --- a/system_tests/eth_sync_test.go +++ b/system_tests/eth_sync_test.go @@ -71,7 +71,7 @@ func TestEthSyncing(t *testing.T) { if progress == nil { Fatal(t, "eth_syncing returned nil but shouldn't have") } - for testClientB.ConsensusNode.TxStreamer.ExecuteNextMsg(ctx, testClientB.ExecNode) { + for testClientB.ConsensusNode.TxStreamer.ExecuteNextMsg(ctx) { } progress, err = testClientB.Client.SyncProgress(ctx) Require(t, err) From c550c7ed5bc17b85b181e7fb62a7b59d8315590b Mon Sep 17 00:00:00 2001 From: Pepper Lebeck-Jobe Date: Sat, 28 Sep 2024 16:50:59 +0300 Subject: [PATCH 285/489] Switch to funcional options for the inputs.Writer This change also saves the stack argument passed to the NewStatelessBlockValidator function to a member of StatelessBlockValidator so that the BlockValidator can use the `stack.InstanceDir()` method to set a reasonable base directory for storing validation inputs as json files. --- arbitrator/wasm-libraries/Cargo.lock | 122 +++++++++++++++++++++++++++ staker/block_validator.go | 6 +- staker/stateless_block_validator.go | 2 + system_tests/common_test.go | 3 +- validator/inputs/writer.go | 70 ++++++++------- validator/inputs/writer_test.go | 29 ++++--- 6 files changed, 187 insertions(+), 45 deletions(-) diff --git a/arbitrator/wasm-libraries/Cargo.lock b/arbitrator/wasm-libraries/Cargo.lock index be2bd828f3..a5a066e5c9 100644 --- a/arbitrator/wasm-libraries/Cargo.lock +++ b/arbitrator/wasm-libraries/Cargo.lock @@ -106,6 +106,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bincode" version = "1.3.3" @@ -218,6 +224,15 @@ dependencies = [ "rand_pcg", ] +[[package]] +name = "cc" +version = "1.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9540e661f81799159abee814118cc139a2004b3a3aa3ea37724a1b66530b90e0" +dependencies = [ + "shlex", +] + [[package]] name = "cfg-if" version = "0.1.10" @@ -264,6 +279,12 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "cpufeatures" version = "0.2.12" @@ -324,6 +345,16 @@ dependencies = [ "syn 2.0.72", ] +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", + "serde", +] + [[package]] name = "derivative" version = "2.2.0" @@ -614,6 +645,15 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +[[package]] +name = "js-sys" +version = "0.3.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "keccak" version = "0.1.5" @@ -651,6 +691,12 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + [[package]] name = "lru" version = "0.12.4" @@ -857,6 +903,12 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "proc-macro-crate" version = "3.1.0" @@ -1214,6 +1266,12 @@ dependencies = [ "keccak", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "simdutf8" version = "0.1.4" @@ -1509,6 +1567,61 @@ dependencies = [ "wee_alloc", ] +[[package]] +name = "wasm-bindgen" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +dependencies = [ + "cfg-if 1.0.0", + "once_cell", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.72", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" + [[package]] name = "wasm-encoder" version = "0.215.0" @@ -1599,6 +1712,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-targets" version = "0.52.6" diff --git a/staker/block_validator.go b/staker/block_validator.go index 30df20a2d2..5a1f123693 100644 --- a/staker/block_validator.go +++ b/staker/block_validator.go @@ -288,11 +288,13 @@ func NewBlockValidator( fatalErr: fatalErr, prevBatchCache: make(map[uint64][]byte), } - valInputsWriter, err := inputs.NewWriter() + valInputsWriter, err := inputs.NewWriter( + inputs.WithBaseDir(ret.stack.InstanceDir()), + inputs.WithSlug("BlockValidator")) if err != nil { return nil, err } - ret.validationInputsWriter = valInputsWriter.SetSlug("BlockValidator") + ret.validationInputsWriter = valInputsWriter if !config().Dangerous.ResetBlockValidation { validated, err := ret.ReadLastValidatedInfo() if err != nil { diff --git a/staker/stateless_block_validator.go b/staker/stateless_block_validator.go index fa34842dc9..9257c5582a 100644 --- a/staker/stateless_block_validator.go +++ b/staker/stateless_block_validator.go @@ -41,6 +41,7 @@ type StatelessBlockValidator struct { streamer TransactionStreamerInterface db ethdb.Database dapReaders []daprovider.Reader + stack *node.Node } type BlockValidatorRegistrer interface { @@ -265,6 +266,7 @@ func NewStatelessBlockValidator( db: arbdb, dapReaders: dapReaders, execSpawners: executionSpawners, + stack: stack, }, nil } diff --git a/system_tests/common_test.go b/system_tests/common_test.go index ed098351e5..d5f65de43e 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -1712,9 +1712,8 @@ func recordBlock(t *testing.T, block uint64, builder *NodeBuilder) { break } } - validationInputsWriter, err := inputs.NewWriter() + validationInputsWriter, err := inputs.NewWriter(inputs.WithSlug(t.Name())) Require(t, err) - validationInputsWriter.SetSlug(t.Name()) inputJson, err := builder.L2.ConsensusNode.StatelessBlockValidator.ValidationInputsAt(ctx, inboxPos, rawdb.TargetWavm) if err != nil { Fatal(t, "failed to get validation inputs", block, err) diff --git a/validator/inputs/writer.go b/validator/inputs/writer.go index b8160a4fa5..1fd23dbb40 100644 --- a/validator/inputs/writer.go +++ b/validator/inputs/writer.go @@ -3,6 +3,7 @@ package inputs import ( "fmt" "os" + "path/filepath" "time" "github.com/offchainlabs/nitro/validator/server_api" @@ -43,6 +44,9 @@ type Writer struct { useTimestampDir bool } +// WriterOption is a function that configures a Writer. +type WriterOption func(*Writer) + // Clock is an interface for getting the current time. type Clock interface { Now() time.Time @@ -55,73 +59,81 @@ func (realClock) Now() time.Time { } // NewWriter creates a new Writer with default settings. -func NewWriter() (*Writer, error) { +func NewWriter(options ...WriterOption) (*Writer, error) { homeDir, err := os.UserHomeDir() if err != nil { return nil, err } baseDir := fmt.Sprintf("%s/.arbitrum/validation-inputs", homeDir) - return &Writer{ + w := &Writer{ clock: realClock{}, baseDir: baseDir, slug: "", - useTimestampDir: true}, nil + useTimestampDir: true, + } + for _, o := range options { + o(w) + } + return w, nil } -// SetClockForTesting sets the clock used by the Writer. +// withTestClock configures the Writer to use the given clock. // // This is only intended for testing. -func (w *Writer) SetClockForTesting(clock Clock) *Writer { - w.clock = clock - return w +func withTestClock(clock Clock) WriterOption { + return func(w *Writer) { + w.clock = clock + } } -// SetSlug configures the Writer to use the given slug as a directory name. -func (w *Writer) SetSlug(slug string) *Writer { - w.slug = slug - return w +// WithSlug configures the Writer to use the given slug as a directory name. +func WithSlug(slug string) WriterOption { + return func(w *Writer) { + w.slug = slug + } } -// ClearSlug clears the slug configuration. +// WithoutSlug clears the slug configuration. // -// This is equivalent to calling SetSlug("") but is more readable. -func (w *Writer) ClearSlug() *Writer { - w.slug = "" - return w +// This is equivalent to the WithSlug("") option but is more readable. +func WithoutSlug() WriterOption { + return WithSlug("") } -// SetBaseDir configures the Writer to use the given base directory. -func (w *Writer) SetBaseDir(baseDir string) *Writer { - w.baseDir = baseDir - return w +// WithBaseDir configures the Writer to use the given base directory. +func WithBaseDir(baseDir string) WriterOption { + return func(w *Writer) { + w.baseDir = baseDir + } } -// SetUseTimestampDir controls the addition of a timestamp directory. -func (w *Writer) SetUseTimestampDir(useTimestampDir bool) *Writer { - w.useTimestampDir = useTimestampDir - return w +// WithTimestampDirEnabled controls the addition of a timestamp directory. +func WithTimestampDirEnabled(useTimestampDir bool) WriterOption { + return func(w *Writer) { + w.useTimestampDir = useTimestampDir + } } // Write writes the given InputJSON to a file in JSON format. -func (w *Writer) Write(inputs *server_api.InputJSON) error { +func (w *Writer) Write(json *server_api.InputJSON) error { dir := w.baseDir if w.slug != "" { - dir = fmt.Sprintf("%s/%s", dir, w.slug) + dir = filepath.Join(dir, w.slug) } if w.useTimestampDir { t := w.clock.Now() tStr := t.Format("20060102_150405") - dir = fmt.Sprintf("%s/%s", dir, tStr) + dir = filepath.Join(dir, tStr) } if err := os.MkdirAll(dir, 0700); err != nil { return err } - contents, err := inputs.Marshal() + contents, err := json.Marshal() if err != nil { return err } if err = os.WriteFile( - fmt.Sprintf("%s/block_inputs_%d.json", dir, inputs.Id), + fmt.Sprintf("%s/block_inputs_%d.json", dir, json.Id), contents, 0600); err != nil { return err } diff --git a/validator/inputs/writer_test.go b/validator/inputs/writer_test.go index 5e80b9aa3a..59cb63dae7 100644 --- a/validator/inputs/writer_test.go +++ b/validator/inputs/writer_test.go @@ -33,13 +33,14 @@ func (c fakeClock) Now() time.Time { } func TestWriting(t *testing.T) { - w, err := NewWriter() + dir := t.TempDir() + w, err := NewWriter( + withTestClock(fakeClock{now: time.Date(2021, 1, 2, 3, 4, 5, 0, time.UTC)}), + WithBaseDir(dir), + ) if err != nil { t.Fatal(err) } - w.SetClockForTesting(fakeClock{now: time.Date(2021, 1, 2, 3, 4, 5, 0, time.UTC)}) - dir := t.TempDir() - w.SetBaseDir(dir) err = w.Write(&server_api.InputJSON{Id: 24601}) if err != nil { t.Fatal(err) @@ -51,13 +52,15 @@ func TestWriting(t *testing.T) { } func TestWritingWithSlug(t *testing.T) { - w, err := NewWriter() + dir := t.TempDir() + w, err := NewWriter( + withTestClock(fakeClock{now: time.Date(2021, 1, 2, 3, 4, 5, 0, time.UTC)}), + WithBaseDir(dir), + WithSlug("foo"), + ) if err != nil { t.Fatal(err) } - w.SetClockForTesting(fakeClock{now: time.Date(2021, 1, 2, 3, 4, 5, 0, time.UTC)}) - dir := t.TempDir() - w.SetBaseDir(dir).SetSlug("foo") err = w.Write(&server_api.InputJSON{Id: 24601}) if err != nil { t.Fatal(err) @@ -69,13 +72,15 @@ func TestWritingWithSlug(t *testing.T) { } func TestWritingWithoutTimestampDir(t *testing.T) { - w, err := NewWriter() + dir := t.TempDir() + w, err := NewWriter( + withTestClock(fakeClock{now: time.Date(2021, 1, 2, 3, 4, 5, 0, time.UTC)}), + WithBaseDir(dir), + WithTimestampDirEnabled(false), + ) if err != nil { t.Fatal(err) } - w.SetClockForTesting(fakeClock{now: time.Date(2021, 1, 2, 3, 4, 5, 0, time.UTC)}) - dir := t.TempDir() - w.SetBaseDir(dir).SetUseTimestampDir(false) err = w.Write(&server_api.InputJSON{Id: 24601}) if err != nil { t.Fatal(err) From ea47d46d7557d3d600968c46d37674381d710081 Mon Sep 17 00:00:00 2001 From: Pepper Lebeck-Jobe Date: Sat, 28 Sep 2024 16:58:05 +0300 Subject: [PATCH 286/489] Fix two last instances of system-specific paths --- validator/inputs/writer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/validator/inputs/writer.go b/validator/inputs/writer.go index 1fd23dbb40..a45e584f52 100644 --- a/validator/inputs/writer.go +++ b/validator/inputs/writer.go @@ -64,7 +64,7 @@ func NewWriter(options ...WriterOption) (*Writer, error) { if err != nil { return nil, err } - baseDir := fmt.Sprintf("%s/.arbitrum/validation-inputs", homeDir) + baseDir := filepath.Join(homeDir, ".arbitrum", "validation-inputs") w := &Writer{ clock: realClock{}, baseDir: baseDir, @@ -133,7 +133,7 @@ func (w *Writer) Write(json *server_api.InputJSON) error { return err } if err = os.WriteFile( - fmt.Sprintf("%s/block_inputs_%d.json", dir, json.Id), + filepath.Join(dir, fmt.Sprintf("block_inputs_%d.json", json.Id)), contents, 0600); err != nil { return err } From 14d57e1da2482b8fb145eb94d6e806876cf63a58 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 30 Sep 2024 11:08:38 -0300 Subject: [PATCH 287/489] DisableStylusCacheMetricsCollection flag --- execution/gethexec/blockchain.go | 30 ++++++++++++----------- execution/gethexec/executionengine.go | 34 +++++++++++++++++++-------- execution/gethexec/node.go | 3 +++ system_tests/common_test.go | 27 +++++++++++---------- 4 files changed, 57 insertions(+), 37 deletions(-) diff --git a/execution/gethexec/blockchain.go b/execution/gethexec/blockchain.go index 9b0c1a6f2f..fda8f49093 100644 --- a/execution/gethexec/blockchain.go +++ b/execution/gethexec/blockchain.go @@ -26,20 +26,21 @@ import ( ) type CachingConfig struct { - Archive bool `koanf:"archive"` - BlockCount uint64 `koanf:"block-count"` - BlockAge time.Duration `koanf:"block-age"` - TrieTimeLimit time.Duration `koanf:"trie-time-limit"` - TrieDirtyCache int `koanf:"trie-dirty-cache"` - TrieCleanCache int `koanf:"trie-clean-cache"` - SnapshotCache int `koanf:"snapshot-cache"` - DatabaseCache int `koanf:"database-cache"` - SnapshotRestoreGasLimit uint64 `koanf:"snapshot-restore-gas-limit"` - MaxNumberOfBlocksToSkipStateSaving uint32 `koanf:"max-number-of-blocks-to-skip-state-saving"` - MaxAmountOfGasToSkipStateSaving uint64 `koanf:"max-amount-of-gas-to-skip-state-saving"` - StylusLRUCacheCapacity uint32 `koanf:"stylus-lru-cache-capacity"` - StateScheme string `koanf:"state-scheme"` - StateHistory uint64 `koanf:"state-history"` + Archive bool `koanf:"archive"` + BlockCount uint64 `koanf:"block-count"` + BlockAge time.Duration `koanf:"block-age"` + TrieTimeLimit time.Duration `koanf:"trie-time-limit"` + TrieDirtyCache int `koanf:"trie-dirty-cache"` + TrieCleanCache int `koanf:"trie-clean-cache"` + SnapshotCache int `koanf:"snapshot-cache"` + DatabaseCache int `koanf:"database-cache"` + SnapshotRestoreGasLimit uint64 `koanf:"snapshot-restore-gas-limit"` + MaxNumberOfBlocksToSkipStateSaving uint32 `koanf:"max-number-of-blocks-to-skip-state-saving"` + MaxAmountOfGasToSkipStateSaving uint64 `koanf:"max-amount-of-gas-to-skip-state-saving"` + StylusLRUCacheCapacity uint32 `koanf:"stylus-lru-cache-capacity"` + DisableStylusCacheMetricsCollection bool `koanf:"disable-stylus-cache-metrics-collection"` + StateScheme string `koanf:"state-scheme"` + StateHistory uint64 `koanf:"state-history"` } func CachingConfigAddOptions(prefix string, f *flag.FlagSet) { @@ -55,6 +56,7 @@ func CachingConfigAddOptions(prefix string, f *flag.FlagSet) { f.Uint32(prefix+".max-number-of-blocks-to-skip-state-saving", DefaultCachingConfig.MaxNumberOfBlocksToSkipStateSaving, "maximum number of blocks to skip state saving to persistent storage (archive node only) -- warning: this option seems to cause issues") f.Uint64(prefix+".max-amount-of-gas-to-skip-state-saving", DefaultCachingConfig.MaxAmountOfGasToSkipStateSaving, "maximum amount of gas in blocks to skip saving state to Persistent storage (archive node only) -- warning: this option seems to cause issues") f.Uint32(prefix+".stylus-lru-cache-capacity", DefaultCachingConfig.StylusLRUCacheCapacity, "capacity, in megabytes, of the LRU cache that keeps initialized stylus programs") + f.Bool(prefix+".disable-stylus-cache-metrics-collection", DefaultCachingConfig.DisableStylusCacheMetricsCollection, "disable metrics collection for the stylus cache") f.String(prefix+".state-scheme", DefaultCachingConfig.StateScheme, "scheme to use for state trie storage (hash, path)") f.Uint64(prefix+".state-history", DefaultCachingConfig.StateHistory, "number of recent blocks to retain state history for (path state-scheme only)") } diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index a0f3a2f59a..b36340757a 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -87,6 +87,8 @@ type ExecutionEngine struct { reorgSequencing bool + disableStylusCacheMetricsCollection bool + prefetchBlock bool cachedL1PriceData *L1PriceData @@ -212,6 +214,16 @@ func (s *ExecutionEngine) EnableReorgSequencing() { s.reorgSequencing = true } +func (s *ExecutionEngine) DisableStylusCacheMetricsCollection() { + if s.Started() { + panic("trying to disable stylus cache metrics collection after start") + } + if s.disableStylusCacheMetricsCollection { + panic("trying to disable stylus cache metrics collection when already set") + } + s.disableStylusCacheMetricsCollection = true +} + func (s *ExecutionEngine) EnablePrefetchBlock() { if s.Started() { panic("trying to enable prefetch block after start") @@ -963,15 +975,17 @@ func (s *ExecutionEngine) Start(ctx_in context.Context) { } } }) - // periodically update stylus lru cache metrics - s.LaunchThread(func(ctx context.Context) { - for { - select { - case <-ctx.Done(): - return - case <-time.After(time.Minute): - programs.GetWasmLruCacheMetrics() + if !s.disableStylusCacheMetricsCollection { + // periodically update stylus lru cache metrics + s.LaunchThread(func(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case <-time.After(time.Minute): + programs.GetWasmLruCacheMetrics() + } } - } - }) + }) + } } diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index 5a1efc6d08..1b8b756502 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -187,6 +187,9 @@ func CreateExecutionNode( if config.EnablePrefetchBlock { execEngine.EnablePrefetchBlock() } + if config.Caching.DisableStylusCacheMetricsCollection { + execEngine.DisableStylusCacheMetricsCollection() + } if err != nil { return nil, err } diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 5902a670ba..6a4e551906 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -155,19 +155,20 @@ func (tc *TestClient) EnsureTxSucceededWithTimeout(transaction *types.Transactio } var TestCachingConfig = gethexec.CachingConfig{ - Archive: false, - BlockCount: 128, - BlockAge: 30 * time.Minute, - TrieTimeLimit: time.Hour, - TrieDirtyCache: 1024, - TrieCleanCache: 600, - SnapshotCache: 400, - DatabaseCache: 2048, - SnapshotRestoreGasLimit: 300_000_000_000, - MaxNumberOfBlocksToSkipStateSaving: 0, - MaxAmountOfGasToSkipStateSaving: 0, - StylusLRUCacheCapacity: 0, - StateScheme: env.GetTestStateScheme(), + Archive: false, + BlockCount: 128, + BlockAge: 30 * time.Minute, + TrieTimeLimit: time.Hour, + TrieDirtyCache: 1024, + TrieCleanCache: 600, + SnapshotCache: 400, + DatabaseCache: 2048, + SnapshotRestoreGasLimit: 300_000_000_000, + MaxNumberOfBlocksToSkipStateSaving: 0, + MaxAmountOfGasToSkipStateSaving: 0, + StylusLRUCacheCapacity: 0, + DisableStylusCacheMetricsCollection: true, + StateScheme: env.GetTestStateScheme(), } var DefaultTestForwarderConfig = gethexec.ForwarderConfig{ From 72879b79d732f74403a286ebd63c6fbe31ccf292 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Mon, 30 Sep 2024 14:34:22 -0300 Subject: [PATCH 288/489] Update nitro contracts --- contracts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts b/contracts index b5fd6ecbaa..6740300fc1 160000 --- a/contracts +++ b/contracts @@ -1 +1 @@ -Subproject commit b5fd6ecbaa0bc37f946f07278c4c543d992b095b +Subproject commit 6740300fc1da9c1d67764068463ed96d7ca9ec2c From 827755ee0bce0eb990e332427f6943a119558c16 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Mon, 30 Sep 2024 16:24:53 -0300 Subject: [PATCH 289/489] Use Golang version 1.23 --- .github/workflows/arbitrator-ci.yml | 2 +- .github/workflows/ci.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- Dockerfile | 4 ++-- Makefile | 14 ++++++++++---- go.mod | 2 +- 6 files changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/arbitrator-ci.yml b/.github/workflows/arbitrator-ci.yml index 392eb876c0..7829fe4086 100644 --- a/.github/workflows/arbitrator-ci.yml +++ b/.github/workflows/arbitrator-ci.yml @@ -50,7 +50,7 @@ jobs: - name: Install go uses: actions/setup-go@v4 with: - go-version: 1.21.x + go-version: 1.23.x - name: Install custom go-ethereum run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5794580f03..a944f08f40 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,7 +46,7 @@ jobs: - name: Install go uses: actions/setup-go@v4 with: - go-version: 1.21.x + go-version: 1.23.x - name: Install wasm-ld run: | diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 1cde8f06b9..26447947d4 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -73,7 +73,7 @@ jobs: - name: Install go uses: actions/setup-go@v4 with: - go-version: 1.21.x + go-version: 1.23.x - name: Install rust stable uses: dtolnay/rust-toolchain@stable diff --git a/Dockerfile b/Dockerfile index 9138ed30ad..aba5432254 100644 --- a/Dockerfile +++ b/Dockerfile @@ -66,7 +66,7 @@ COPY --from=wasm-libs-builder /workspace/ / FROM wasm-base AS wasm-bin-builder # pinned go version -RUN curl -L https://golang.org/dl/go1.21.10.linux-`dpkg --print-architecture`.tar.gz | tar -C /usr/local -xzf - +RUN curl -L https://golang.org/dl/go1.23.1.linux-`dpkg --print-architecture`.tar.gz | tar -C /usr/local -xzf - COPY ./Makefile ./go.mod ./go.sum ./ COPY ./arbcompress ./arbcompress COPY ./arbos ./arbos @@ -220,7 +220,7 @@ RUN ./download-machine.sh consensus-v30 0xb0de9cb89e4d944ae6023a3b62276e54804c24 RUN ./download-machine.sh consensus-v31 0x260f5fa5c3176a856893642e149cf128b5a8de9f828afec8d11184415dd8dc69 RUN ./download-machine.sh consensus-v32 0x184884e1eb9fefdc158f6c8ac912bb183bf3cf83f0090317e0bc4ac5860baa39 -FROM golang:1.21.10-bookworm AS node-builder +FROM golang:1.23.1-bookworm AS node-builder WORKDIR /workspace ARG version="" ARG datetime="" diff --git a/Makefile b/Makefile index c3cf1a5144..919eb3f4bf 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,12 @@ ifneq ($(origin NITRO_MODIFIED),undefined) GOLANG_LDFLAGS += -X github.com/offchainlabs/nitro/cmd/util/confighelpers.modified=$(NITRO_MODIFIED) endif +# As of Go 1.23, the package github.com/fjl/memsize no longer works because of a restriction added +# by the Go toolchain. The Go 1.23 compiler no longer allows access to runtime symbols via +# go:linkname, which prevents memsize from accessing the Stop-the-World functionality of the Go +# runtime. To solve that we add the following compiler directive. +GOLANG_LDFLAGS += -checklinkname=0 + ifneq ($(origin GOLANG_LDFLAGS),undefined) GOLANG_PARAMS = -ldflags="-extldflags '-ldl' $(GOLANG_LDFLAGS)" endif @@ -226,17 +232,17 @@ test-go: .make/test-go .PHONY: test-go-challenge test-go-challenge: test-go-deps - gotestsum --format short-verbose --no-color=false -- -timeout 120m ./system_tests/... -run TestChallenge -tags challengetest + gotestsum --format short-verbose --no-color=false -- $(GOLANG_PARAMS) -timeout 120m ./system_tests/... -run TestChallenge -tags challengetest @printf $(done) .PHONY: test-go-stylus test-go-stylus: test-go-deps - gotestsum --format short-verbose --no-color=false -- -timeout 120m ./system_tests/... -run TestProgramArbitrator -tags stylustest + gotestsum --format short-verbose --no-color=false -- $(GOLANG_PARAMS) -timeout 120m ./system_tests/... -run TestProgramArbitrator -tags stylustest @printf $(done) .PHONY: test-go-redis test-go-redis: test-go-deps - TEST_REDIS=redis://localhost:6379/0 gotestsum --format short-verbose --no-color=false -- -p 1 -run TestRedis ./system_tests/... ./arbnode/... + TEST_REDIS=redis://localhost:6379/0 gotestsum --format short-verbose --no-color=false -- $(GOLANG_PARAMS) -p 1 -run TestRedis ./system_tests/... ./arbnode/... @printf $(done) .PHONY: test-gen-proofs @@ -541,7 +547,7 @@ contracts/test/prover/proofs/%.json: $(arbitrator_cases)/%.wasm $(prover_bin) @touch $@ .make/test-go: $(DEP_PREDICATE) $(go_source) build-node-deps test-go-deps $(ORDER_ONLY_PREDICATE) .make - gotestsum --format short-verbose --no-color=false + gotestsum --format short-verbose --no-color=false -- $(GOLANG_PARAMS) @touch $@ .make/test-rust: $(DEP_PREDICATE) wasm-ci-build $(ORDER_ONLY_PREDICATE) .make diff --git a/go.mod b/go.mod index da49b0d8b9..e3a6f59ef9 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/offchainlabs/nitro -go 1.21 +go 1.22 replace github.com/VictoriaMetrics/fastcache => ./fastcache From b5c9f52e9b20aaa46f76a222cfd1eaf8767b5bba Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Mon, 30 Sep 2024 16:46:35 -0300 Subject: [PATCH 290/489] Add -ldflags to CI gotestsum --- .github/workflows/gotestsum.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gotestsum.sh b/.github/workflows/gotestsum.sh index ed631847b7..785d8a560a 100755 --- a/.github/workflows/gotestsum.sh +++ b/.github/workflows/gotestsum.sh @@ -49,7 +49,7 @@ done packages=$(go list ./...) for package in $packages; do - cmd="stdbuf -oL gotestsum --format short-verbose --packages=\"$package\" --rerun-fails=2 --no-color=false --" + cmd="stdbuf -oL gotestsum --format short-verbose --packages=\"$package\" --rerun-fails=2 --no-color=false -- -ldflags='-checklinkname=0'" if [ "$timeout" != "" ]; then cmd="$cmd -timeout $timeout" From 2254e64fd2c52cc6f9b0840a51c0549453a52ed1 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Mon, 30 Sep 2024 17:16:31 -0300 Subject: [PATCH 291/489] Add -ldflags to Arbitrator CI --- .github/workflows/arbitrator-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/arbitrator-ci.yml b/.github/workflows/arbitrator-ci.yml index 7829fe4086..a6f478c4b3 100644 --- a/.github/workflows/arbitrator-ci.yml +++ b/.github/workflows/arbitrator-ci.yml @@ -59,7 +59,7 @@ jobs: cd go-ethereum # Enable KZG point evaluation precompile early sed -i 's#var PrecompiledContractsBerlin = map\[common.Address\]PrecompiledContract{#\0 common.BytesToAddress([]byte{0x0a}): \&kzgPointEvaluation{},#g' core/vm/contracts.go - go build -o /usr/local/bin/geth ./cmd/geth + go build -o /usr/local/bin/geth -ldflags='-checklinkname=0' ./cmd/geth - name: Setup nodejs uses: actions/setup-node@v3 From 5e8c4a2535733f5cea6b50b57594b684e01464f6 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 30 Sep 2024 13:44:24 -0300 Subject: [PATCH 292/489] Stylus long term cache metrics --- arbitrator/stylus/src/cache.rs | 60 +++++++++++++++++++++++---- arbitrator/stylus/src/lib.rs | 16 +++---- arbos/programs/native.go | 59 ++++++++++++++++++++------ execution/gethexec/executionengine.go | 4 +- system_tests/program_test.go | 26 ++++++------ 5 files changed, 121 insertions(+), 44 deletions(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index c1fdaaccee..827e2beaa3 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -29,8 +29,16 @@ pub struct LruCounters { pub does_not_fit: u32, } +pub struct LongTermCounters { + pub hits: u32, + pub misses: u32, +} + pub struct InitCache { long_term: HashMap, + long_term_size_bytes: usize, + long_term_counters: LongTermCounters, + lru: CLruCache, lru_counters: LruCounters, } @@ -91,6 +99,20 @@ pub struct LruCacheMetrics { pub does_not_fit: u32, } +#[repr(C)] +pub struct LongTermCacheMetrics { + pub size_bytes: u64, + pub count: u32, + pub hits: u32, + pub misses: u32, +} + +#[repr(C)] +pub struct CacheMetrics { + pub lru: LruCacheMetrics, + pub long_term: LongTermCacheMetrics, +} + pub fn deserialize_module( module: &[u8], version: u16, @@ -117,6 +139,9 @@ impl InitCache { fn new(size_bytes: usize) -> Self { Self { long_term: HashMap::new(), + long_term_size_bytes: 0, + long_term_counters: LongTermCounters { hits: 0, misses: 0 }, + lru: CLruCache::with_config( CLruCacheConfig::new(NonZeroUsize::new(size_bytes).unwrap()) .with_scale(CustomWeightScale), @@ -142,8 +167,11 @@ impl InitCache { // See if the item is in the long term cache if let Some(item) = cache.long_term.get(&key) { - return Some(item.data()); + let data = item.data(); + cache.long_term_counters.hits += 1; + return Some(data); } + cache.long_term_counters.misses += 1; // See if the item is in the LRU cache, promoting if so if let Some(item) = cache.lru.get(&key) { @@ -174,6 +202,7 @@ impl InitCache { if let Some(item) = cache.lru.peek(&key).cloned() { if long_term_tag == Self::ARBOS_TAG { cache.long_term.insert(key, item.clone()); + cache.long_term_size_bytes += item.entry_size_estimate_bytes; } else { // only calls get to move the key to the head of the LRU list cache.lru.get(&key); @@ -195,6 +224,7 @@ impl InitCache { }; } else { cache.long_term.insert(key, item); + cache.long_term_size_bytes += entry_size_estimate_bytes; } Ok(data) } @@ -207,6 +237,7 @@ impl InitCache { let key = CacheKey::new(module_hash, version, debug); let mut cache = cache!(); if let Some(item) = cache.long_term.remove(&key) { + cache.long_term_size_bytes -= item.entry_size_estimate_bytes; if cache.lru.put_with_weight(key, item).is_err() { eprintln!("{}", Self::DOES_NOT_FIT_MSG); } @@ -225,23 +256,32 @@ impl InitCache { eprintln!("{}", Self::DOES_NOT_FIT_MSG); } } + cache.long_term_size_bytes = 0; } - pub fn get_lru_metrics() -> LruCacheMetrics { + pub fn get_metrics() -> CacheMetrics { let mut cache = cache!(); - let count = cache.lru.len(); - let metrics = LruCacheMetrics { - // add 1 to each entry to account that we subtracted 1 in the weight calculation - size_bytes: (cache.lru.weight() + count).try_into().unwrap(), + let lru_count = cache.lru.len(); + let lru_metrics = LruCacheMetrics { + // adds 1 to each entry to account that we subtracted 1 in the weight calculation + size_bytes: (cache.lru.weight() + lru_count).try_into().unwrap(), - count: count.try_into().unwrap(), + count: lru_count.try_into().unwrap(), hits: cache.lru_counters.hits, misses: cache.lru_counters.misses, does_not_fit: cache.lru_counters.does_not_fit, }; + let long_term_metrics = LongTermCacheMetrics { + size_bytes: cache.long_term_size_bytes.try_into().unwrap(), + count: cache.long_term.len().try_into().unwrap(), + + hits: cache.long_term_counters.hits, + misses: cache.long_term_counters.misses, + }; + // Empty counters. // go side, which is the only consumer of this function besides tests, // will read those counters and increment its own prometheus counters with them. @@ -250,8 +290,12 @@ impl InitCache { misses: 0, does_not_fit: 0, }; + cache.long_term_counters = LongTermCounters { hits: 0, misses: 0 }; - metrics + CacheMetrics { + lru: lru_metrics, + long_term: long_term_metrics, + } } // only used for testing diff --git a/arbitrator/stylus/src/lib.rs b/arbitrator/stylus/src/lib.rs index abea428167..c16f3d7598 100644 --- a/arbitrator/stylus/src/lib.rs +++ b/arbitrator/stylus/src/lib.rs @@ -11,7 +11,7 @@ use arbutil::{ format::DebugBytes, Bytes32, }; -use cache::{deserialize_module, InitCache, LruCacheMetrics}; +use cache::{deserialize_module, InitCache, CacheMetrics}; use evm_api::NativeRequestHandler; use eyre::ErrReport; use native::NativeInstance; @@ -364,10 +364,10 @@ pub unsafe extern "C" fn stylus_drop_vec(vec: RustBytes) { } } -/// Gets lru cache metrics. +/// Gets cache metrics. #[no_mangle] -pub extern "C" fn stylus_get_lru_cache_metrics() -> LruCacheMetrics { - InitCache::get_lru_metrics() +pub extern "C" fn stylus_get_cache_metrics() -> CacheMetrics { + InitCache::get_metrics() } /// Clears lru cache. @@ -377,18 +377,18 @@ pub extern "C" fn stylus_clear_lru_cache() { InitCache::clear_lru_cache() } -/// Gets lru entry size in bytes. +/// Gets entry size in bytes. /// Only used for testing purposes. #[no_mangle] -pub extern "C" fn stylus_get_lru_entry_size_estimate_bytes( +pub extern "C" fn stylus_get_entry_size_estimate_bytes( module: GoSliceData, version: u16, debug: bool, ) -> u64 { match deserialize_module(module.slice(), version, debug) { Err(error) => panic!("tried to get invalid asm!: {error}"), - Ok((_, _, lru_entry_size_estimate_bytes)) => { - lru_entry_size_estimate_bytes.try_into().unwrap() + Ok((_, _, entry_size_estimate_bytes)) => { + entry_size_estimate_bytes.try_into().unwrap() } } } diff --git a/arbos/programs/native.go b/arbos/programs/native.go index 5fbc512211..38da63013d 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -52,6 +52,11 @@ var ( stylusLRUCacheSizeHitsCounter = metrics.NewRegisteredCounter("arb/arbos/stylus/cache/lru/hits", nil) stylusLRUCacheSizeMissesCounter = metrics.NewRegisteredCounter("arb/arbos/stylus/cache/lru/misses", nil) stylusLRUCacheSizeDoesNotFitCounter = metrics.NewRegisteredCounter("arb/arbos/stylus/cache/lru/does_not_fit", nil) + + stylusLongTermCacheSizeBytesGauge = metrics.NewRegisteredGauge("arb/arbos/stylus/cache/long_term/size_bytes", nil) + stylusLongTermCacheSizeCountGauge = metrics.NewRegisteredGauge("arb/arbos/stylus/cache/long_term/count", nil) + stylusLongTermCacheSizeHitsCounter = metrics.NewRegisteredCounter("arb/arbos/stylus/cache/long_term/hits", nil) + stylusLongTermCacheSizeMissesCounter = metrics.NewRegisteredCounter("arb/arbos/stylus/cache/long_term/misses", nil) ) func activateProgram( @@ -333,24 +338,52 @@ func SetWasmLruCacheCapacity(capacityBytes uint64) { C.stylus_set_cache_lru_capacity(u64(capacityBytes)) } -// exported for testing +func UpdateWasmCacheMetrics() { + metrics := C.stylus_get_cache_metrics() + + stylusLRUCacheSizeBytesGauge.Update(int64(metrics.lru.size_bytes)) + stylusLRUCacheSizeCountGauge.Update(int64(metrics.lru.count)) + stylusLRUCacheSizeHitsCounter.Inc(int64(metrics.lru.hits)) + stylusLRUCacheSizeMissesCounter.Inc(int64(metrics.lru.misses)) + stylusLRUCacheSizeDoesNotFitCounter.Inc(int64(metrics.lru.does_not_fit)) + + stylusLongTermCacheSizeBytesGauge.Update(int64(metrics.long_term.size_bytes)) + stylusLongTermCacheSizeCountGauge.Update(int64(metrics.long_term.count)) + stylusLongTermCacheSizeHitsCounter.Inc(int64(metrics.long_term.hits)) + stylusLongTermCacheSizeMissesCounter.Inc(int64(metrics.long_term.misses)) +} + +// Used for testing type WasmLruCacheMetrics struct { SizeBytes uint64 Count uint32 } -func GetWasmLruCacheMetrics() *WasmLruCacheMetrics { - metrics := C.stylus_get_lru_cache_metrics() +// Used for testing +type WasmLongTermCacheMetrics struct { + SizeBytes uint64 + Count uint32 +} - stylusLRUCacheSizeBytesGauge.Update(int64(metrics.size_bytes)) - stylusLRUCacheSizeCountGauge.Update(int64(metrics.count)) - stylusLRUCacheSizeHitsCounter.Inc(int64(metrics.hits)) - stylusLRUCacheSizeMissesCounter.Inc(int64(metrics.misses)) - stylusLRUCacheSizeDoesNotFitCounter.Inc(int64(metrics.does_not_fit)) +// Used for testing +type WasmCacheMetrics struct { + Lru WasmLruCacheMetrics + LongTerm WasmLongTermCacheMetrics +} - return &WasmLruCacheMetrics{ - SizeBytes: uint64(metrics.size_bytes), - Count: uint32(metrics.count), +// Used for testing +func GetWasmCacheMetrics() *WasmCacheMetrics { + metrics := C.stylus_get_cache_metrics() + + return &WasmCacheMetrics{ + Lru: WasmLruCacheMetrics{ + SizeBytes: uint64(metrics.lru.size_bytes), + Count: uint32(metrics.lru.count), + }, + LongTerm: WasmLongTermCacheMetrics{ + SizeBytes: uint64(metrics.long_term.size_bytes), + Count: uint32(metrics.long_term.count), + }, } } @@ -360,8 +393,8 @@ func ClearWasmLruCache() { } // Used for testing -func GetLruEntrySizeEstimateBytes(module []byte, version uint16, debug bool) uint64 { - return uint64(C.stylus_get_lru_entry_size_estimate_bytes(goSlice(module), u16(version), cbool(debug))) +func GetEntrySizeEstimateBytes(module []byte, version uint16, debug bool) uint64 { + return uint64(C.stylus_get_entry_size_estimate_bytes(goSlice(module), u16(version), cbool(debug))) } const DefaultTargetDescriptionArm = "arm64-linux-unknown+neon" diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index b36340757a..23573a0277 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -976,14 +976,14 @@ func (s *ExecutionEngine) Start(ctx_in context.Context) { } }) if !s.disableStylusCacheMetricsCollection { - // periodically update stylus lru cache metrics + // periodically update stylus cache metrics s.LaunchThread(func(ctx context.Context) { for { select { case <-ctx.Done(): return case <-time.After(time.Minute): - programs.GetWasmLruCacheMetrics() + programs.UpdateWasmCacheMetrics() } } }) diff --git a/system_tests/program_test.go b/system_tests/program_test.go index 1cbbf268f1..51a35b9981 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -2008,7 +2008,7 @@ func checkWasmStoreContent(t *testing.T, wasmDb ethdb.KeyValueStore, targets []s } } -func deployWasmAndGetLruEntrySizeEstimateBytes( +func deployWasmAndGetEntrySizeEstimateBytes( t *testing.T, builder *NodeBuilder, auth bind.TransactOpts, @@ -2039,12 +2039,12 @@ func deployWasmAndGetLruEntrySizeEstimateBytes( module, err := statedb.TryGetActivatedAsm(rawdb.LocalTarget(), log.ModuleHash) Require(t, err, ", wasmName:", wasmName) - lruEntrySizeEstimateBytes := programs.GetLruEntrySizeEstimateBytes(module, log.Version, true) + entrySizeEstimateBytes := programs.GetEntrySizeEstimateBytes(module, log.Version, true) // just a sanity check - if lruEntrySizeEstimateBytes == 0 { - Fatal(t, "lruEntrySizeEstimateBytes is 0, wasmName:", wasmName) + if entrySizeEstimateBytes == 0 { + Fatal(t, "entrySizeEstimateBytes is 0, wasmName:", wasmName) } - return programAddress, lruEntrySizeEstimateBytes + return programAddress, entrySizeEstimateBytes } func TestWasmLruCache(t *testing.T) { @@ -2057,9 +2057,9 @@ func TestWasmLruCache(t *testing.T) { auth.GasLimit = 32000000 auth.Value = oneEth - fallibleProgramAddress, fallibleLruEntrySizeEstimateBytes := deployWasmAndGetLruEntrySizeEstimateBytes(t, builder, auth, "fallible") - keccakProgramAddress, keccakLruEntrySizeEstimateBytes := deployWasmAndGetLruEntrySizeEstimateBytes(t, builder, auth, "keccak") - mathProgramAddress, mathLruEntrySizeEstimateBytes := deployWasmAndGetLruEntrySizeEstimateBytes(t, builder, auth, "math") + fallibleProgramAddress, fallibleLruEntrySizeEstimateBytes := deployWasmAndGetEntrySizeEstimateBytes(t, builder, auth, "fallible") + keccakProgramAddress, keccakLruEntrySizeEstimateBytes := deployWasmAndGetEntrySizeEstimateBytes(t, builder, auth, "keccak") + mathProgramAddress, mathLruEntrySizeEstimateBytes := deployWasmAndGetEntrySizeEstimateBytes(t, builder, auth, "math") t.Log( "lruEntrySizeEstimateBytes, ", "fallible:", fallibleLruEntrySizeEstimateBytes, @@ -2068,7 +2068,7 @@ func TestWasmLruCache(t *testing.T) { ) programs.ClearWasmLruCache() - lruMetrics := programs.GetWasmLruCacheMetrics() + lruMetrics := programs.GetWasmCacheMetrics().Lru if lruMetrics.Count != 0 { t.Fatalf("lruMetrics.Count, expected: %v, actual: %v", 0, lruMetrics.Count) } @@ -2082,7 +2082,7 @@ func TestWasmLruCache(t *testing.T) { Require(t, l2client.SendTransaction(ctx, tx)) _, err := EnsureTxSucceeded(ctx, l2client, tx) Require(t, err) - lruMetrics = programs.GetWasmLruCacheMetrics() + lruMetrics = programs.GetWasmCacheMetrics().Lru if lruMetrics.Count != 0 { t.Fatalf("lruMetrics.Count, expected: %v, actual: %v", 0, lruMetrics.Count) } @@ -2098,7 +2098,7 @@ func TestWasmLruCache(t *testing.T) { Require(t, l2client.SendTransaction(ctx, tx)) _, err = EnsureTxSucceeded(ctx, l2client, tx) Require(t, err) - lruMetrics = programs.GetWasmLruCacheMetrics() + lruMetrics = programs.GetWasmCacheMetrics().Lru if lruMetrics.Count != 1 { t.Fatalf("lruMetrics.Count, expected: %v, actual: %v", 1, lruMetrics.Count) } @@ -2111,7 +2111,7 @@ func TestWasmLruCache(t *testing.T) { Require(t, l2client.SendTransaction(ctx, tx)) _, err = EnsureTxSucceeded(ctx, l2client, tx) Require(t, err) - lruMetrics = programs.GetWasmLruCacheMetrics() + lruMetrics = programs.GetWasmCacheMetrics().Lru if lruMetrics.Count != 2 { t.Fatalf("lruMetrics.Count, expected: %v, actual: %v", 2, lruMetrics.Count) } @@ -2124,7 +2124,7 @@ func TestWasmLruCache(t *testing.T) { Require(t, l2client.SendTransaction(ctx, tx)) _, err = EnsureTxSucceeded(ctx, l2client, tx) Require(t, err) - lruMetrics = programs.GetWasmLruCacheMetrics() + lruMetrics = programs.GetWasmCacheMetrics().Lru if lruMetrics.Count != 2 { t.Fatalf("lruMetrics.Count, expected: %v, actual: %v", 2, lruMetrics.Count) } From 02f1dc0508af91e8c548c82202d14a271442cd1a Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 30 Sep 2024 17:35:04 -0300 Subject: [PATCH 293/489] Rust lint --- arbitrator/stylus/src/lib.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/arbitrator/stylus/src/lib.rs b/arbitrator/stylus/src/lib.rs index c16f3d7598..f5598198eb 100644 --- a/arbitrator/stylus/src/lib.rs +++ b/arbitrator/stylus/src/lib.rs @@ -11,7 +11,7 @@ use arbutil::{ format::DebugBytes, Bytes32, }; -use cache::{deserialize_module, InitCache, CacheMetrics}; +use cache::{deserialize_module, CacheMetrics, InitCache}; use evm_api::NativeRequestHandler; use eyre::ErrReport; use native::NativeInstance; @@ -387,8 +387,6 @@ pub extern "C" fn stylus_get_entry_size_estimate_bytes( ) -> u64 { match deserialize_module(module.slice(), version, debug) { Err(error) => panic!("tried to get invalid asm!: {error}"), - Ok((_, _, entry_size_estimate_bytes)) => { - entry_size_estimate_bytes.try_into().unwrap() - } + Ok((_, _, entry_size_estimate_bytes)) => entry_size_estimate_bytes.try_into().unwrap(), } } From e67b66a1161aeec749065595d04c37847fd1eba4 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Mon, 30 Sep 2024 18:05:34 -0300 Subject: [PATCH 294/489] Add -ldflags to CI redis test --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a944f08f40..efeee151be 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -168,7 +168,7 @@ jobs: if: matrix.test-mode == 'defaults' run: | echo "Running redis tests" >> full.log - TEST_REDIS=redis://localhost:6379/0 gotestsum --format short-verbose -- -p 1 -run TestRedis ./arbnode/... ./system_tests/... -coverprofile=coverage-redis.txt -covermode=atomic -coverpkg=./... + TEST_REDIS=redis://localhost:6379/0 gotestsum --format short-verbose -- -ldflags='-checklinkname=0' -p 1 -run TestRedis ./arbnode/... ./system_tests/... -coverprofile=coverage-redis.txt -covermode=atomic -coverpkg=./... - name: run challenge tests if: matrix.test-mode == 'challenge' From 0b40afb6eb5ff02376cd5ea307cd02a74d42e353 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Mon, 30 Sep 2024 21:45:52 -0500 Subject: [PATCH 295/489] Upgrade getNodeCreationBlockForLogLookup log to Info --- staker/rollup_watcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/staker/rollup_watcher.go b/staker/rollup_watcher.go index 3219566549..324ec23ab9 100644 --- a/staker/rollup_watcher.go +++ b/staker/rollup_watcher.go @@ -111,7 +111,7 @@ func (r *RollupWatcher) getNodeCreationBlock(ctx context.Context, nodeNum uint64 if r.supportedL3Method.Load() { return nil, fmt.Errorf("getNodeCreationBlockForLogLookup failed despite previously succeeding: %w", err) } - log.Trace("failed to call getNodeCreationBlockForLogLookup, falling back on node CreatedAtBlock field", "err", err) + log.Info("getNodeCreationBlockForLogLookup does not seem to exist, falling back on node CreatedAtBlock field", "err", err) r.unSupportedL3Method.Store(true) } else { return nil, err From 5b27cd50d49b7f9e9d050e3a76c38e48220c39ef Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Tue, 1 Oct 2024 15:19:34 -0300 Subject: [PATCH 296/489] Remove checklinkname=0 build option --- .github/workflows/arbitrator-ci.yml | 2 +- .github/workflows/ci.yml | 2 +- .github/workflows/gotestsum.sh | 2 +- Makefile | 14 ++++---------- go.mod | 2 +- 5 files changed, 8 insertions(+), 14 deletions(-) diff --git a/.github/workflows/arbitrator-ci.yml b/.github/workflows/arbitrator-ci.yml index a6f478c4b3..7829fe4086 100644 --- a/.github/workflows/arbitrator-ci.yml +++ b/.github/workflows/arbitrator-ci.yml @@ -59,7 +59,7 @@ jobs: cd go-ethereum # Enable KZG point evaluation precompile early sed -i 's#var PrecompiledContractsBerlin = map\[common.Address\]PrecompiledContract{#\0 common.BytesToAddress([]byte{0x0a}): \&kzgPointEvaluation{},#g' core/vm/contracts.go - go build -o /usr/local/bin/geth -ldflags='-checklinkname=0' ./cmd/geth + go build -o /usr/local/bin/geth ./cmd/geth - name: Setup nodejs uses: actions/setup-node@v3 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index efeee151be..a944f08f40 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -168,7 +168,7 @@ jobs: if: matrix.test-mode == 'defaults' run: | echo "Running redis tests" >> full.log - TEST_REDIS=redis://localhost:6379/0 gotestsum --format short-verbose -- -ldflags='-checklinkname=0' -p 1 -run TestRedis ./arbnode/... ./system_tests/... -coverprofile=coverage-redis.txt -covermode=atomic -coverpkg=./... + TEST_REDIS=redis://localhost:6379/0 gotestsum --format short-verbose -- -p 1 -run TestRedis ./arbnode/... ./system_tests/... -coverprofile=coverage-redis.txt -covermode=atomic -coverpkg=./... - name: run challenge tests if: matrix.test-mode == 'challenge' diff --git a/.github/workflows/gotestsum.sh b/.github/workflows/gotestsum.sh index 785d8a560a..ed631847b7 100755 --- a/.github/workflows/gotestsum.sh +++ b/.github/workflows/gotestsum.sh @@ -49,7 +49,7 @@ done packages=$(go list ./...) for package in $packages; do - cmd="stdbuf -oL gotestsum --format short-verbose --packages=\"$package\" --rerun-fails=2 --no-color=false -- -ldflags='-checklinkname=0'" + cmd="stdbuf -oL gotestsum --format short-verbose --packages=\"$package\" --rerun-fails=2 --no-color=false --" if [ "$timeout" != "" ]; then cmd="$cmd -timeout $timeout" diff --git a/Makefile b/Makefile index 919eb3f4bf..c3cf1a5144 100644 --- a/Makefile +++ b/Makefile @@ -27,12 +27,6 @@ ifneq ($(origin NITRO_MODIFIED),undefined) GOLANG_LDFLAGS += -X github.com/offchainlabs/nitro/cmd/util/confighelpers.modified=$(NITRO_MODIFIED) endif -# As of Go 1.23, the package github.com/fjl/memsize no longer works because of a restriction added -# by the Go toolchain. The Go 1.23 compiler no longer allows access to runtime symbols via -# go:linkname, which prevents memsize from accessing the Stop-the-World functionality of the Go -# runtime. To solve that we add the following compiler directive. -GOLANG_LDFLAGS += -checklinkname=0 - ifneq ($(origin GOLANG_LDFLAGS),undefined) GOLANG_PARAMS = -ldflags="-extldflags '-ldl' $(GOLANG_LDFLAGS)" endif @@ -232,17 +226,17 @@ test-go: .make/test-go .PHONY: test-go-challenge test-go-challenge: test-go-deps - gotestsum --format short-verbose --no-color=false -- $(GOLANG_PARAMS) -timeout 120m ./system_tests/... -run TestChallenge -tags challengetest + gotestsum --format short-verbose --no-color=false -- -timeout 120m ./system_tests/... -run TestChallenge -tags challengetest @printf $(done) .PHONY: test-go-stylus test-go-stylus: test-go-deps - gotestsum --format short-verbose --no-color=false -- $(GOLANG_PARAMS) -timeout 120m ./system_tests/... -run TestProgramArbitrator -tags stylustest + gotestsum --format short-verbose --no-color=false -- -timeout 120m ./system_tests/... -run TestProgramArbitrator -tags stylustest @printf $(done) .PHONY: test-go-redis test-go-redis: test-go-deps - TEST_REDIS=redis://localhost:6379/0 gotestsum --format short-verbose --no-color=false -- $(GOLANG_PARAMS) -p 1 -run TestRedis ./system_tests/... ./arbnode/... + TEST_REDIS=redis://localhost:6379/0 gotestsum --format short-verbose --no-color=false -- -p 1 -run TestRedis ./system_tests/... ./arbnode/... @printf $(done) .PHONY: test-gen-proofs @@ -547,7 +541,7 @@ contracts/test/prover/proofs/%.json: $(arbitrator_cases)/%.wasm $(prover_bin) @touch $@ .make/test-go: $(DEP_PREDICATE) $(go_source) build-node-deps test-go-deps $(ORDER_ONLY_PREDICATE) .make - gotestsum --format short-verbose --no-color=false -- $(GOLANG_PARAMS) + gotestsum --format short-verbose --no-color=false @touch $@ .make/test-rust: $(DEP_PREDICATE) wasm-ci-build $(ORDER_ONLY_PREDICATE) .make diff --git a/go.mod b/go.mod index e3a6f59ef9..761554ade4 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/offchainlabs/nitro -go 1.22 +go 1.23 replace github.com/VictoriaMetrics/fastcache => ./fastcache From bcc7c5b1229956e34b950f937a9ca280c1bf8e07 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Tue, 1 Oct 2024 15:49:35 -0300 Subject: [PATCH 297/489] Update go-ethereum to remove memsize pkg --- go-ethereum | 2 +- go.mod | 1 - go.sum | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/go-ethereum b/go-ethereum index 17cd001675..b068464bf5 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 17cd00167543a5a2b0b083e32820051100154c2f +Subproject commit b068464bf59ab5414f72c2d4aba855b8af5edc17 diff --git a/go.mod b/go.mod index 761554ade4..bbe851fe02 100644 --- a/go.mod +++ b/go.mod @@ -94,7 +94,6 @@ require ( github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/ethereum/c-kzg-4844 v0.4.0 // indirect - github.com/fjl/memsize v0.0.2 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/gammazero/deque v0.2.1 // indirect github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect diff --git a/go.sum b/go.sum index c0193be769..8a432408d4 100644 --- a/go.sum +++ b/go.sum @@ -233,8 +233,6 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= -github.com/fjl/memsize v0.0.2 h1:27txuSD9or+NZlnOWdKUxeBzTAUkWCVh+4Gf2dWFOzA= -github.com/fjl/memsize v0.0.2/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= From 6a7d177ce5881e5a4295946b6c268890040bf2d7 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Mon, 30 Sep 2024 17:16:31 -0300 Subject: [PATCH 298/489] Add -ldflags to Arbitrator CI --- .github/workflows/arbitrator-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/arbitrator-ci.yml b/.github/workflows/arbitrator-ci.yml index 7829fe4086..a6f478c4b3 100644 --- a/.github/workflows/arbitrator-ci.yml +++ b/.github/workflows/arbitrator-ci.yml @@ -59,7 +59,7 @@ jobs: cd go-ethereum # Enable KZG point evaluation precompile early sed -i 's#var PrecompiledContractsBerlin = map\[common.Address\]PrecompiledContract{#\0 common.BytesToAddress([]byte{0x0a}): \&kzgPointEvaluation{},#g' core/vm/contracts.go - go build -o /usr/local/bin/geth ./cmd/geth + go build -o /usr/local/bin/geth -ldflags='-checklinkname=0' ./cmd/geth - name: Setup nodejs uses: actions/setup-node@v3 From 4dc36758d732db409e8f25513e932322bf713527 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Tue, 1 Oct 2024 16:38:05 -0300 Subject: [PATCH 299/489] Use latest geth in arbitrator CI --- .github/workflows/arbitrator-ci.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/arbitrator-ci.yml b/.github/workflows/arbitrator-ci.yml index a6f478c4b3..47646017ac 100644 --- a/.github/workflows/arbitrator-ci.yml +++ b/.github/workflows/arbitrator-ci.yml @@ -55,11 +55,9 @@ jobs: - name: Install custom go-ethereum run: | cd /tmp - git clone --branch v1.13.8 --depth 1 https://github.com/ethereum/go-ethereum.git + git clone --branch v1.14.11 --depth 1 https://github.com/ethereum/go-ethereum.git cd go-ethereum - # Enable KZG point evaluation precompile early - sed -i 's#var PrecompiledContractsBerlin = map\[common.Address\]PrecompiledContract{#\0 common.BytesToAddress([]byte{0x0a}): \&kzgPointEvaluation{},#g' core/vm/contracts.go - go build -o /usr/local/bin/geth -ldflags='-checklinkname=0' ./cmd/geth + go build -o /usr/local/bin/geth ./cmd/geth - name: Setup nodejs uses: actions/setup-node@v3 From a4784603c38e0a2b8d52c21ef4ef09e16376a83c Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Tue, 1 Oct 2024 18:50:37 +0200 Subject: [PATCH 300/489] InitCache: add items found in LRU to long term cache (if long_term_tag is 1) --- arbitrator/stylus/src/cache.rs | 17 ++++++++++++----- arbitrator/stylus/src/native.rs | 7 +++---- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index c1fdaaccee..21933c51cd 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -136,9 +136,10 @@ impl InitCache { } /// Retrieves a cached value, updating items as necessary. - pub fn get(module_hash: Bytes32, version: u16, debug: bool) -> Option<(Module, Store)> { - let mut cache = cache!(); + /// If long_term_tag is 1 and the item is only in LRU will insert to long term cache. + pub fn get(module_hash: Bytes32, version: u16, long_term_tag: u32, debug: bool) -> Option<(Module, Store)> { let key = CacheKey::new(module_hash, version, debug); + let mut cache = cache!(); // See if the item is in the long term cache if let Some(item) = cache.long_term.get(&key) { @@ -146,12 +147,18 @@ impl InitCache { } // See if the item is in the LRU cache, promoting if so - if let Some(item) = cache.lru.get(&key) { - let data = item.data(); + if let Some(item) = cache.lru.peek(&key).cloned() { cache.lru_counters.hits += 1; - return Some(data); + if long_term_tag == Self::ARBOS_TAG { + cache.long_term.insert(key, item.clone()); + } else { + // only calls get to move the key to the head of the LRU list + cache.lru.get(&key); + } + return Some(item.data()); } cache.lru_counters.misses += 1; + None } diff --git a/arbitrator/stylus/src/native.rs b/arbitrator/stylus/src/native.rs index 516c6602e7..c751a670cc 100644 --- a/arbitrator/stylus/src/native.rs +++ b/arbitrator/stylus/src/native.rs @@ -121,13 +121,12 @@ impl> NativeInstance { let compile = CompileConfig::version(version, debug); let env = WasmEnv::new(compile, None, evm, evm_data); let module_hash = env.evm_data.module_hash; - - if let Some((module, store)) = InitCache::get(module_hash, version, debug) { - return Self::from_module(module, store, env); - } if !env.evm_data.cached { long_term_tag = 0; } + if let Some((module, store)) = InitCache::get(module_hash, version, long_term_tag, debug) { + return Self::from_module(module, store, env); + } let (module, store) = InitCache::insert(module_hash, module, version, long_term_tag, debug)?; Self::from_module(module, store, env) From d61710fa1b2ef90486d69edb0b3c9623034c6ca4 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Tue, 1 Oct 2024 22:21:21 +0200 Subject: [PATCH 301/489] rustfmt InitCache.get --- arbitrator/stylus/src/cache.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index 21933c51cd..d849d39be0 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -137,7 +137,12 @@ impl InitCache { /// Retrieves a cached value, updating items as necessary. /// If long_term_tag is 1 and the item is only in LRU will insert to long term cache. - pub fn get(module_hash: Bytes32, version: u16, long_term_tag: u32, debug: bool) -> Option<(Module, Store)> { + pub fn get( + module_hash: Bytes32, + version: u16, + long_term_tag: u32, + debug: bool, + ) -> Option<(Module, Store)> { let key = CacheKey::new(module_hash, version, debug); let mut cache = cache!(); From 328a386f6eb020b2ee3a35752b8a381508729ce1 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Tue, 1 Oct 2024 22:21:43 +0200 Subject: [PATCH 302/489] update geth pin --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 0c3f6eba21..b1075d3786 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 0c3f6eba21cbe0196b298dfbd3fa7d51dffd627e +Subproject commit b1075d3786b28a6a3a06fe0e0ab8d1cdecc72f55 From eea41eb8f12c7bec65a1129997186e6c75e97dd1 Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Tue, 1 Oct 2024 16:47:51 -0700 Subject: [PATCH 303/489] chore: Remove bridge binding from BatchPoster --- arbnode/batch_poster.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/arbnode/batch_poster.go b/arbnode/batch_poster.go index 44b360e76e..7e044c5592 100644 --- a/arbnode/batch_poster.go +++ b/arbnode/batch_poster.go @@ -98,7 +98,6 @@ type BatchPoster struct { arbOSVersionGetter execution.FullExecutionClient config BatchPosterConfigFetcher seqInbox *bridgegen.SequencerInbox - bridge *bridgegen.Bridge syncMonitor *SyncMonitor seqInboxABI *abi.ABI seqInboxAddr common.Address @@ -309,10 +308,7 @@ func NewBatchPoster(ctx context.Context, opts *BatchPosterOpts) (*BatchPoster, e if err != nil { return nil, err } - bridge, err := bridgegen.NewBridge(opts.DeployInfo.Bridge, opts.L1Reader.Client()) - if err != nil { - return nil, err - } + if err = opts.Config().Validate(); err != nil { return nil, err } @@ -340,7 +336,6 @@ func NewBatchPoster(ctx context.Context, opts *BatchPosterOpts) (*BatchPoster, e arbOSVersionGetter: opts.VersionGetter, syncMonitor: opts.SyncMonitor, config: opts.Config, - bridge: bridge, seqInbox: seqInbox, seqInboxABI: seqInboxABI, seqInboxAddr: opts.DeployInfo.SequencerInbox, From d7c21bb17915a288eac5ebacd1b2e705697311c2 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Wed, 2 Oct 2024 17:19:18 +0530 Subject: [PATCH 304/489] Log the DAS public key on startup --- das/sign_after_store_das_writer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/das/sign_after_store_das_writer.go b/das/sign_after_store_das_writer.go index 40b03847d8..b09ed091cc 100644 --- a/das/sign_after_store_das_writer.go +++ b/das/sign_after_store_das_writer.go @@ -81,6 +81,7 @@ func NewSignAfterStoreDASWriter(ctx context.Context, config DataAvailabilityConf if err != nil { return nil, err } + log.Info("DAS public key used for signing", "key", hexutil.Encode(blsSignatures.PublicKeyToBytes(publicKey))) keyset := &daprovider.DataAvailabilityKeyset{ AssumedHonest: 1, From baabb7d2e184a9753b4b1f473a9075dffe854342 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 2 Oct 2024 15:38:10 -0300 Subject: [PATCH 305/489] ErrExceedsMaxMempoolSize logged as a warning initially --- arbnode/dataposter/data_poster.go | 2 +- staker/staker.go | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/arbnode/dataposter/data_poster.go b/arbnode/dataposter/data_poster.go index acbf9c4cc8..0502b894a0 100644 --- a/arbnode/dataposter/data_poster.go +++ b/arbnode/dataposter/data_poster.go @@ -336,7 +336,7 @@ func (p *DataPoster) UsingNoOpStorage() bool { return p.usingNoOpStorage } -var ErrExceedsMaxMempoolSize = errors.New("posting this transaction will exceed max mempool size") +var ErrExceedsMaxMempoolSize = errors.New("Node is behind, posting this transaction will exceed max mempool size") // Does basic check whether posting transaction with specified nonce would // result in exceeding maximum queue length or maximum transactions in mempool. diff --git a/staker/staker.go b/staker/staker.go index 2489bbbcf7..21d007703b 100644 --- a/staker/staker.go +++ b/staker/staker.go @@ -512,7 +512,8 @@ func (s *Staker) Start(ctxIn context.Context) { } s.StopWaiter.Start(ctxIn, s) backoff := time.Second - ephemeralErrorHandler := util.NewEphemeralErrorHandler(10*time.Minute, "is ahead of on-chain nonce", 0) + isAheadOfOnChainNonceEphemeralErrorHandler := util.NewEphemeralErrorHandler(10*time.Minute, "is ahead of on-chain nonce", 0) + exceedsMaxMempoolSizeEphemeralErrorHandler := util.NewEphemeralErrorHandler(10*time.Minute, dataposter.ErrExceedsMaxMempoolSize.Error(), 0) s.CallIteratively(func(ctx context.Context) (returningWait time.Duration) { defer func() { panicErr := recover() @@ -546,7 +547,8 @@ func (s *Staker) Start(ctxIn context.Context) { } } if err == nil { - ephemeralErrorHandler.Reset() + isAheadOfOnChainNonceEphemeralErrorHandler.Reset() + exceedsMaxMempoolSizeEphemeralErrorHandler.Reset() backoff = time.Second stakerLastSuccessfulActionGauge.Update(time.Now().Unix()) stakerActionSuccessCounter.Inc(1) @@ -564,7 +566,8 @@ func (s *Staker) Start(ctxIn context.Context) { } else { logLevel = log.Warn } - logLevel = ephemeralErrorHandler.LogLevel(err, logLevel) + logLevel = isAheadOfOnChainNonceEphemeralErrorHandler.LogLevel(err, logLevel) + logLevel = exceedsMaxMempoolSizeEphemeralErrorHandler.LogLevel(err, logLevel) logLevel("error acting as staker", "err", err) return backoff }) From e63345b55813c1e50f16d7ef6f74c5e056635a27 Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Thu, 3 Oct 2024 07:35:04 +0800 Subject: [PATCH 306/489] refactor: remove dumplicate config, fix unit test --- cmd/genericconf/config.go | 21 --------------------- das/google_cloud_storage_service_test.go | 8 +++++--- go.mod | 3 ++- 3 files changed, 7 insertions(+), 25 deletions(-) diff --git a/cmd/genericconf/config.go b/cmd/genericconf/config.go index 8bde4a972b..06e1fcd12d 100644 --- a/cmd/genericconf/config.go +++ b/cmd/genericconf/config.go @@ -65,27 +65,6 @@ var DefaultS3Config = S3Config{ SecretKey: "", } -type GoogleCloudStorageConfig struct { - AccessToken string `koanf:"access-token"` - Bucket string `koanf:"bucket"` - ObjectPrefix string `koanf:"object-prefix"` - MaxRetention time.Duration `koanf:"max-retention"` -} - -var DefaultGoogleCloudStorageConfig = GoogleCloudStorageConfig{ - AccessToken: "", - Bucket: "", - ObjectPrefix: "", - MaxRetention: time.Hour * 24, -} - -func GoogleCloudConfigAddOptions(prefix string, f *flag.FlagSet) { - f.String(prefix+".access-token", DefaultGoogleCloudStorageConfig.AccessToken, "Google Cloud Storage access token") - f.String(prefix+".bucket", DefaultGoogleCloudStorageConfig.Bucket, "Google Cloud Storage bucket") - f.String(prefix+".object-prefix", DefaultGoogleCloudStorageConfig.ObjectPrefix, "prefix to add to Google Cloud Storage objects") - f.Duration(prefix+".max-retention", DefaultGoogleCloudStorageConfig.MaxRetention, "store requests with expiry times farther in the future than max-retention will be rejected") -} - func HandlerFromLogType(logType string, output io.Writer) (slog.Handler, error) { if logType == "plaintext" { return log.NewTerminalHandler(output, false), nil diff --git a/das/google_cloud_storage_service_test.go b/das/google_cloud_storage_service_test.go index 5482b0c1c2..21cca0abb2 100644 --- a/das/google_cloud_storage_service_test.go +++ b/das/google_cloud_storage_service_test.go @@ -6,7 +6,6 @@ import ( "context" "errors" "github.com/ethereum/go-ethereum/common" - "github.com/offchainlabs/nitro/cmd/genericconf" "github.com/offchainlabs/nitro/das/dastree" "testing" "time" @@ -38,7 +37,7 @@ func (c *mockGCSClient) Upload(ctx context.Context, bucket, objectPrefix string, return nil } -func NewTestGoogleCloudStorageService(ctx context.Context, googleCloudStorageConfig genericconf.GoogleCloudStorageConfig) (StorageService, error) { +func NewTestGoogleCloudStorageService(ctx context.Context, googleCloudStorageConfig GoogleCloudStorageServiceConfig) (StorageService, error) { return &GoogleCloudStorageService{ bucket: googleCloudStorageConfig.Bucket, objectPrefix: googleCloudStorageConfig.ObjectPrefix, @@ -52,7 +51,10 @@ func NewTestGoogleCloudStorageService(ctx context.Context, googleCloudStorageCon func TestNewGoogleCloudStorageService(t *testing.T) { ctx := context.Background() expiry := uint64(time.Now().Add(time.Hour).Unix()) - googleCloudService, err := NewTestGoogleCloudStorageService(ctx, genericconf.DefaultGoogleCloudStorageConfig) + googleCloudStorageServiceConfig := DefaultGoogleCloudStorageServiceConfig + googleCloudStorageServiceConfig.Enable = true + googleCloudStorageServiceConfig.MaxRetention = time.Hour * 24 + googleCloudService, err := NewTestGoogleCloudStorageService(ctx, googleCloudStorageServiceConfig) Require(t, err) val1 := []byte("The first value") diff --git a/go.mod b/go.mod index f44203786e..b6180ee640 100644 --- a/go.mod +++ b/go.mod @@ -48,6 +48,7 @@ require ( golang.org/x/sys v0.21.0 golang.org/x/term v0.21.0 golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d + google.golang.org/api v0.187.0 gopkg.in/natefinch/lumberjack.v2 v2.0.0 ) @@ -58,6 +59,7 @@ require ( cloud.google.com/go/compute/metadata v0.3.0 // indirect cloud.google.com/go/iam v1.1.8 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fjl/memsize v0.0.2 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/google/go-querystring v1.1.0 // indirect @@ -69,7 +71,6 @@ require ( go.opentelemetry.io/otel v1.24.0 // indirect go.opentelemetry.io/otel/metric v1.24.0 // indirect go.opentelemetry.io/otel/trace v1.24.0 // indirect - google.golang.org/api v0.187.0 // indirect google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d // indirect From 3df28169b187187c46dd6bdf0544963e38d06973 Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Thu, 3 Oct 2024 08:20:24 +0800 Subject: [PATCH 307/489] chore: adjust go mod --- go.mod | 1 - go.sum | 2 -- 2 files changed, 3 deletions(-) diff --git a/go.mod b/go.mod index b6180ee640..e6b82a0d42 100644 --- a/go.mod +++ b/go.mod @@ -59,7 +59,6 @@ require ( cloud.google.com/go/compute/metadata v0.3.0 // indirect cloud.google.com/go/iam v1.1.8 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/fjl/memsize v0.0.2 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/google/go-querystring v1.1.0 // indirect diff --git a/go.sum b/go.sum index 6120382844..fa44536f2f 100644 --- a/go.sum +++ b/go.sum @@ -249,8 +249,6 @@ github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4 github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fjl/memsize v0.0.2 h1:27txuSD9or+NZlnOWdKUxeBzTAUkWCVh+4Gf2dWFOzA= -github.com/fjl/memsize v0.0.2/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= From da58307a6b051005c632e01f33be6a8599beb917 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Thu, 3 Oct 2024 02:46:11 +0200 Subject: [PATCH 308/489] system_tests: fix cache tag used when wrapping wasm database in test --- system_tests/common_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 209e7c2d35..fba6aa2fc6 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -1309,7 +1309,7 @@ func createNonL1BlockChainWithStackConfig( Require(t, err) wasmData, err := stack.OpenDatabaseWithExtraOptions("wasm", 0, 0, "wasm/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("wasm")) Require(t, err) - chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmData, 0, execConfig.StylusTarget.WasmTargets()) + chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmData, 1, execConfig.StylusTarget.WasmTargets()) arbDb, err := stack.OpenDatabaseWithExtraOptions("arbitrumdata", 0, 0, "arbitrumdata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("arbitrumdata")) Require(t, err) @@ -1401,7 +1401,7 @@ func Create2ndNodeWithConfig( Require(t, err) wasmData, err := chainStack.OpenDatabaseWithExtraOptions("wasm", 0, 0, "wasm/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("wasm")) Require(t, err) - chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmData, 0, execConfig.StylusTarget.WasmTargets()) + chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmData, 1, execConfig.StylusTarget.WasmTargets()) arbDb, err := chainStack.OpenDatabaseWithExtraOptions("arbitrumdata", 0, 0, "arbitrumdata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("arbitrumdata")) Require(t, err) From 6c69a96856c4be0118a84b554640b09b7d353ef3 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Thu, 3 Oct 2024 02:52:45 +0200 Subject: [PATCH 309/489] add stylus wasm long term cache test --- arbitrator/stylus/src/lib.rs | 7 +++ arbos/programs/native.go | 5 ++ system_tests/program_test.go | 106 ++++++++++++++++++++++++++++++++++- 3 files changed, 117 insertions(+), 1 deletion(-) diff --git a/arbitrator/stylus/src/lib.rs b/arbitrator/stylus/src/lib.rs index f5598198eb..feac828989 100644 --- a/arbitrator/stylus/src/lib.rs +++ b/arbitrator/stylus/src/lib.rs @@ -377,6 +377,13 @@ pub extern "C" fn stylus_clear_lru_cache() { InitCache::clear_lru_cache() } +/// Clears long term cache. +/// Only used for testing purposes. +#[no_mangle] +pub extern "C" fn stylus_clear_long_term_cache(arbos_tag: u32) { + InitCache::clear_long_term(arbos_tag); +} + /// Gets entry size in bytes. /// Only used for testing purposes. #[no_mangle] diff --git a/arbos/programs/native.go b/arbos/programs/native.go index 38da63013d..5baacea381 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -392,6 +392,11 @@ func ClearWasmLruCache() { C.stylus_clear_lru_cache() } +// Used for testing +func ClearWasmLongTermCache(arbos_tag uint32) { + C.stylus_clear_long_term_cache(u32(arbos_tag)) +} + // Used for testing func GetEntrySizeEstimateBytes(module []byte, version uint16, debug bool) uint64 { return uint64(C.stylus_get_entry_size_estimate_bytes(goSlice(module), u16(version), cbool(debug))) diff --git a/system_tests/program_test.go b/system_tests/program_test.go index 21081dc341..1686e212bf 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -1385,7 +1385,7 @@ func TestProgramCacheManager(t *testing.T) { isManager, err := arbWasmCache.IsCacheManager(nil, manager) assert(!isManager, err) - // athorize the manager + // authorize the manager ensure(arbOwner.AddWasmCacheManager(&ownerAuth, manager)) assert(arbWasmCache.IsCacheManager(nil, manager)) all, err := arbWasmCache.AllCacheManagers(nil) @@ -2137,3 +2137,107 @@ func TestWasmLruCache(t *testing.T) { t.Fatalf("lruMetrics.SizeBytes, expected: %v, actual: %v", keccakLruEntrySizeEstimateBytes+mathLruEntrySizeEstimateBytes, lruMetrics.SizeBytes) } } + +func TestWasmLongTermCache(t *testing.T) { + builder, ownerAuth, cleanup := setupProgramTest(t, true) + ctx := builder.ctx + l2info := builder.L2Info + l2client := builder.L2.Client + defer cleanup() + + ensure := func(tx *types.Transaction, err error) *types.Receipt { + t.Helper() + Require(t, err) + receipt, err := EnsureTxSucceeded(ctx, l2client, tx) + Require(t, err) + return receipt + } + + manager, tx, _, err := mocksgen.DeploySimpleCacheManager(&ownerAuth, l2client) + ensure(tx, err) + + arbWasmCache, err := pgen.NewArbWasmCache(types.ArbWasmCacheAddress, builder.L2.Client) + Require(t, err) + arbOwner, err := pgen.NewArbOwner(types.ArbOwnerAddress, builder.L2.Client) + Require(t, err) + ensure(arbOwner.SetInkPrice(&ownerAuth, 10_000)) + + ownerAuth.GasLimit = 32000000 + ownerAuth.Value = oneEth + + fallibleProgramAddress, fallibleEntrySize := deployWasmAndGetEntrySizeEstimateBytes(t, builder, ownerAuth, "fallible") + keccakProgramAddress, keccakEntrySize := deployWasmAndGetEntrySizeEstimateBytes(t, builder, ownerAuth, "keccak") + mathProgramAddress, mathEntrySize := deployWasmAndGetEntrySizeEstimateBytes(t, builder, ownerAuth, "math") + t.Log( + "lruEntrySizeEstimateBytes, ", + "fallible:", fallibleEntrySize, + "keccak:", keccakEntrySize, + "math:", mathEntrySize, + ) + + isManager, err := arbWasmCache.IsCacheManager(nil, manager) + Require(t, err) + t.Log("isManager", isManager) + ownerAuth.Value = common.Big0 + ensure(arbOwner.AddWasmCacheManager(&ownerAuth, manager)) + + checkLongTermMetrics := func(expected programs.WasmLongTermCacheMetrics) { + t.Helper() + longTermMetrics := programs.GetWasmCacheMetrics().LongTerm + if longTermMetrics.Count != expected.Count { + t.Fatalf("longTermMetrics.Count, expected: %v, actual: %v", expected.Count, longTermMetrics.Count) + } + if longTermMetrics.SizeBytes != expected.SizeBytes { + t.Fatalf("longTermMetrics.SizeBytes, expected: %v, actual: %v", expected.SizeBytes, longTermMetrics.SizeBytes) + } + } + + programs.ClearWasmLongTermCache(1) + checkLongTermMetrics(programs.WasmLongTermCacheMetrics{ + Count: 0, + SizeBytes: 0, + }) + + // fallible wasm program will not be cached since caching is not set for this program + tx = l2info.PrepareTxTo("Owner", &fallibleProgramAddress, l2info.TransferGas, nil, []byte{0x01}) + ensure(tx, l2client.SendTransaction(ctx, tx)) + checkLongTermMetrics(programs.WasmLongTermCacheMetrics{ + Count: 0, + SizeBytes: 0, + }) + + ensure(arbWasmCache.CacheProgram(&ownerAuth, fallibleProgramAddress)) + // fallible wasm program will be cached + tx = l2info.PrepareTxTo("Owner", &fallibleProgramAddress, l2info.TransferGas, nil, []byte{0x01}) + ensure(tx, l2client.SendTransaction(ctx, tx)) + checkLongTermMetrics(programs.WasmLongTermCacheMetrics{ + Count: 1, + SizeBytes: fallibleEntrySize, + }) + + // keccak wasm program will be cached + ensure(arbWasmCache.CacheProgram(&ownerAuth, keccakProgramAddress)) + tx = l2info.PrepareTxTo("Owner", &keccakProgramAddress, l2info.TransferGas, nil, []byte{0x01}) + ensure(tx, l2client.SendTransaction(ctx, tx)) + checkLongTermMetrics(programs.WasmLongTermCacheMetrics{ + Count: 2, + SizeBytes: fallibleEntrySize + keccakEntrySize, + }) + + // math wasm program will not be cached + tx = l2info.PrepareTxTo("Owner", &mathProgramAddress, l2info.TransferGas, nil, []byte{0x01}) + ensure(tx, l2client.SendTransaction(ctx, tx)) + checkLongTermMetrics(programs.WasmLongTermCacheMetrics{ + Count: 2, + SizeBytes: fallibleEntrySize + keccakEntrySize, + }) + + // math wasm program will be cached + ensure(arbWasmCache.CacheProgram(&ownerAuth, mathProgramAddress)) + tx = l2info.PrepareTxTo("Owner", &mathProgramAddress, l2info.TransferGas, nil, []byte{0x01}) + ensure(tx, l2client.SendTransaction(ctx, tx)) + checkLongTermMetrics(programs.WasmLongTermCacheMetrics{ + Count: 3, + SizeBytes: fallibleEntrySize + keccakEntrySize + mathEntrySize, + }) +} From 8b1e45baa1f1689e1cd2fb44831813715e4ed025 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Thu, 3 Oct 2024 10:57:18 -0300 Subject: [PATCH 310/489] Set contracts submodule to develop --- contracts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts b/contracts index 6740300fc1..b140ed63ac 160000 --- a/contracts +++ b/contracts @@ -1 +1 @@ -Subproject commit 6740300fc1da9c1d67764068463ed96d7ca9ec2c +Subproject commit b140ed63acdb53cb906ffd1fa3c36fdbd474364e From 7b0fe6d1c8bcb5e36f445d49243d073d7b146378 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Thu, 3 Oct 2024 11:23:03 -0300 Subject: [PATCH 311/489] Use builder variable instead of creating aliases --- system_tests/program_gas_test.go | 108 ++++++++++++++----------------- 1 file changed, 49 insertions(+), 59 deletions(-) diff --git a/system_tests/program_gas_test.go b/system_tests/program_gas_test.go index cdc75345cb..063a21d3b0 100644 --- a/system_tests/program_gas_test.go +++ b/system_tests/program_gas_test.go @@ -24,10 +24,11 @@ import ( ) func TestProgramSimpleCost(t *testing.T) { - ctx, l2info, l2client, auth := setupGasCostTest(t) - stylusProgram := deployWasm(t, ctx, auth, l2client, rustFile("hostio-test")) - evmProgram := deployEvmContract(t, ctx, auth, l2client, mocksgen.HostioTestMetaData) - otherProgram := deployWasm(t, ctx, auth, l2client, rustFile("storage")) + builder := setupGasCostTest(t) + auth := builder.L2Info.GetDefaultTransactOpts("Owner", builder.ctx) + stylusProgram := deployWasm(t, builder.ctx, auth, builder.L2.Client, rustFile("hostio-test")) + evmProgram := deployEvmContract(t, builder.ctx, auth, builder.L2.Client, mocksgen.HostioTestMetaData) + otherProgram := deployWasm(t, builder.ctx, auth, builder.L2.Client, rustFile("storage")) matchSnake := regexp.MustCompile("_[a-z]") for _, tc := range []struct { @@ -40,7 +41,7 @@ func TestProgramSimpleCost(t *testing.T) { {hostio: "transient_load_bytes32", opcode: vm.TLOAD, params: []any{common.HexToHash("dead")}}, {hostio: "transient_store_bytes32", opcode: vm.TSTORE, params: []any{common.HexToHash("dead"), common.HexToHash("beef")}}, {hostio: "return_data_size", opcode: vm.RETURNDATASIZE, maxDiff: 1.0}, - {hostio: "account_balance", opcode: vm.BALANCE, params: []any{l2info.GetAddress("Owner")}}, + {hostio: "account_balance", opcode: vm.BALANCE, params: []any{builder.L2Info.GetAddress("Owner")}}, {hostio: "account_code", opcode: vm.EXTCODECOPY, params: []any{otherProgram}}, {hostio: "account_code_size", opcode: vm.EXTCODESIZE, params: []any{otherProgram}}, {hostio: "account_codehash", opcode: vm.EXTCODEHASH, params: []any{otherProgram}}, @@ -71,17 +72,16 @@ func TestProgramSimpleCost(t *testing.T) { packer, _ := util.NewCallParser(mocksgen.HostioTestABI, solFunc) data, err := packer(tc.params...) Require(t, err) - compareGasUsage(t, ctx, l2client, l2info, evmProgram, stylusProgram, data, nil, - compareGasForEach, tc.maxDiff, compareGasPair{tc.opcode, tc.hostio}) + compareGasUsage(t, builder, evmProgram, stylusProgram, data, nil, compareGasForEach, tc.maxDiff, compareGasPair{tc.opcode, tc.hostio}) }) } } func TestProgramStorageCost(t *testing.T) { - ctx, l2info, l2client, auth := setupGasCostTest(t) - - stylusMulticall := deployWasm(t, ctx, auth, l2client, rustFile("multicall")) - evmMulticall := deployEvmContract(t, ctx, auth, l2client, mocksgen.MultiCallTestMetaData) + builder := setupGasCostTest(t) + auth := builder.L2Info.GetDefaultTransactOpts("Owner", builder.ctx) + stylusMulticall := deployWasm(t, builder.ctx, auth, builder.L2.Client, rustFile("multicall")) + evmMulticall := deployEvmContract(t, builder.ctx, auth, builder.L2.Client, mocksgen.MultiCallTestMetaData) const numSlots = 42 rander := testhelpers.NewPseudoRandomDataSource(t, 0) @@ -109,18 +109,17 @@ func TestProgramStorageCost(t *testing.T) { {"writeAgainAgain", writeRandAData}, } { t.Run(tc.name, func(t *testing.T) { - compareGasUsage(t, ctx, l2client, l2info, evmMulticall, stylusMulticall, tc.data, nil, - compareGasSum, 0, compareGasPair{vm.SSTORE, "storage_flush_cache"}, - compareGasPair{vm.SLOAD, "storage_load_bytes32"}) + compareGasUsage(t, builder, evmMulticall, stylusMulticall, tc.data, nil, compareGasSum, 0, + compareGasPair{vm.SSTORE, "storage_flush_cache"}, compareGasPair{vm.SLOAD, "storage_load_bytes32"}) }) } } func TestProgramLogCost(t *testing.T) { - ctx, l2info, l2client, auth := setupGasCostTest(t) - - stylusProgram := deployWasm(t, ctx, auth, l2client, rustFile("hostio-test")) - evmProgram := deployEvmContract(t, ctx, auth, l2client, mocksgen.HostioTestMetaData) + builder := setupGasCostTest(t) + auth := builder.L2Info.GetDefaultTransactOpts("Owner", builder.ctx) + stylusProgram := deployWasm(t, builder.ctx, auth, builder.L2.Client, rustFile("hostio-test")) + evmProgram := deployEvmContract(t, builder.ctx, auth, builder.L2.Client, mocksgen.HostioTestMetaData) packer, _ := util.NewCallParser(mocksgen.HostioTestABI, "emitLog") for ntopics := int8(0); ntopics < 5; ntopics++ { @@ -137,8 +136,7 @@ func TestProgramLogCost(t *testing.T) { data, err := packer(args...) Require(t, err) opcode := vm.LOG0 + vm.OpCode(ntopics) - compareGasUsage(t, ctx, l2client, l2info, evmProgram, stylusProgram, data, nil, - compareGasForEach, 0, compareGasPair{opcode, "emit_log"}) + compareGasUsage(t, builder, evmProgram, stylusProgram, data, nil, compareGasForEach, 0, compareGasPair{opcode, "emit_log"}) }) } } @@ -146,12 +144,12 @@ func TestProgramLogCost(t *testing.T) { } func TestProgramCallCost(t *testing.T) { - ctx, l2info, l2client, auth := setupGasCostTest(t) - - stylusMulticall := deployWasm(t, ctx, auth, l2client, rustFile("multicall")) - evmMulticall := deployEvmContract(t, ctx, auth, l2client, mocksgen.MultiCallTestMetaData) - otherStylusProgram := deployWasm(t, ctx, auth, l2client, rustFile("hostio-test")) - otherEvmProgram := deployEvmContract(t, ctx, auth, l2client, mocksgen.HostioTestMetaData) + builder := setupGasCostTest(t) + auth := builder.L2Info.GetDefaultTransactOpts("Owner", builder.ctx) + stylusMulticall := deployWasm(t, builder.ctx, auth, builder.L2.Client, rustFile("multicall")) + evmMulticall := deployEvmContract(t, builder.ctx, auth, builder.L2.Client, mocksgen.MultiCallTestMetaData) + otherStylusProgram := deployWasm(t, builder.ctx, auth, builder.L2.Client, rustFile("hostio-test")) + otherEvmProgram := deployEvmContract(t, builder.ctx, auth, builder.L2.Client, mocksgen.HostioTestMetaData) packer, _ := util.NewCallParser(mocksgen.HostioTestABI, "msgValue") otherData, err := packer() Require(t, err) @@ -167,18 +165,18 @@ func TestProgramCallCost(t *testing.T) { burnData, err := burnArbGas(big.NewInt(0)) Require(t, err) data := argsForMulticall(pair.opcode, arbTest, nil, burnData) - compareGasUsage(t, ctx, l2client, l2info, evmMulticall, stylusMulticall, data, nil, compareGasForEach, 0, pair) + compareGasUsage(t, builder, evmMulticall, stylusMulticall, data, nil, compareGasForEach, 0, pair) }) t.Run(pair.hostio+"/evmContract", func(t *testing.T) { data := argsForMulticall(pair.opcode, otherEvmProgram, nil, otherData) - compareGasUsage(t, ctx, l2client, l2info, evmMulticall, stylusMulticall, data, nil, compareGasForEach, 0, pair, + compareGasUsage(t, builder, evmMulticall, stylusMulticall, data, nil, compareGasForEach, 0, pair, compareGasPair{vm.RETURNDATACOPY, "read_return_data"}) // also test read_return_data }) t.Run(pair.hostio+"/stylusContract", func(t *testing.T) { data := argsForMulticall(pair.opcode, otherStylusProgram, nil, otherData) - compareGasUsage(t, ctx, l2client, l2info, evmMulticall, stylusMulticall, data, nil, compareGasForEach, 0, pair, + compareGasUsage(t, builder, evmMulticall, stylusMulticall, data, nil, compareGasForEach, 0, pair, compareGasPair{vm.RETURNDATACOPY, "read_return_data"}) // also test read_return_data }) @@ -187,30 +185,29 @@ func TestProgramCallCost(t *testing.T) { for i := 0; i < 9; i++ { data = multicallAppend(data, pair.opcode, otherEvmProgram, otherData) } - compareGasUsage(t, ctx, l2client, l2info, evmMulticall, stylusMulticall, data, nil, compareGasForEach, 0, pair) + compareGasUsage(t, builder, evmMulticall, stylusMulticall, data, nil, compareGasForEach, 0, pair) }) } t.Run("call_contract/evmContractWithValue", func(t *testing.T) { value := big.NewInt(1000) data := argsForMulticall(vm.CALL, otherEvmProgram, value, otherData) - compareGasUsage(t, ctx, l2client, l2info, evmMulticall, stylusMulticall, data, value, compareGasForEach, 0, compareGasPair{vm.CALL, "call_contract"}) + compareGasUsage(t, builder, evmMulticall, stylusMulticall, data, value, compareGasForEach, 0, compareGasPair{vm.CALL, "call_contract"}) }) } func TestProgramCreateCost(t *testing.T) { - ctx, l2info, l2client, auth := setupGasCostTest(t) - - stylusCreate := deployWasm(t, ctx, auth, l2client, rustFile("create")) - evmCreate := deployEvmContract(t, ctx, auth, l2client, mocksgen.CreateTestMetaData) + builder := setupGasCostTest(t) + auth := builder.L2Info.GetDefaultTransactOpts("Owner", builder.ctx) + stylusCreate := deployWasm(t, builder.ctx, auth, builder.L2.Client, rustFile("create")) + evmCreate := deployEvmContract(t, builder.ctx, auth, builder.L2.Client, mocksgen.CreateTestMetaData) deployCode := common.FromHex(mocksgen.ProgramTestMetaData.Bin) t.Run("create1", func(t *testing.T) { data := []byte{0x01} data = append(data, (common.Hash{}).Bytes()...) // endowment data = append(data, deployCode...) - compareGasUsage(t, ctx, l2client, l2info, evmCreate, stylusCreate, data, nil, - compareGasForEach, 0, compareGasPair{vm.CREATE, "create1"}) + compareGasUsage(t, builder, evmCreate, stylusCreate, data, nil, compareGasForEach, 0, compareGasPair{vm.CREATE, "create1"}) }) t.Run("create2", func(t *testing.T) { @@ -218,16 +215,15 @@ func TestProgramCreateCost(t *testing.T) { data = append(data, (common.Hash{}).Bytes()...) // endowment data = append(data, (common.HexToHash("beef")).Bytes()...) // salt data = append(data, deployCode...) - compareGasUsage(t, ctx, l2client, l2info, evmCreate, stylusCreate, data, nil, - compareGasForEach, 0, compareGasPair{vm.CREATE2, "create2"}) + compareGasUsage(t, builder, evmCreate, stylusCreate, data, nil, compareGasForEach, 0, compareGasPair{vm.CREATE2, "create2"}) }) } func TestProgramKeccakCost(t *testing.T) { - ctx, l2info, l2client, auth := setupGasCostTest(t) - - stylusProgram := deployWasm(t, ctx, auth, l2client, rustFile("hostio-test")) - evmProgram := deployEvmContract(t, ctx, auth, l2client, mocksgen.HostioTestMetaData) + builder := setupGasCostTest(t) + auth := builder.L2Info.GetDefaultTransactOpts("Owner", builder.ctx) + stylusProgram := deployWasm(t, builder.ctx, auth, builder.L2.Client, rustFile("hostio-test")) + evmProgram := deployEvmContract(t, builder.ctx, auth, builder.L2.Client, mocksgen.HostioTestMetaData) packer, _ := util.NewCallParser(mocksgen.HostioTestABI, "keccak") for i := 1; i < 5; i++ { @@ -239,22 +235,18 @@ func TestProgramKeccakCost(t *testing.T) { data, err := packer(preImage) Require(t, err) const maxDiff = 1.1 - compareGasUsage(t, ctx, l2client, l2info, evmProgram, stylusProgram, data, nil, - compareGasForEach, maxDiff, compareGasPair{vm.KECCAK256, "native_keccak256"}) + compareGasUsage(t, builder, evmProgram, stylusProgram, data, nil, compareGasForEach, maxDiff, compareGasPair{vm.KECCAK256, "native_keccak256"}) }) } } -func setupGasCostTest(t *testing.T) (ctx context.Context, l2info *BlockchainTestInfo, l2client *ethclient.Client, auth bind.TransactOpts) { +func setupGasCostTest(t *testing.T) *NodeBuilder { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) builder := NewNodeBuilder(ctx).DefaultConfig(t, true) cleanup := builder.Build(t) t.Cleanup(cleanup) - l2info = builder.L2Info - l2client = builder.L2.Client - auth = builder.L2Info.GetDefaultTransactOpts("Owner", ctx) - return + return builder } // deployEvmContract deploys an Evm contract and return its address. @@ -285,9 +277,7 @@ const ( // Then, it ensures the given opcodes and hostios cost roughly the same amount of gas. func compareGasUsage( t *testing.T, - ctx context.Context, - client *ethclient.Client, - info *BlockchainTestInfo, + builder *NodeBuilder, evmContract common.Address, stylusContract common.Address, txData []byte, @@ -301,14 +291,14 @@ func compareGasUsage( } const txGas uint64 = 32_000_000 - tx := info.PrepareTxTo("Owner", &evmContract, txGas, txValue, txData) - evmGas := sendAndEnsureTransaction(t, ctx, client, tx) - evmGasUsage, err := evmOpcodesGasUsage(ctx, client.Client(), tx) + tx := builder.L2Info.PrepareTxTo("Owner", &evmContract, txGas, txValue, txData) + evmGas := sendAndEnsureTransaction(t, builder.ctx, builder.L2.Client, tx) + evmGasUsage, err := evmOpcodesGasUsage(builder.ctx, builder.L2.Client.Client(), tx) Require(t, err) - tx = info.PrepareTxTo("Owner", &stylusContract, txGas, txValue, txData) - stylusGas := sendAndEnsureTransaction(t, ctx, client, tx) - stylusGasUsage, err := stylusHostiosGasUsage(ctx, client.Client(), tx) + tx = builder.L2Info.PrepareTxTo("Owner", &stylusContract, txGas, txValue, txData) + stylusGas := sendAndEnsureTransaction(t, builder.ctx, builder.L2.Client, tx) + stylusGasUsage, err := stylusHostiosGasUsage(builder.ctx, builder.L2.Client.Client(), tx) Require(t, err) t.Logf("evm total usage: %v - stylus total usage: %v", evmGas, stylusGas) From 103ad5b8df9dc2e46b4cea8a456c4b47d7872adc Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Thu, 3 Oct 2024 11:38:46 -0300 Subject: [PATCH 312/489] Replace util function with existing TestClient one --- system_tests/common_test.go | 13 ++++++++----- system_tests/program_gas_test.go | 26 ++++++++++---------------- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 67bc01c92d..93c38b5eae 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -138,8 +138,8 @@ func (tc *TestClient) GetBaseFeeAt(t *testing.T, blockNum *big.Int) *big.Int { return GetBaseFeeAt(t, tc.Client, tc.ctx, blockNum) } -func (tc *TestClient) SendWaitTestTransactions(t *testing.T, txs []*types.Transaction) { - SendWaitTestTransactions(t, tc.ctx, tc.Client, txs) +func (tc *TestClient) SendWaitTestTransactions(t *testing.T, txs []*types.Transaction) []*types.Receipt { + return SendWaitTestTransactions(t, tc.ctx, tc.Client, txs) } func (tc *TestClient) DeploySimple(t *testing.T, auth bind.TransactOpts) (common.Address, *mocksgen.Simple) { @@ -763,15 +763,18 @@ func (b *NodeBuilder) BridgeBalance(t *testing.T, account string, amount *big.In return BridgeBalance(t, account, amount, b.L1Info, b.L2Info, b.L1.Client, b.L2.Client, b.ctx) } -func SendWaitTestTransactions(t *testing.T, ctx context.Context, client *ethclient.Client, txs []*types.Transaction) { +func SendWaitTestTransactions(t *testing.T, ctx context.Context, client *ethclient.Client, txs []*types.Transaction) []*types.Receipt { t.Helper() + receipts := make([]*types.Receipt, len(txs)) for _, tx := range txs { Require(t, client.SendTransaction(ctx, tx)) } - for _, tx := range txs { - _, err := EnsureTxSucceeded(ctx, client, tx) + for i, tx := range txs { + var err error + receipts[i], err = EnsureTxSucceeded(ctx, client, tx) Require(t, err) } + return receipts } func TransferBalance( diff --git a/system_tests/program_gas_test.go b/system_tests/program_gas_test.go index 063a21d3b0..fdcd53eeac 100644 --- a/system_tests/program_gas_test.go +++ b/system_tests/program_gas_test.go @@ -291,14 +291,18 @@ func compareGasUsage( } const txGas uint64 = 32_000_000 - tx := builder.L2Info.PrepareTxTo("Owner", &evmContract, txGas, txValue, txData) - evmGas := sendAndEnsureTransaction(t, builder.ctx, builder.L2.Client, tx) - evmGasUsage, err := evmOpcodesGasUsage(builder.ctx, builder.L2.Client.Client(), tx) + txs := []*types.Transaction{ + builder.L2Info.PrepareTxTo("Owner", &evmContract, txGas, txValue, txData), + builder.L2Info.PrepareTxTo("Owner", &stylusContract, txGas, txValue, txData), + } + receipts := builder.L2.SendWaitTestTransactions(t, txs) + + evmGas := receipts[0].GasUsedForL2() + evmGasUsage, err := evmOpcodesGasUsage(builder.ctx, builder.L2.Client.Client(), txs[0]) Require(t, err) - tx = builder.L2Info.PrepareTxTo("Owner", &stylusContract, txGas, txValue, txData) - stylusGas := sendAndEnsureTransaction(t, builder.ctx, builder.L2.Client, tx) - stylusGasUsage, err := stylusHostiosGasUsage(builder.ctx, builder.L2.Client.Client(), tx) + stylusGas := receipts[1].GasUsedForL2() + stylusGasUsage, err := stylusHostiosGasUsage(builder.ctx, builder.L2.Client.Client(), txs[1]) Require(t, err) t.Logf("evm total usage: %v - stylus total usage: %v", evmGas, stylusGas) @@ -415,13 +419,3 @@ func checkPercentDiff(t *testing.T, a, b float64, maxAllowedDifference float64) Fatal(t, fmt.Sprintf("gas usages are too different; got %v, max allowed is %v", percentageDifference, maxAllowedDifference)) } } - -// sendAndEnsureTransaction sends a transaction, ensures it succeed, and returns the total gas cost. -func sendAndEnsureTransaction(t *testing.T, ctx context.Context, client *ethclient.Client, tx *types.Transaction) uint64 { - t.Helper() - err := client.SendTransaction(ctx, tx) - Require(t, err) - receipt, err := EnsureTxSucceeded(ctx, client, tx) - Require(t, err) - return receipt.GasUsedForL2() -} From 80c931869df8820a5f9ae3fe3242936f52b440db Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Thu, 3 Oct 2024 17:05:05 +0200 Subject: [PATCH 313/489] system_test: use stylus long term cache only in specific tests --- system_tests/common_test.go | 30 +++++++++++++++++++++++------- system_tests/program_test.go | 4 +++- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/system_tests/common_test.go b/system_tests/common_test.go index fba6aa2fc6..d2fda0e135 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -249,6 +249,7 @@ type NodeBuilder struct { initMessage *arbostypes.ParsedInitMessage l3InitMessage *arbostypes.ParsedInitMessage withProdConfirmPeriodBlocks bool + wasmCacheTag uint32 // Created nodes L1 *TestClient @@ -352,6 +353,15 @@ func (b *NodeBuilder) WithExtraArchs(targets []string) *NodeBuilder { return b } +func (b *NodeBuilder) WithStylusLongTermCache(enabled bool) *NodeBuilder { + if enabled { + b.wasmCacheTag = 1 + } else { + b.wasmCacheTag = 0 + } + return b +} + func (b *NodeBuilder) Build(t *testing.T) func() { b.CheckConfig(t) if b.withL1 { @@ -425,6 +435,8 @@ func buildOnParentChain( initMessage *arbostypes.ParsedInitMessage, addresses *chaininfo.RollupAddresses, + + wasmCacheTag uint32, ) *TestClient { if parentChainTestClient == nil { t.Fatal("must build parent chain before building chain") @@ -436,7 +448,7 @@ func buildOnParentChain( var arbDb ethdb.Database var blockchain *core.BlockChain _, chainTestClient.Stack, chainDb, arbDb, blockchain = createNonL1BlockChainWithStackConfig( - t, chainInfo, dataDir, chainConfig, initMessage, stackConfig, execConfig) + t, chainInfo, dataDir, chainConfig, initMessage, stackConfig, execConfig, wasmCacheTag) var sequencerTxOptsPtr *bind.TransactOpts var dataSigner signature.DataSignerFunc @@ -524,6 +536,8 @@ func (b *NodeBuilder) BuildL3OnL2(t *testing.T) func() { b.l3InitMessage, b.l3Addresses, + + b.wasmCacheTag, ) return func() { @@ -552,6 +566,8 @@ func (b *NodeBuilder) BuildL2OnL1(t *testing.T) func() { b.initMessage, b.addresses, + + b.wasmCacheTag, ) return func() { @@ -573,7 +589,7 @@ func (b *NodeBuilder) BuildL2(t *testing.T) func() { var arbDb ethdb.Database var blockchain *core.BlockChain b.L2Info, b.L2.Stack, chainDb, arbDb, blockchain = createL2BlockChain( - t, b.L2Info, b.dataDir, b.chainConfig, b.execConfig) + t, b.L2Info, b.dataDir, b.chainConfig, b.execConfig, b.wasmCacheTag) Require(t, b.execConfig.Validate()) execConfig := b.execConfig @@ -624,7 +640,7 @@ func (b *NodeBuilder) RestartL2Node(t *testing.T) { } b.L2.cleanup() - l2info, stack, chainDb, arbDb, blockchain := createNonL1BlockChainWithStackConfig(t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, b.execConfig) + l2info, stack, chainDb, arbDb, blockchain := createNonL1BlockChainWithStackConfig(t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, b.execConfig, b.wasmCacheTag) execConfigFetcher := func() *gethexec.Config { return b.execConfig } execNode, err := gethexec.CreateExecutionNode(b.ctx, stack, chainDb, blockchain, nil, execConfigFetcher) @@ -1284,13 +1300,13 @@ func deployOnParentChain( } func createL2BlockChain( - t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, execConfig *gethexec.Config, + t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, execConfig *gethexec.Config, wasmCacheTag uint32, ) (*BlockchainTestInfo, *node.Node, ethdb.Database, ethdb.Database, *core.BlockChain) { - return createNonL1BlockChainWithStackConfig(t, l2info, dataDir, chainConfig, nil, nil, execConfig) + return createNonL1BlockChainWithStackConfig(t, l2info, dataDir, chainConfig, nil, nil, execConfig, wasmCacheTag) } func createNonL1BlockChainWithStackConfig( - t *testing.T, info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, initMessage *arbostypes.ParsedInitMessage, stackConfig *node.Config, execConfig *gethexec.Config, + t *testing.T, info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, initMessage *arbostypes.ParsedInitMessage, stackConfig *node.Config, execConfig *gethexec.Config, wasmCacheTag uint32, ) (*BlockchainTestInfo, *node.Node, ethdb.Database, ethdb.Database, *core.BlockChain) { if info == nil { info = NewArbTestInfo(t, chainConfig.ChainID) @@ -1309,7 +1325,7 @@ func createNonL1BlockChainWithStackConfig( Require(t, err) wasmData, err := stack.OpenDatabaseWithExtraOptions("wasm", 0, 0, "wasm/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("wasm")) Require(t, err) - chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmData, 1, execConfig.StylusTarget.WasmTargets()) + chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmData, wasmCacheTag, execConfig.StylusTarget.WasmTargets()) arbDb, err := stack.OpenDatabaseWithExtraOptions("arbitrumdata", 0, 0, "arbitrumdata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("arbitrumdata")) Require(t, err) diff --git a/system_tests/program_test.go b/system_tests/program_test.go index 1686e212bf..a8c325daf3 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -2139,7 +2139,9 @@ func TestWasmLruCache(t *testing.T) { } func TestWasmLongTermCache(t *testing.T) { - builder, ownerAuth, cleanup := setupProgramTest(t, true) + builder, ownerAuth, cleanup := setupProgramTest(t, true, func(builder *NodeBuilder) { + builder.WithStylusLongTermCache(true) + }) ctx := builder.ctx l2info := builder.L2Info l2client := builder.L2.Client From 86721a1ba2b7971010df963353fdb73f3ec334e8 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 3 Oct 2024 13:14:13 -0300 Subject: [PATCH 314/489] Revert to previous ErrExceedsMaxMempoolSize message --- arbnode/dataposter/data_poster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arbnode/dataposter/data_poster.go b/arbnode/dataposter/data_poster.go index 0502b894a0..acbf9c4cc8 100644 --- a/arbnode/dataposter/data_poster.go +++ b/arbnode/dataposter/data_poster.go @@ -336,7 +336,7 @@ func (p *DataPoster) UsingNoOpStorage() bool { return p.usingNoOpStorage } -var ErrExceedsMaxMempoolSize = errors.New("Node is behind, posting this transaction will exceed max mempool size") +var ErrExceedsMaxMempoolSize = errors.New("posting this transaction will exceed max mempool size") // Does basic check whether posting transaction with specified nonce would // result in exceeding maximum queue length or maximum transactions in mempool. From 00dde9823c4c0f9fdf57d619aca202b6667cb78d Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 3 Oct 2024 14:16:30 -0300 Subject: [PATCH 315/489] error getting staker balance as a warn --- staker/staker.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/staker/staker.go b/staker/staker.go index 21d007703b..5c5e7b60ef 100644 --- a/staker/staker.go +++ b/staker/staker.go @@ -305,7 +305,6 @@ func NewStaker( validatorUtilsAddress common.Address, fatalErr chan<- error, ) (*Staker, error) { - if err := config().Validate(); err != nil { return nil, err } @@ -1224,7 +1223,7 @@ func (s *Staker) updateStakerBalanceMetric(ctx context.Context) { } balance, err := s.client.BalanceAt(ctx, *txSenderAddress, nil) if err != nil { - log.Error("error getting staker balance", "txSenderAddress", *txSenderAddress, "err", err) + log.Warn("error getting staker balance", "txSenderAddress", *txSenderAddress, "err", err) return } stakerBalanceGauge.Update(arbmath.BalancePerEther(balance)) From f0b477a6031cc599458ba63d7c559ccd11e5c744 Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Fri, 4 Oct 2024 01:42:47 +0800 Subject: [PATCH 316/489] fix: lint --- das/google_cloud_storage_service_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/das/google_cloud_storage_service_test.go b/das/google_cloud_storage_service_test.go index 21cca0abb2..799d999bad 100644 --- a/das/google_cloud_storage_service_test.go +++ b/das/google_cloud_storage_service_test.go @@ -50,6 +50,7 @@ func NewTestGoogleCloudStorageService(ctx context.Context, googleCloudStorageCon func TestNewGoogleCloudStorageService(t *testing.T) { ctx := context.Background() + // #nosec G115 expiry := uint64(time.Now().Add(time.Hour).Unix()) googleCloudStorageServiceConfig := DefaultGoogleCloudStorageServiceConfig googleCloudStorageServiceConfig.Enable = true From 1a93f16fc89ded62030f495a20063524cc9f349c Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 3 Oct 2024 15:31:30 -0300 Subject: [PATCH 317/489] blockValidationPendingEphemeralErrorHandler --- staker/staker.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/staker/staker.go b/staker/staker.go index 5c5e7b60ef..45e6f6f551 100644 --- a/staker/staker.go +++ b/staker/staker.go @@ -513,6 +513,7 @@ func (s *Staker) Start(ctxIn context.Context) { backoff := time.Second isAheadOfOnChainNonceEphemeralErrorHandler := util.NewEphemeralErrorHandler(10*time.Minute, "is ahead of on-chain nonce", 0) exceedsMaxMempoolSizeEphemeralErrorHandler := util.NewEphemeralErrorHandler(10*time.Minute, dataposter.ErrExceedsMaxMempoolSize.Error(), 0) + blockValidationPendingEphemeralErrorHandler := util.NewEphemeralErrorHandler(10*time.Minute, "block validation is still pending", 0) s.CallIteratively(func(ctx context.Context) (returningWait time.Duration) { defer func() { panicErr := recover() @@ -548,6 +549,7 @@ func (s *Staker) Start(ctxIn context.Context) { if err == nil { isAheadOfOnChainNonceEphemeralErrorHandler.Reset() exceedsMaxMempoolSizeEphemeralErrorHandler.Reset() + blockValidationPendingEphemeralErrorHandler.Reset() backoff = time.Second stakerLastSuccessfulActionGauge.Update(time.Now().Unix()) stakerActionSuccessCounter.Inc(1) @@ -567,6 +569,7 @@ func (s *Staker) Start(ctxIn context.Context) { } logLevel = isAheadOfOnChainNonceEphemeralErrorHandler.LogLevel(err, logLevel) logLevel = exceedsMaxMempoolSizeEphemeralErrorHandler.LogLevel(err, logLevel) + logLevel = blockValidationPendingEphemeralErrorHandler.LogLevel(err, logLevel) logLevel("error acting as staker", "err", err) return backoff }) From b2b54f4fcc0597cfdd98f7de5fb0f7b7ca3a58a6 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Thu, 3 Oct 2024 15:38:25 -0300 Subject: [PATCH 318/489] Move math_pow gas test to a separate function --- system_tests/program_gas_test.go | 79 +++++++++++++++++++++++--------- 1 file changed, 58 insertions(+), 21 deletions(-) diff --git a/system_tests/program_gas_test.go b/system_tests/program_gas_test.go index fdcd53eeac..e434d72f2e 100644 --- a/system_tests/program_gas_test.go +++ b/system_tests/program_gas_test.go @@ -56,7 +56,6 @@ func TestProgramSimpleCost(t *testing.T) { {hostio: "contract_address", opcode: vm.ADDRESS, maxDiff: 0.5}, {hostio: "math_div", opcode: vm.DIV, params: []any{big.NewInt(1), big.NewInt(3)}}, {hostio: "math_mod", opcode: vm.MOD, params: []any{big.NewInt(1), big.NewInt(3)}}, - {hostio: "math_pow", opcode: vm.EXP, params: []any{big.NewInt(1), new(big.Int).Lsh(big.NewInt(1), 255)}, maxDiff: 2}, // worst case {hostio: "math_add_mod", opcode: vm.ADDMOD, params: []any{big.NewInt(1), big.NewInt(3), big.NewInt(5)}, maxDiff: 0.5}, {hostio: "math_mul_mod", opcode: vm.MULMOD, params: []any{big.NewInt(1), big.NewInt(3), big.NewInt(5)}, maxDiff: 0.5}, {hostio: "msg_sender", opcode: vm.CALLER, maxDiff: 0.5}, @@ -77,6 +76,32 @@ func TestProgramSimpleCost(t *testing.T) { } } +func TestProgramPowCost(t *testing.T) { + builder := setupGasCostTest(t) + auth := builder.L2Info.GetDefaultTransactOpts("Owner", builder.ctx) + stylusProgram := deployWasm(t, builder.ctx, auth, builder.L2.Client, rustFile("hostio-test")) + evmProgram := deployEvmContract(t, builder.ctx, auth, builder.L2.Client, mocksgen.HostioTestMetaData) + packer, _ := util.NewCallParser(mocksgen.HostioTestABI, "mathPow") + + for _, exponentNumBytes := range []uint{1, 2, 10, 32} { + name := fmt.Sprintf("exponentNumBytes%v", exponentNumBytes) + t.Run(name, func(t *testing.T) { + exponent := new(big.Int).Lsh(big.NewInt(1), exponentNumBytes*8-1) + params := []any{big.NewInt(1), exponent} + data, err := packer(params...) + Require(t, err) + evmGasUsage, stylusGasUsage := measureGasUsage(t, builder, evmProgram, stylusProgram, data, nil) + expectedGas := 2.652 + 1.75*float64(exponentNumBytes+1) + t.Logf("evm EXP usage: %v - stylus math_pow usage: %v - expected math_pow usage: %v", + evmGasUsage[vm.EXP][0], stylusGasUsage["math_pow"][0], expectedGas) + // The math_pow HostIO uses significally less gas than the EXP opcode. So, + // instead of comparing it to EVM, we compare it to the expected gas usage + // for each test case. + checkPercentDiff(t, stylusGasUsage["math_pow"][0], expectedGas, 0.001) + }) + } +} + func TestProgramStorageCost(t *testing.T) { builder := setupGasCostTest(t) auth := builder.L2Info.GetDefaultTransactOpts("Owner", builder.ctx) @@ -261,6 +286,35 @@ func deployEvmContract(t *testing.T, ctx context.Context, auth bind.TransactOpts return address } +// measureGasUsage calls an EVM and a Wasm contract passing the same data and the same value. +func measureGasUsage( + t *testing.T, + builder *NodeBuilder, + evmContract common.Address, + stylusContract common.Address, + txData []byte, + txValue *big.Int, +) (map[vm.OpCode][]uint64, map[string][]float64) { + const txGas uint64 = 32_000_000 + txs := []*types.Transaction{ + builder.L2Info.PrepareTxTo("Owner", &evmContract, txGas, txValue, txData), + builder.L2Info.PrepareTxTo("Owner", &stylusContract, txGas, txValue, txData), + } + receipts := builder.L2.SendWaitTestTransactions(t, txs) + + evmGas := receipts[0].GasUsedForL2() + evmGasUsage, err := evmOpcodesGasUsage(builder.ctx, builder.L2.Client.Client(), txs[0]) + Require(t, err) + + stylusGas := receipts[1].GasUsedForL2() + stylusGasUsage, err := stylusHostiosGasUsage(builder.ctx, builder.L2.Client.Client(), txs[1]) + Require(t, err) + + t.Logf("evm total usage: %v - stylus total usage: %v", evmGas, stylusGas) + + return evmGasUsage, stylusGasUsage +} + type compareGasPair struct { opcode vm.OpCode hostio string @@ -273,8 +327,8 @@ const ( compareGasSum ) -// compareGasUsage calls an EVM and a Wasm contract passing the same data and the same value. -// Then, it ensures the given opcodes and hostios cost roughly the same amount of gas. +// compareGasUsage calls measureGasUsage and then it ensures the given opcodes and hostios cost +// roughly the same amount of gas. func compareGasUsage( t *testing.T, builder *NodeBuilder, @@ -289,24 +343,7 @@ func compareGasUsage( if evmContract == stylusContract { Fatal(t, "evm and stylus contract are the same") } - - const txGas uint64 = 32_000_000 - txs := []*types.Transaction{ - builder.L2Info.PrepareTxTo("Owner", &evmContract, txGas, txValue, txData), - builder.L2Info.PrepareTxTo("Owner", &stylusContract, txGas, txValue, txData), - } - receipts := builder.L2.SendWaitTestTransactions(t, txs) - - evmGas := receipts[0].GasUsedForL2() - evmGasUsage, err := evmOpcodesGasUsage(builder.ctx, builder.L2.Client.Client(), txs[0]) - Require(t, err) - - stylusGas := receipts[1].GasUsedForL2() - stylusGasUsage, err := stylusHostiosGasUsage(builder.ctx, builder.L2.Client.Client(), txs[1]) - Require(t, err) - - t.Logf("evm total usage: %v - stylus total usage: %v", evmGas, stylusGas) - + evmGasUsage, stylusGasUsage := measureGasUsage(t, builder, evmContract, stylusContract, txData, txValue) for i := range pairs { opcode := pairs[i].opcode hostio := pairs[i].hostio From 95732ed32d14edad339b1c6cad66bedbb8672beb Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Thu, 3 Oct 2024 16:17:28 -0300 Subject: [PATCH 319/489] Make percentDiff formula easier to understand --- system_tests/program_gas_test.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/system_tests/program_gas_test.go b/system_tests/program_gas_test.go index e434d72f2e..119897cbfe 100644 --- a/system_tests/program_gas_test.go +++ b/system_tests/program_gas_test.go @@ -40,28 +40,28 @@ func TestProgramSimpleCost(t *testing.T) { {hostio: "exit_early", opcode: vm.STOP}, {hostio: "transient_load_bytes32", opcode: vm.TLOAD, params: []any{common.HexToHash("dead")}}, {hostio: "transient_store_bytes32", opcode: vm.TSTORE, params: []any{common.HexToHash("dead"), common.HexToHash("beef")}}, - {hostio: "return_data_size", opcode: vm.RETURNDATASIZE, maxDiff: 1.0}, + {hostio: "return_data_size", opcode: vm.RETURNDATASIZE, maxDiff: 1.5}, {hostio: "account_balance", opcode: vm.BALANCE, params: []any{builder.L2Info.GetAddress("Owner")}}, {hostio: "account_code", opcode: vm.EXTCODECOPY, params: []any{otherProgram}}, - {hostio: "account_code_size", opcode: vm.EXTCODESIZE, params: []any{otherProgram}}, + {hostio: "account_code_size", opcode: vm.EXTCODESIZE, params: []any{otherProgram}, maxDiff: 0.3}, {hostio: "account_codehash", opcode: vm.EXTCODEHASH, params: []any{otherProgram}}, - {hostio: "evm_gas_left", opcode: vm.GAS, maxDiff: 1.0}, - {hostio: "evm_ink_left", opcode: vm.GAS, maxDiff: 1.0}, + {hostio: "evm_gas_left", opcode: vm.GAS, maxDiff: 1.5}, + {hostio: "evm_ink_left", opcode: vm.GAS, maxDiff: 1.5}, {hostio: "block_basefee", opcode: vm.BASEFEE, maxDiff: 0.5}, - {hostio: "chainid", opcode: vm.CHAINID, maxDiff: 1.0}, + {hostio: "chainid", opcode: vm.CHAINID, maxDiff: 1.5}, {hostio: "block_coinbase", opcode: vm.COINBASE, maxDiff: 0.5}, - {hostio: "block_gas_limit", opcode: vm.GASLIMIT, maxDiff: 1.0}, - {hostio: "block_number", opcode: vm.NUMBER, maxDiff: 1.0}, - {hostio: "block_timestamp", opcode: vm.TIMESTAMP, maxDiff: 1.0}, + {hostio: "block_gas_limit", opcode: vm.GASLIMIT, maxDiff: 1.5}, + {hostio: "block_number", opcode: vm.NUMBER, maxDiff: 1.5}, + {hostio: "block_timestamp", opcode: vm.TIMESTAMP, maxDiff: 1.5}, {hostio: "contract_address", opcode: vm.ADDRESS, maxDiff: 0.5}, {hostio: "math_div", opcode: vm.DIV, params: []any{big.NewInt(1), big.NewInt(3)}}, {hostio: "math_mod", opcode: vm.MOD, params: []any{big.NewInt(1), big.NewInt(3)}}, - {hostio: "math_add_mod", opcode: vm.ADDMOD, params: []any{big.NewInt(1), big.NewInt(3), big.NewInt(5)}, maxDiff: 0.5}, - {hostio: "math_mul_mod", opcode: vm.MULMOD, params: []any{big.NewInt(1), big.NewInt(3), big.NewInt(5)}, maxDiff: 0.5}, + {hostio: "math_add_mod", opcode: vm.ADDMOD, params: []any{big.NewInt(1), big.NewInt(3), big.NewInt(5)}, maxDiff: 0.7}, + {hostio: "math_mul_mod", opcode: vm.MULMOD, params: []any{big.NewInt(1), big.NewInt(3), big.NewInt(5)}, maxDiff: 0.7}, {hostio: "msg_sender", opcode: vm.CALLER, maxDiff: 0.5}, {hostio: "msg_value", opcode: vm.CALLVALUE, maxDiff: 0.5}, {hostio: "tx_gas_price", opcode: vm.GASPRICE, maxDiff: 0.5}, - {hostio: "tx_ink_price", opcode: vm.GASPRICE, maxDiff: 1.0}, + {hostio: "tx_ink_price", opcode: vm.GASPRICE, maxDiff: 1.5}, {hostio: "tx_origin", opcode: vm.ORIGIN, maxDiff: 0.5}, } { t.Run(tc.hostio, func(t *testing.T) { @@ -259,7 +259,7 @@ func TestProgramKeccakCost(t *testing.T) { preImage[len(preImage)-1] = 0 data, err := packer(preImage) Require(t, err) - const maxDiff = 1.1 + const maxDiff = 2.5 // stylus keccak charges significantly less gas compareGasUsage(t, builder, evmProgram, stylusProgram, data, nil, compareGasForEach, maxDiff, compareGasPair{vm.KECCAK256, "native_keccak256"}) }) } @@ -451,7 +451,7 @@ func checkPercentDiff(t *testing.T, a, b float64, maxAllowedDifference float64) if maxAllowedDifference == 0 { maxAllowedDifference = 0.25 } - percentageDifference := math.Abs(a-b) / ((a + b) / 2) + percentageDifference := (max(a, b) / min(a, b)) - 1 if percentageDifference > maxAllowedDifference { Fatal(t, fmt.Sprintf("gas usages are too different; got %v, max allowed is %v", percentageDifference, maxAllowedDifference)) } From 4676459bf41d35cdc59a3dac3dfec6026a2e9923 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Thu, 3 Oct 2024 23:04:42 +0200 Subject: [PATCH 320/489] fix tracing long term cache size when adding item from lru --- arbitrator/stylus/src/cache.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index 1502fa82e6..0a4b73c2a6 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -184,6 +184,7 @@ impl InitCache { cache.lru_counters.hits += 1; if long_term_tag == Self::ARBOS_TAG { cache.long_term.insert(key, item.clone()); + cache.long_term_size_bytes += item.entry_size_estimate_bytes; } else { // only calls get to move the key to the head of the LRU list cache.lru.get(&key); From 0ab06c87cce2e8199580b8b69349711589ec2f37 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Thu, 3 Oct 2024 23:10:36 +0200 Subject: [PATCH 321/489] add test for adding wasm from lru cache to long term cache --- system_tests/program_test.go | 222 +++++++++++++++++++++++++++++++---- 1 file changed, 197 insertions(+), 25 deletions(-) diff --git a/system_tests/program_test.go b/system_tests/program_test.go index a8c325daf3..6ca04976e5 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -2138,6 +2138,28 @@ func TestWasmLruCache(t *testing.T) { } } +func checkLongTermCacheMetrics(t *testing.T, expected programs.WasmLongTermCacheMetrics) { + t.Helper() + longTermMetrics := programs.GetWasmCacheMetrics().LongTerm + if longTermMetrics.Count != expected.Count { + t.Fatalf("longTermMetrics.Count, expected: %v, actual: %v", expected.Count, longTermMetrics.Count) + } + if longTermMetrics.SizeBytes != expected.SizeBytes { + t.Fatalf("longTermMetrics.SizeBytes, expected: %v, actual: %v", expected.SizeBytes, longTermMetrics.SizeBytes) + } +} + +func checkLruCacheMetrics(t *testing.T, expected programs.WasmLruCacheMetrics) { + t.Helper() + lruMetrics := programs.GetWasmCacheMetrics().Lru + if lruMetrics.Count != expected.Count { + t.Fatalf("lruMetrics.Count, expected: %v, actual: %v", expected.Count, lruMetrics.Count) + } + if lruMetrics.SizeBytes != expected.SizeBytes { + t.Fatalf("lruMetrics.SizeBytes, expected: %v, actual: %v", expected.SizeBytes, lruMetrics.SizeBytes) + } +} + func TestWasmLongTermCache(t *testing.T) { builder, ownerAuth, cleanup := setupProgramTest(t, true, func(builder *NodeBuilder) { builder.WithStylusLongTermCache(true) @@ -2155,9 +2177,6 @@ func TestWasmLongTermCache(t *testing.T) { return receipt } - manager, tx, _, err := mocksgen.DeploySimpleCacheManager(&ownerAuth, l2client) - ensure(tx, err) - arbWasmCache, err := pgen.NewArbWasmCache(types.ArbWasmCacheAddress, builder.L2.Client) Require(t, err) arbOwner, err := pgen.NewArbOwner(types.ArbOwnerAddress, builder.L2.Client) @@ -2177,33 +2196,18 @@ func TestWasmLongTermCache(t *testing.T) { "math:", mathEntrySize, ) - isManager, err := arbWasmCache.IsCacheManager(nil, manager) - Require(t, err) - t.Log("isManager", isManager) ownerAuth.Value = common.Big0 - ensure(arbOwner.AddWasmCacheManager(&ownerAuth, manager)) - - checkLongTermMetrics := func(expected programs.WasmLongTermCacheMetrics) { - t.Helper() - longTermMetrics := programs.GetWasmCacheMetrics().LongTerm - if longTermMetrics.Count != expected.Count { - t.Fatalf("longTermMetrics.Count, expected: %v, actual: %v", expected.Count, longTermMetrics.Count) - } - if longTermMetrics.SizeBytes != expected.SizeBytes { - t.Fatalf("longTermMetrics.SizeBytes, expected: %v, actual: %v", expected.SizeBytes, longTermMetrics.SizeBytes) - } - } programs.ClearWasmLongTermCache(1) - checkLongTermMetrics(programs.WasmLongTermCacheMetrics{ + checkLongTermCacheMetrics(t, programs.WasmLongTermCacheMetrics{ Count: 0, SizeBytes: 0, }) // fallible wasm program will not be cached since caching is not set for this program - tx = l2info.PrepareTxTo("Owner", &fallibleProgramAddress, l2info.TransferGas, nil, []byte{0x01}) + tx := l2info.PrepareTxTo("Owner", &fallibleProgramAddress, l2info.TransferGas, nil, []byte{0x01}) ensure(tx, l2client.SendTransaction(ctx, tx)) - checkLongTermMetrics(programs.WasmLongTermCacheMetrics{ + checkLongTermCacheMetrics(t, programs.WasmLongTermCacheMetrics{ Count: 0, SizeBytes: 0, }) @@ -2212,7 +2216,7 @@ func TestWasmLongTermCache(t *testing.T) { // fallible wasm program will be cached tx = l2info.PrepareTxTo("Owner", &fallibleProgramAddress, l2info.TransferGas, nil, []byte{0x01}) ensure(tx, l2client.SendTransaction(ctx, tx)) - checkLongTermMetrics(programs.WasmLongTermCacheMetrics{ + checkLongTermCacheMetrics(t, programs.WasmLongTermCacheMetrics{ Count: 1, SizeBytes: fallibleEntrySize, }) @@ -2221,7 +2225,7 @@ func TestWasmLongTermCache(t *testing.T) { ensure(arbWasmCache.CacheProgram(&ownerAuth, keccakProgramAddress)) tx = l2info.PrepareTxTo("Owner", &keccakProgramAddress, l2info.TransferGas, nil, []byte{0x01}) ensure(tx, l2client.SendTransaction(ctx, tx)) - checkLongTermMetrics(programs.WasmLongTermCacheMetrics{ + checkLongTermCacheMetrics(t, programs.WasmLongTermCacheMetrics{ Count: 2, SizeBytes: fallibleEntrySize + keccakEntrySize, }) @@ -2229,7 +2233,7 @@ func TestWasmLongTermCache(t *testing.T) { // math wasm program will not be cached tx = l2info.PrepareTxTo("Owner", &mathProgramAddress, l2info.TransferGas, nil, []byte{0x01}) ensure(tx, l2client.SendTransaction(ctx, tx)) - checkLongTermMetrics(programs.WasmLongTermCacheMetrics{ + checkLongTermCacheMetrics(t, programs.WasmLongTermCacheMetrics{ Count: 2, SizeBytes: fallibleEntrySize + keccakEntrySize, }) @@ -2238,8 +2242,176 @@ func TestWasmLongTermCache(t *testing.T) { ensure(arbWasmCache.CacheProgram(&ownerAuth, mathProgramAddress)) tx = l2info.PrepareTxTo("Owner", &mathProgramAddress, l2info.TransferGas, nil, []byte{0x01}) ensure(tx, l2client.SendTransaction(ctx, tx)) - checkLongTermMetrics(programs.WasmLongTermCacheMetrics{ + checkLongTermCacheMetrics(t, programs.WasmLongTermCacheMetrics{ Count: 3, SizeBytes: fallibleEntrySize + keccakEntrySize + mathEntrySize, }) + + statedb, err := builder.L2.ExecNode.Backend.ArbInterface().BlockChain().State() + Require(t, err) + fallibleProgramHash := statedb.GetCodeHash(fallibleProgramAddress) + keccakProgramHash := statedb.GetCodeHash(keccakProgramAddress) + mathProgramHash := statedb.GetCodeHash(mathProgramAddress) + + ensure(arbWasmCache.EvictCodehash(&ownerAuth, keccakProgramHash)) + checkLongTermCacheMetrics(t, programs.WasmLongTermCacheMetrics{ + Count: 2, + SizeBytes: fallibleEntrySize + mathEntrySize, + }) + + // keccak wasm program will not be cached + tx = l2info.PrepareTxTo("Owner", &keccakProgramAddress, l2info.TransferGas, nil, []byte{0x01}) + ensure(tx, l2client.SendTransaction(ctx, tx)) + checkLongTermCacheMetrics(t, programs.WasmLongTermCacheMetrics{ + Count: 2, + SizeBytes: fallibleEntrySize + mathEntrySize, + }) + + // keccak wasm program will be cached + ensure(arbWasmCache.CacheProgram(&ownerAuth, keccakProgramAddress)) + tx = l2info.PrepareTxTo("Owner", &mathProgramAddress, l2info.TransferGas, nil, []byte{0x01}) + ensure(tx, l2client.SendTransaction(ctx, tx)) + checkLongTermCacheMetrics(t, programs.WasmLongTermCacheMetrics{ + Count: 3, + SizeBytes: fallibleEntrySize + keccakEntrySize + mathEntrySize, + }) + + ensure(arbWasmCache.EvictCodehash(&ownerAuth, fallibleProgramHash)) + checkLongTermCacheMetrics(t, programs.WasmLongTermCacheMetrics{ + Count: 2, + SizeBytes: keccakEntrySize + mathEntrySize, + }) + + ensure(arbWasmCache.EvictCodehash(&ownerAuth, mathProgramHash)) + checkLongTermCacheMetrics(t, programs.WasmLongTermCacheMetrics{ + Count: 1, + SizeBytes: keccakEntrySize, + }) + + ensure(arbWasmCache.EvictCodehash(&ownerAuth, keccakProgramHash)) + checkLongTermCacheMetrics(t, programs.WasmLongTermCacheMetrics{ + Count: 0, + SizeBytes: 0, + }) +} + +func TestRepopulateWasmLongTermCacheFromLru(t *testing.T) { + builder, ownerAuth, cleanup := setupProgramTest(t, true, func(builder *NodeBuilder) { + builder.WithStylusLongTermCache(true) + }) + ctx := builder.ctx + l2info := builder.L2Info + l2client := builder.L2.Client + defer cleanup() + + ensure := func(tx *types.Transaction, err error) *types.Receipt { + t.Helper() + Require(t, err) + receipt, err := EnsureTxSucceeded(ctx, l2client, tx) + Require(t, err) + return receipt + } + + arbWasmCache, err := pgen.NewArbWasmCache(types.ArbWasmCacheAddress, builder.L2.Client) + Require(t, err) + arbOwner, err := pgen.NewArbOwner(types.ArbOwnerAddress, builder.L2.Client) + Require(t, err) + ensure(arbOwner.SetInkPrice(&ownerAuth, 10_000)) + + ownerAuth.GasLimit = 32000000 + ownerAuth.Value = oneEth + + fallibleProgramAddress, fallibleEntrySize := deployWasmAndGetEntrySizeEstimateBytes(t, builder, ownerAuth, "fallible") + keccakProgramAddress, keccakEntrySize := deployWasmAndGetEntrySizeEstimateBytes(t, builder, ownerAuth, "keccak") + mathProgramAddress, mathEntrySize := deployWasmAndGetEntrySizeEstimateBytes(t, builder, ownerAuth, "math") + + ownerAuth.Value = common.Big0 + + programs.ClearWasmLongTermCache(1) + programs.ClearWasmLruCache() + // only 2 out of 3 programs should fit lru + programs.SetWasmLruCacheCapacity( + fallibleEntrySize + keccakEntrySize + mathEntrySize - 1, + ) + + checkLongTermCacheMetrics(t, programs.WasmLongTermCacheMetrics{ + Count: 0, + SizeBytes: 0, + }) + checkLruCacheMetrics(t, programs.WasmLruCacheMetrics{ + Count: 0, + SizeBytes: 0, + }) + + ensure(arbWasmCache.CacheProgram(&ownerAuth, fallibleProgramAddress)) + checkLruCacheMetrics(t, programs.WasmLruCacheMetrics{ + Count: 0, + SizeBytes: 0, + }) + checkLongTermCacheMetrics(t, programs.WasmLongTermCacheMetrics{ + Count: 1, + SizeBytes: fallibleEntrySize, + }) + + // clear long term cache to emulate restart + programs.ClearWasmLongTermCache(1) + programs.ClearWasmLruCache() + + checkLruCacheMetrics(t, programs.WasmLruCacheMetrics{ + Count: 0, + SizeBytes: 0, + }) + checkLongTermCacheMetrics(t, programs.WasmLongTermCacheMetrics{ + Count: 0, + SizeBytes: 0, + }) + + nonce := builder.L2Info.GetInfoWithPrivKey("Owner").Nonce.Load() + tx := l2info.PrepareTxTo("Owner", &fallibleProgramAddress, l2info.TransferGas, nil, []byte{0x01}) + _, err = arbutil.SendTxAsCall(ctx, l2client, tx, l2info.GetAddress("Owner"), nil, true) + Require(t, err) + // restore nonce in L2Info + builder.L2Info.GetInfoWithPrivKey("Owner").Nonce.Store(nonce) + checkLruCacheMetrics(t, programs.WasmLruCacheMetrics{ + Count: 1, + SizeBytes: fallibleEntrySize, + }) + checkLongTermCacheMetrics(t, programs.WasmLongTermCacheMetrics{ + Count: 0, + SizeBytes: 0, + }) + + tx = l2info.PrepareTxTo("Owner", &keccakProgramAddress, l2info.TransferGas, nil, []byte{0x01}) + ensure(tx, l2client.SendTransaction(ctx, tx)) + checkLruCacheMetrics(t, programs.WasmLruCacheMetrics{ + Count: 2, + SizeBytes: fallibleEntrySize + keccakEntrySize, + }) + checkLongTermCacheMetrics(t, programs.WasmLongTermCacheMetrics{ + Count: 0, + SizeBytes: 0, + }) + + tx = l2info.PrepareTxTo("Owner", &fallibleProgramAddress, l2info.TransferGas, nil, []byte{0x01}) + ensure(tx, l2client.SendTransaction(ctx, tx)) + checkLruCacheMetrics(t, programs.WasmLruCacheMetrics{ + Count: 2, + SizeBytes: fallibleEntrySize + keccakEntrySize, + }) + checkLongTermCacheMetrics(t, programs.WasmLongTermCacheMetrics{ + Count: 1, + SizeBytes: fallibleEntrySize, + }) + + // mathProgram should end up in lru cache and as result fallibleProgram should be evicted as least recently used item (tx that restores the program back to long term cache shouldn't promote the lru item); fallibleProgram should remain in long term cache + tx = l2info.PrepareTxTo("Owner", &mathProgramAddress, l2info.TransferGas, nil, []byte{0x01}) + ensure(tx, l2client.SendTransaction(ctx, tx)) + checkLruCacheMetrics(t, programs.WasmLruCacheMetrics{ + Count: 2, + SizeBytes: keccakEntrySize + mathEntrySize, + }) + checkLongTermCacheMetrics(t, programs.WasmLongTermCacheMetrics{ + Count: 1, + SizeBytes: fallibleEntrySize, + }) } From f65156d6a6f3d61110503b5ba97fefa50c02405e Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Thu, 3 Oct 2024 23:38:16 +0200 Subject: [PATCH 322/489] don't clone cache item twice --- arbitrator/stylus/src/cache.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index 0a4b73c2a6..208f45e26c 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -182,14 +182,15 @@ impl InitCache { // See if the item is in the LRU cache, promoting if so if let Some(item) = cache.lru.peek(&key).cloned() { cache.lru_counters.hits += 1; + let data = item.data(); if long_term_tag == Self::ARBOS_TAG { - cache.long_term.insert(key, item.clone()); cache.long_term_size_bytes += item.entry_size_estimate_bytes; + cache.long_term.insert(key, item); } else { // only calls get to move the key to the head of the LRU list cache.lru.get(&key); } - return Some(item.data()); + return Some(data); } cache.lru_counters.misses += 1; From 30abdc5f8aee6cfcd1e9c0cec52940d541aa1377 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Thu, 3 Oct 2024 20:33:25 -0500 Subject: [PATCH 323/489] Update aws-sd-go-v2 --- go.mod | 39 ++++++++++++------------- go.sum | 91 ++++++++++++++++++++++++---------------------------------- 2 files changed, 57 insertions(+), 73 deletions(-) diff --git a/go.mod b/go.mod index bbe851fe02..8e7302a9b0 100644 --- a/go.mod +++ b/go.mod @@ -11,11 +11,11 @@ require ( github.com/Shopify/toxiproxy v2.1.4+incompatible github.com/alicebob/miniredis/v2 v2.32.1 github.com/andybalholm/brotli v1.0.4 - github.com/aws/aws-sdk-go-v2 v1.21.2 - github.com/aws/aws-sdk-go-v2/config v1.18.45 - github.com/aws/aws-sdk-go-v2/credentials v1.13.43 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.10 - github.com/aws/aws-sdk-go-v2/service/s3 v1.26.9 + github.com/aws/aws-sdk-go-v2 v1.31.0 + github.com/aws/aws-sdk-go-v2/config v1.27.40 + github.com/aws/aws-sdk-go-v2/credentials v1.17.38 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.27 + github.com/aws/aws-sdk-go-v2/service/s3 v1.64.1 github.com/cavaliergopher/grab/v3 v3.0.1 github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 github.com/codeclysm/extract/v3 v3.0.2 @@ -58,20 +58,20 @@ require ( github.com/StackExchange/wmi v1.2.1 // indirect github.com/VictoriaMetrics/fastcache v1.12.1 // indirect github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.1 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.1 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.1 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.5 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.4 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 // indirect - github.com/aws/smithy-go v1.15.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.5 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.14 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.18 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.18 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.18 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.20 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.20 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.18 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.23.4 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.27.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.31.4 // indirect + github.com/aws/smithy-go v1.22.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.10.0 // indirect github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect @@ -121,7 +121,6 @@ require ( github.com/holiman/bloomfilter/v2 v2.0.3 // indirect github.com/huin/goupnp v1.3.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5 // indirect github.com/juju/loggo v0.0.0-20180524022052-584905176618 // indirect github.com/klauspost/compress v1.17.2 // indirect diff --git a/go.sum b/go.sum index 8a432408d4..9758c1f2b3 100644 --- a/go.sum +++ b/go.sum @@ -70,64 +70,53 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go-v2 v1.9.2/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= -github.com/aws/aws-sdk-go-v2 v1.16.3/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU= -github.com/aws/aws-sdk-go-v2 v1.21.2 h1:+LXZ0sgo8quN9UOKXXzAWRT3FWd4NxeXWOZom9pE7GA= -github.com/aws/aws-sdk-go-v2 v1.21.2/go.mod h1:ErQhvNuEMhJjweavOYhxVkn2RUx7kQXVATHrjKtxIpM= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.1 h1:SdK4Ppk5IzLs64ZMvr6MrSficMtjY2oS0WOORXTlxwU= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.1/go.mod h1:n8Bs1ElDD2wJ9kCRTczA83gYbBmjSwZp3umc6zF4EeM= +github.com/aws/aws-sdk-go-v2 v1.31.0 h1:3V05LbxTSItI5kUqNwhJrrrY1BAXxXt0sN0l72QmG5U= +github.com/aws/aws-sdk-go-v2 v1.31.0/go.mod h1:ztolYtaEUtdpf9Wftr31CJfLVjOnD/CVRkKOOYgF8hA= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.5 h1:xDAuZTn4IMm8o1LnBZvmrL8JA1io4o3YWNXgohbf20g= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.5/go.mod h1:wYSv6iDS621sEFLfKvpPE2ugjTuGlAG7iROg0hLOkfc= github.com/aws/aws-sdk-go-v2/config v1.8.3/go.mod h1:4AEiLtAb8kLs7vgw2ZV3p2VZ1+hBavOc84hqxVNpCyw= -github.com/aws/aws-sdk-go-v2/config v1.15.5/go.mod h1:ZijHHh0xd/A+ZY53az0qzC5tT46kt4JVCePf2NX9Lk4= -github.com/aws/aws-sdk-go-v2/config v1.18.45 h1:Aka9bI7n8ysuwPeFdm77nfbyHCAKQ3z9ghB3S/38zes= -github.com/aws/aws-sdk-go-v2/config v1.18.45/go.mod h1:ZwDUgFnQgsazQTnWfeLWk5GjeqTQTL8lMkoE1UXzxdE= +github.com/aws/aws-sdk-go-v2/config v1.27.40 h1:sie4mPBGFOO+Z27+yHzvyN31G20h/bf2xb5mCbpLv2Q= +github.com/aws/aws-sdk-go-v2/config v1.27.40/go.mod h1:4KW7Aa5tNo+0VHnuLnnE1vPHtwMurlNZNS65IdcewHA= github.com/aws/aws-sdk-go-v2/credentials v1.4.3/go.mod h1:FNNC6nQZQUuyhq5aE5c7ata8o9e4ECGmS4lAXC7o1mQ= -github.com/aws/aws-sdk-go-v2/credentials v1.12.0/go.mod h1:9YWk7VW+eyKsoIL6/CljkTrNVWBSK9pkqOPUuijid4A= -github.com/aws/aws-sdk-go-v2/credentials v1.13.43 h1:LU8vo40zBlo3R7bAvBVy/ku4nxGEyZe9N8MqAeFTzF8= -github.com/aws/aws-sdk-go-v2/credentials v1.13.43/go.mod h1:zWJBz1Yf1ZtX5NGax9ZdNjhhI4rgjfgsyk6vTY1yfVg= +github.com/aws/aws-sdk-go-v2/credentials v1.17.38 h1:iM90eRhCeZtlkzCNCG1JysOzJXGYf5rx80aD1lUgNDU= +github.com/aws/aws-sdk-go-v2/credentials v1.17.38/go.mod h1:TCVYPZeQuLaYNEkf/TVn6k5k/zdVZZ7xH9po548VNNg= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.6.0/go.mod h1:gqlclDEZp4aqJOancXK6TN24aKhT0W0Ae9MHk3wzTMM= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.4/go.mod h1:u/s5/Z+ohUQOPXl00m2yJVyioWDECsbpXTQlaqSlufc= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 h1:PIktER+hwIG286DqXyvVENjgLTAwGgoeriLDD5C+YlQ= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13/go.mod h1:f/Ib/qYjhV2/qdsf79H3QP/eRE4AkVyEf6sk7XfZ1tg= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.10 h1:JL7cY85hyjlgfA29MMyAlItX+JYIH9XsxgMBS7jtlqA= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.10/go.mod h1:p+ul5bLZSDRRXCZ/vePvfmZBH9akozXBJA5oMshWa5U= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.10/go.mod h1:F+EZtuIwjlv35kRJPyBGcsA4f7bnSoz15zOQ2lJq1Z4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 h1:nFBQlGtkbPzp/NjZLuFxRqmT91rLJkgvsEQs68h962Y= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43/go.mod h1:auo+PiyLl0n1l8A0e8RIeR8tOzYPfZZH/JNlrJ8igTQ= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.4/go.mod h1:8glyUqVIM4AmeenIsPo0oVh3+NUwnsQml2OFupfQW+0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 h1:JRVhO25+r3ar2mKGP7E0LDl8K9/G36gjlqca5iQbaqc= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37/go.mod h1:Qe+2KtKml+FEsQF/DHmDV+xjtche/hwoF75EG4UlHW8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.14 h1:C/d03NAmh8C4BZXhuRNboF/DqhBkBCeDiJDcaqIT5pA= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.14/go.mod h1:7I0Ju7p9mCIdlrfS+JCgqcYD0VXz/N4yozsox+0o078= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.27 h1:1oLpQSTuqbizOUEYdxAwH+Eveg+FOCOkg84Yijba6Kc= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.27/go.mod h1:afo0vF9P3pjy1ny+cb45lzBjtKeEb5t5MPRxeTXpujw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.18 h1:kYQ3H1u0ANr9KEKlGs/jTLrBFPo8P8NaH/w7A01NeeM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.18/go.mod h1:r506HmK5JDUh9+Mw4CfGJGSSoqIiLCndAuqXuhbv67Y= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.18 h1:Z7IdFUONvTcvS7YuhtVxN99v2cCoHRXOS4mTr0B/pUc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.18/go.mod h1:DkKMmksZVVyat+Y+r1dEOgJEfUeA7UngIHWeKsi0yNc= github.com/aws/aws-sdk-go-v2/internal/ini v1.2.4/go.mod h1:ZcBrrI3zBKlhGFNYWvju0I3TR93I7YIgAfy82Fh4lcQ= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.11/go.mod h1:0MR+sS1b/yxsfAPvAESrw8NfwUoxMinDyw6EYR9BS2U= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 h1:hze8YsjSh8Wl1rYa1CJpRmXP21BvOBuc76YhW0HsuQ4= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45/go.mod h1:lD5M20o09/LCuQ2mE62Mb/iSdSlCNuj6H5ci7tW7OsE= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.1 h1:C21IDZCm9Yu5xqjb3fKmxDoYvJXtw1DNlOmLZEIlY1M= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.1/go.mod h1:l/BbcfqDCT3hePawhy4ZRtewjtdkl6GWtd9/U+1penQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.18 h1:OWYvKL53l1rbsUmW7bQyJVsYU/Ii3bbAAQIIFNbM0Tk= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.18/go.mod h1:CUx0G1v3wG6l01tUB+j7Y8kclA8NSqK4ef0YG79a4cg= github.com/aws/aws-sdk-go-v2/service/appconfig v1.4.2/go.mod h1:FZ3HkCe+b10uFZZkFdvf98LHW21k49W8o8J366lqVKY= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.1 h1:T4pFel53bkHjL2mMo+4DKE6r6AuoZnM0fg7k1/ratr4= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.1/go.mod h1:GeUru+8VzrTXV/83XyMJ80KpH8xO89VPoUileyNQ+tc= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.5 h1:9LSZqt4v1JiehyZTrQnRFf2mY/awmyYNNY/b7zqtduU= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.5/go.mod h1:S8TVP66AAkMMdYYCNZGvrdEq9YRm+qLXjio4FqRnrEE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.5 h1:QFASJGfT8wMXtuP3D5CRmMjARHv9ZmzFUMJznHDOY3w= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.5/go.mod h1:QdZ3OmoIjSX+8D1OPAzPxDfjXASbBMDsz9qvtyIhtik= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.20 h1:rTWjG6AvWekO2B1LHeM3ktU7MqyX9rzWQ7hgzneZW7E= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.20/go.mod h1:RGW2DDpVc8hu6Y6yG8G5CHVmVOAn1oV8rNKOHRJyswg= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.2/go.mod h1:72HRZDLMtmVQiLG2tLfQcaWLCssELvGl+Zf2WVxMmR8= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.4/go.mod h1:uKkN7qmSIsNJVyMtxNQoCEYMvFEXbOg9fwCJPdfp2u8= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37 h1:WWZA/I2K4ptBS1kg0kV1JbBtG/umed0vwHRrmcr9z7k= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37/go.mod h1:vBmDnwWXWxNPFRMmG2m/3MKOe+xEcMDo1tanpaWCcck= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.4 h1:RE/DlZLYrz1OOmq8F28IXHLksuuvlpzUbvJ+SESCZBI= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.4/go.mod h1:oudbsSdDtazNj47z1ut1n37re9hDsKpk2ZI3v7KSxq0= -github.com/aws/aws-sdk-go-v2/service/s3 v1.26.9 h1:LCQKnopq2t4oQS3VKivlYTzAHCTJZZoQICM9fny7KHY= -github.com/aws/aws-sdk-go-v2/service/s3 v1.26.9/go.mod h1:iMYipLPXlWpBJ0KFX7QJHZ84rBydHBY8as2aQICTPWk= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.20 h1:Xbwbmk44URTiHNx6PNo0ujDE6ERlsCKJD3u1zfnzAPg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.20/go.mod h1:oAfOFzUB14ltPZj1rWwRc3d/6OgD76R8KlvU3EqM9Fg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.18 h1:eb+tFOIl9ZsUe2259/BKPeniKuz4/02zZFH/i4Nf8Rg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.18/go.mod h1:GVCC2IJNJTmdlyEsSmofEy7EfJncP7DNnXDzRjJ5Keg= +github.com/aws/aws-sdk-go-v2/service/s3 v1.64.1 h1:jjHf+M6vCp/WzbyFEroY4/Nx8dJac520A0EPwlYk0Do= +github.com/aws/aws-sdk-go-v2/service/s3 v1.64.1/go.mod h1:NLTqRLe3pUNu3nTEHI6XlHLKYmc8fbHUdMxAB6+s41Q= github.com/aws/aws-sdk-go-v2/service/sso v1.4.2/go.mod h1:NBvT9R1MEF+Ud6ApJKM0G+IkPchKS7p7c2YPKwHmBOk= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.4/go.mod h1:cPDwJwsP4Kff9mldCXAmddjJL6JGQqtA3Mzer2zyr88= -github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 h1:JuPGc7IkOP4AaqcZSIcyqLpFSqBWK32rM9+a1g6u73k= -github.com/aws/aws-sdk-go-v2/service/sso v1.15.2/go.mod h1:gsL4keucRCgW+xA85ALBpRFfdSLH4kHOVSnLMSuBECo= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3 h1:HFiiRkf1SdaAmV3/BHOFZ9DjFynPHj8G/UIO1lQS+fk= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3/go.mod h1:a7bHA82fyUXOm+ZSWKU6PIoBxrjSprdLoM8xPYvzYVg= +github.com/aws/aws-sdk-go-v2/service/sso v1.23.4 h1:ck/Y8XWNR1gHa4BFkwE3oSu7XDJGwl+8TI7E/RB2EcQ= +github.com/aws/aws-sdk-go-v2/service/sso v1.23.4/go.mod h1:XRlMvmad0ZNL+75C5FYdMvbbLkd6qiqz6foR1nA1PXY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.27.4 h1:4f2/JKYZHAZbQ7koBpZ012bKi32NHPY0m7TDuJgsbug= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.27.4/go.mod h1:FnvDM4sfa+isJ3kDXIzAB9GAwVSzFzSy97uZ3IsHo4E= github.com/aws/aws-sdk-go-v2/service/sts v1.7.2/go.mod h1:8EzeIqfWt2wWT4rJVu3f21TfrhJ8AEMzVybRNSb/b4g= -github.com/aws/aws-sdk-go-v2/service/sts v1.16.4/go.mod h1:lfSYenAXtavyX2A1LsViglqlG9eEFYxNryTZS5rn3QE= -github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 h1:0BkLfgeDjfZnZ+MhB3ONb01u9pwFYTCZVhlsSSBvlbU= -github.com/aws/aws-sdk-go-v2/service/sts v1.23.2/go.mod h1:Eows6e1uQEsc4ZaHANmsPRzAKcVDrcmjjWiih2+HUUQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.31.4 h1:uK6dUUdJtqutK1XO/tmNaQMJiPLCJY/eAeOOmqQ6ygY= +github.com/aws/aws-sdk-go-v2/service/sts v1.31.4/go.mod h1:yMWe0F+XG0DkRZK5ODZhG7BEFYhLXi2dqGsv6tX0cgI= github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= -github.com/aws/smithy-go v1.11.2/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM= -github.com/aws/smithy-go v1.15.0 h1:PS/durmlzvAFpQHDs4wi4sNNP9ExsqZh6IlfdHXgKK8= -github.com/aws/smithy-go v1.15.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM= +github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -354,8 +343,6 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github/v62 v62.0.0 h1:/6mGCaRywZz9MuHyw9gD1CwsbmBX8GWsbFkwMmHdhl4= @@ -441,9 +428,7 @@ github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0Gqw github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= From 36f954dfe0e66bb26832657f5ac806de9ec70755 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Thu, 3 Oct 2024 21:13:05 -0500 Subject: [PATCH 324/489] Fix submodule-pin-check CI --- .github/workflows/submodule-pin-check.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index 90419b530e..49df877a34 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -18,7 +18,8 @@ jobs: with: fetch-depth: 0 submodules: true - ref: "${{ github.event.pull_request.merge_commit_sha }}" + persist-credentials: false + ref: "refs/pull/${{ github.event.pull_request.number }}/merge" - name: Check all submodules are ancestors of origin/HEAD or configured branch run: | From a8519aedb68606c48a818b6380a99fda4451ca37 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Thu, 3 Oct 2024 21:15:49 -0500 Subject: [PATCH 325/489] Use head sha instead --- .github/workflows/submodule-pin-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index 49df877a34..60dd8ad827 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -19,7 +19,7 @@ jobs: fetch-depth: 0 submodules: true persist-credentials: false - ref: "refs/pull/${{ github.event.pull_request.number }}/merge" + ref: "${{ github.event.pull_request.head.sha }}" - name: Check all submodules are ancestors of origin/HEAD or configured branch run: | From d5f5f11c6e880dca803fb9ef141f0971b41ef743 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 4 Oct 2024 13:14:28 +0200 Subject: [PATCH 326/489] refactor wasm lru cache test --- system_tests/program_test.go | 99 ++++++++++++++++-------------------- 1 file changed, 43 insertions(+), 56 deletions(-) diff --git a/system_tests/program_test.go b/system_tests/program_test.go index 6ca04976e5..aab207e0f6 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -2059,83 +2059,68 @@ func TestWasmLruCache(t *testing.T) { l2client := builder.L2.Client defer cleanup() + ensure := func(tx *types.Transaction, err error) *types.Receipt { + t.Helper() + Require(t, err) + receipt, err := EnsureTxSucceeded(ctx, l2client, tx) + Require(t, err) + return receipt + } + auth.GasLimit = 32000000 auth.Value = oneEth - fallibleProgramAddress, fallibleLruEntrySizeEstimateBytes := deployWasmAndGetEntrySizeEstimateBytes(t, builder, auth, "fallible") - keccakProgramAddress, keccakLruEntrySizeEstimateBytes := deployWasmAndGetEntrySizeEstimateBytes(t, builder, auth, "keccak") - mathProgramAddress, mathLruEntrySizeEstimateBytes := deployWasmAndGetEntrySizeEstimateBytes(t, builder, auth, "math") + fallibleProgramAddress, fallibleEntrySize := deployWasmAndGetEntrySizeEstimateBytes(t, builder, auth, "fallible") + keccakProgramAddress, keccakEntrySize := deployWasmAndGetEntrySizeEstimateBytes(t, builder, auth, "keccak") + mathProgramAddress, mathEntrySize := deployWasmAndGetEntrySizeEstimateBytes(t, builder, auth, "math") t.Log( "lruEntrySizeEstimateBytes, ", - "fallible:", fallibleLruEntrySizeEstimateBytes, - "keccak:", keccakLruEntrySizeEstimateBytes, - "math:", mathLruEntrySizeEstimateBytes, + "fallible:", fallibleEntrySize, + "keccak:", keccakEntrySize, + "math:", mathEntrySize, ) programs.ClearWasmLruCache() - lruMetrics := programs.GetWasmCacheMetrics().Lru - if lruMetrics.Count != 0 { - t.Fatalf("lruMetrics.Count, expected: %v, actual: %v", 0, lruMetrics.Count) - } - if lruMetrics.SizeBytes != 0 { - t.Fatalf("lruMetrics.SizeBytes, expected: %v, actual: %v", 0, lruMetrics.SizeBytes) - } + checkLruCacheMetrics(t, programs.WasmLruCacheMetrics{ + Count: 0, + SizeBytes: 0, + }) - programs.SetWasmLruCacheCapacity(fallibleLruEntrySizeEstimateBytes - 1) + programs.SetWasmLruCacheCapacity(fallibleEntrySize - 1) // fallible wasm program will not be cached since its size is greater than lru cache capacity tx := l2info.PrepareTxTo("Owner", &fallibleProgramAddress, l2info.TransferGas, nil, []byte{0x01}) - Require(t, l2client.SendTransaction(ctx, tx)) - _, err := EnsureTxSucceeded(ctx, l2client, tx) - Require(t, err) - lruMetrics = programs.GetWasmCacheMetrics().Lru - if lruMetrics.Count != 0 { - t.Fatalf("lruMetrics.Count, expected: %v, actual: %v", 0, lruMetrics.Count) - } - if lruMetrics.SizeBytes != 0 { - t.Fatalf("lruMetrics.SizeBytes, expected: %v, actual: %v", 0, lruMetrics.SizeBytes) - } + ensure(tx, l2client.SendTransaction(ctx, tx)) + checkLruCacheMetrics(t, programs.WasmLruCacheMetrics{ + Count: 0, + SizeBytes: 0, + }) programs.SetWasmLruCacheCapacity( - fallibleLruEntrySizeEstimateBytes + keccakLruEntrySizeEstimateBytes + mathLruEntrySizeEstimateBytes - 1, + fallibleEntrySize + keccakEntrySize + mathEntrySize - 1, ) // fallible wasm program will be cached tx = l2info.PrepareTxTo("Owner", &fallibleProgramAddress, l2info.TransferGas, nil, []byte{0x01}) - Require(t, l2client.SendTransaction(ctx, tx)) - _, err = EnsureTxSucceeded(ctx, l2client, tx) - Require(t, err) - lruMetrics = programs.GetWasmCacheMetrics().Lru - if lruMetrics.Count != 1 { - t.Fatalf("lruMetrics.Count, expected: %v, actual: %v", 1, lruMetrics.Count) - } - if lruMetrics.SizeBytes != fallibleLruEntrySizeEstimateBytes { - t.Fatalf("lruMetrics.SizeBytes, expected: %v, actual: %v", fallibleLruEntrySizeEstimateBytes, lruMetrics.SizeBytes) - } + ensure(tx, l2client.SendTransaction(ctx, tx)) + checkLruCacheMetrics(t, programs.WasmLruCacheMetrics{ + Count: 1, + SizeBytes: fallibleEntrySize, + }) // keccak wasm program will be cached tx = l2info.PrepareTxTo("Owner", &keccakProgramAddress, l2info.TransferGas, nil, []byte{0x01}) - Require(t, l2client.SendTransaction(ctx, tx)) - _, err = EnsureTxSucceeded(ctx, l2client, tx) - Require(t, err) - lruMetrics = programs.GetWasmCacheMetrics().Lru - if lruMetrics.Count != 2 { - t.Fatalf("lruMetrics.Count, expected: %v, actual: %v", 2, lruMetrics.Count) - } - if lruMetrics.SizeBytes != fallibleLruEntrySizeEstimateBytes+keccakLruEntrySizeEstimateBytes { - t.Fatalf("lruMetrics.SizeBytes, expected: %v, actual: %v", fallibleLruEntrySizeEstimateBytes+keccakLruEntrySizeEstimateBytes, lruMetrics.SizeBytes) - } + ensure(tx, l2client.SendTransaction(ctx, tx)) + checkLruCacheMetrics(t, programs.WasmLruCacheMetrics{ + Count: 2, + SizeBytes: fallibleEntrySize + keccakEntrySize, + }) // math wasm program will be cached, but fallible will be evicted since (fallible + keccak + math) > lruCacheCapacity tx = l2info.PrepareTxTo("Owner", &mathProgramAddress, l2info.TransferGas, nil, []byte{0x01}) - Require(t, l2client.SendTransaction(ctx, tx)) - _, err = EnsureTxSucceeded(ctx, l2client, tx) - Require(t, err) - lruMetrics = programs.GetWasmCacheMetrics().Lru - if lruMetrics.Count != 2 { - t.Fatalf("lruMetrics.Count, expected: %v, actual: %v", 2, lruMetrics.Count) - } - if lruMetrics.SizeBytes != keccakLruEntrySizeEstimateBytes+mathLruEntrySizeEstimateBytes { - t.Fatalf("lruMetrics.SizeBytes, expected: %v, actual: %v", keccakLruEntrySizeEstimateBytes+mathLruEntrySizeEstimateBytes, lruMetrics.SizeBytes) - } + ensure(tx, l2client.SendTransaction(ctx, tx)) + checkLruCacheMetrics(t, programs.WasmLruCacheMetrics{ + Count: 2, + SizeBytes: keccakEntrySize + mathEntrySize, + }) } func checkLongTermCacheMetrics(t *testing.T, expected programs.WasmLongTermCacheMetrics) { @@ -2403,7 +2388,9 @@ func TestRepopulateWasmLongTermCacheFromLru(t *testing.T) { SizeBytes: fallibleEntrySize, }) - // mathProgram should end up in lru cache and as result fallibleProgram should be evicted as least recently used item (tx that restores the program back to long term cache shouldn't promote the lru item); fallibleProgram should remain in long term cache + // mathProgram should end up in lru cache and + // as result fallibleProgram should be evicted as least recently used item (tx that restores the program back to long term cache shouldn't promote the lru item); + // fallibleProgram should remain in long term cache tx = l2info.PrepareTxTo("Owner", &mathProgramAddress, l2info.TransferGas, nil, []byte{0x01}) ensure(tx, l2client.SendTransaction(ctx, tx)) checkLruCacheMetrics(t, programs.WasmLruCacheMetrics{ From 798d7177e8c9c160c1a36c954daa982a20e95306 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Fri, 4 Oct 2024 17:15:18 +0530 Subject: [PATCH 327/489] Add a test to keep server_api json.go in sync with parse_inputs.rs --- arbitrator/prover/src/lib.rs | 36 ++++++ arbitrator/prover/src/parse_input.rs | 27 +++- .../validationinputjson_rustfiledata_test.go | 120 ++++++++++++++++++ validator/server_arb/machine.go | 14 ++ 4 files changed, 192 insertions(+), 5 deletions(-) create mode 100644 system_tests/validationinputjson_rustfiledata_test.go diff --git a/arbitrator/prover/src/lib.rs b/arbitrator/prover/src/lib.rs index 08473c2598..993ed1b365 100644 --- a/arbitrator/prover/src/lib.rs +++ b/arbitrator/prover/src/lib.rs @@ -33,9 +33,12 @@ use machine::{ PreimageResolver, }; use once_cell::sync::OnceCell; +use parse_input::FileData; use static_assertions::const_assert_eq; use std::{ ffi::CStr, + fs::File, + io::BufReader, num::NonZeroUsize, os::raw::{c_char, c_int}, path::Path, @@ -84,6 +87,39 @@ pub unsafe extern "C" fn arbitrator_load_machine( } } +#[no_mangle] +pub unsafe extern "C" fn arbitrator_deserialize_and_serialize_file_data( + read_path: *const c_char, + write_path: *const c_char, +) -> c_int { + let read_path = cstr_to_string(read_path); + let write_path = cstr_to_string(write_path); + + let file = File::open(read_path); + let reader = match file { + Ok(file) => BufReader::new(file), + Err(err) => { + eprintln!("Failed to open read_path of FileData: {}", err); + return 1; + } + }; + let data = match FileData::from_reader(reader) { + Ok(data) => data, + Err(err) => { + eprintln!("Failed to deserialize FileData: {}", err); + return 2; + } + }; + + match data.write_to_file(&write_path) { + Ok(()) => 0, + Err(err) => { + eprintln!("Failed to serialize FileData: {}", err); + 3 + } + } +} + unsafe fn arbitrator_load_machine_impl( binary_path: *const c_char, library_paths: *const *const c_char, diff --git a/arbitrator/prover/src/parse_input.rs b/arbitrator/prover/src/parse_input.rs index fa7adb4c41..edfaddf26f 100644 --- a/arbitrator/prover/src/parse_input.rs +++ b/arbitrator/prover/src/parse_input.rs @@ -1,12 +1,14 @@ use arbutil::Bytes32; use serde::Deserialize; +use serde::Serialize; use serde_json; use serde_with::base64::Base64; use serde_with::As; use serde_with::DisplayFromStr; use std::{ collections::HashMap, - io::{self, BufRead}, + fs::File, + io::{self, BufRead, BufWriter}, }; /// prefixed_hex deserializes hex strings which are prefixed with `0x` @@ -16,7 +18,7 @@ use std::{ /// It is an error to use this deserializer on a string that does not /// begin with `0x`. mod prefixed_hex { - use serde::{self, Deserialize, Deserializer}; + use serde::{self, Deserialize, Deserializer, Serializer}; pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> where @@ -29,6 +31,14 @@ mod prefixed_hex { Err(serde::de::Error::custom("missing 0x prefix")) } } + + pub fn serialize(bytes: &Vec, serializer: S) -> Result + where + S: Serializer, + { + let hex_string = format!("0x{}", hex::encode(bytes)); + serializer.serialize_str(&hex_string) + } } #[derive(Debug)] @@ -61,7 +71,7 @@ impl From> for UserWasm { } } -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, Deserialize, Serialize)] #[serde(rename_all = "PascalCase")] pub struct BatchInfo { pub number: u64, @@ -69,7 +79,7 @@ pub struct BatchInfo { pub data_b64: Vec, } -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, Serialize)] #[serde(rename_all = "PascalCase")] pub struct StartState { #[serde(with = "prefixed_hex")] @@ -88,7 +98,7 @@ pub struct StartState { /// /// Note: It is important to change this file whenever the go JSON /// serialization changes. -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, Serialize)] #[serde(rename_all = "PascalCase")] pub struct FileData { pub id: u64, @@ -109,4 +119,11 @@ impl FileData { let data = serde_json::from_reader(&mut reader)?; Ok(data) } + + pub fn write_to_file(&self, file_path: &str) -> io::Result<()> { + let file = File::create(file_path)?; + let writer = BufWriter::new(file); + serde_json::to_writer_pretty(writer, &self)?; + Ok(()) + } } diff --git a/system_tests/validationinputjson_rustfiledata_test.go b/system_tests/validationinputjson_rustfiledata_test.go new file mode 100644 index 0000000000..a27a68b12c --- /dev/null +++ b/system_tests/validationinputjson_rustfiledata_test.go @@ -0,0 +1,120 @@ +package arbtest + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" + + "github.com/offchainlabs/nitro/arbutil" + "github.com/offchainlabs/nitro/validator" + "github.com/offchainlabs/nitro/validator/server_api" + "github.com/offchainlabs/nitro/validator/server_arb" +) + +func getWriteDataFromRustSide(t *testing.T, inputJSON *server_api.InputJSON) []byte { + t.Helper() + dir := t.TempDir() + + readData, err := inputJSON.Marshal() + Require(t, err) + readPath := filepath.Join(dir, fmt.Sprintf("block_inputs_%d_read.json", inputJSON.Id)) + Require(t, os.WriteFile(readPath, readData, 0600)) + + writePath := filepath.Join(dir, fmt.Sprintf("block_inputs_%d_write.json", inputJSON.Id)) + Require(t, server_arb.DeserializeAndSerializeFileData(readPath, writePath)) + writeData, err := os.ReadFile(writePath) + Require(t, err) + + return writeData +} + +func TestGoInputJSONRustFileDataRoundtripWithoutUserWasms(t *testing.T) { + preimages := make(map[arbutil.PreimageType]map[common.Hash][]byte) + preimages[arbutil.Keccak256PreimageType] = make(map[common.Hash][]byte) + preimages[arbutil.Keccak256PreimageType][common.MaxHash] = []byte{1} + + // Don't include DebugChain as it isn't used on rust side + sampleValidationInput := &validator.ValidationInput{ + Id: 1, + HasDelayedMsg: true, + DelayedMsgNr: 2, + Preimages: preimages, + BatchInfo: []validator.BatchInfo{{Number: 3}}, + DelayedMsg: []byte{4}, + StartState: validator.GoGlobalState{ + BlockHash: common.MaxHash, + SendRoot: common.MaxHash, + Batch: 5, + PosInBatch: 6, + }, + } + sampleValidationInputJSON := server_api.ValidationInputToJson(sampleValidationInput) + writeData := getWriteDataFromRustSide(t, sampleValidationInputJSON) + + var resWithoutUserWasms server_api.InputJSON + Require(t, json.Unmarshal(writeData, &resWithoutUserWasms)) + if !reflect.DeepEqual(*sampleValidationInputJSON, resWithoutUserWasms) { + t.Fatal("ValidationInputJSON without UserWasms, mismatch on rust and go side") + } + +} + +type inputJSONWithUserWasmsOnly struct { + UserWasms map[ethdb.WasmTarget]map[common.Hash][]byte +} + +// UnmarshalJSON is a custom function defined to encapsulate how UserWasms are handled on the rust side. +// When ValidationInputToJson is called on ValidationInput, it compresses the wasm data byte array and +// then encodes this to a base64 string, this when deserialized on the rust side through FileData- the +// compressed data is first uncompressed and also the module hash (Bytes32) is read without the 0x prefix, +// so we define a custom UnmarshalJSON to extract UserWasms map from the data written by rust side. +func (u *inputJSONWithUserWasmsOnly) UnmarshalJSON(data []byte) error { + type rawUserWasms struct { + UserWasms map[ethdb.WasmTarget]map[string]string + } + var rawUWs rawUserWasms + if err := json.Unmarshal(data, &rawUWs); err != nil { + return err + } + tmp := make(map[ethdb.WasmTarget]map[common.Hash][]byte) + for wasmTarget, innerMap := range rawUWs.UserWasms { + tmp[wasmTarget] = make(map[common.Hash][]byte) + for hashKey, value := range innerMap { + valBytes, err := base64.StdEncoding.DecodeString(value) + if err != nil { + return err + } + tmp[wasmTarget][common.HexToHash("0x"+hashKey)] = valBytes + } + } + u.UserWasms = tmp + return nil +} + +func TestGoInputJSONRustFileDataRoundtripWithUserWasms(t *testing.T) { + userWasms := make(map[ethdb.WasmTarget]map[common.Hash][]byte) + userWasms["arch1"] = make(map[common.Hash][]byte) + userWasms["arch1"][common.MaxHash] = []byte{2} + + // Don't include DebugChain as it isn't used on rust side + sampleValidationInput := &validator.ValidationInput{ + Id: 1, + UserWasms: userWasms, + BatchInfo: []validator.BatchInfo{{Number: 1}}, // This needs to be set for FileData to successfully deserialize, else it errors for invalid type null + } + sampleValidationInputJSON := server_api.ValidationInputToJson(sampleValidationInput) + writeData := getWriteDataFromRustSide(t, sampleValidationInputJSON) + + var resUserWasmsOnly inputJSONWithUserWasmsOnly + Require(t, json.Unmarshal(writeData, &resUserWasmsOnly)) + if !reflect.DeepEqual(userWasms, resUserWasmsOnly.UserWasms) { + t.Fatal("ValidationInputJSON with UserWasms only, mismatch on rust and go side") + } +} diff --git a/validator/server_arb/machine.go b/validator/server_arb/machine.go index 1e73e6b212..d11f015916 100644 --- a/validator/server_arb/machine.go +++ b/validator/server_arb/machine.go @@ -312,6 +312,20 @@ func (m *ArbitratorMachine) DeserializeAndReplaceState(path string) error { } } +func DeserializeAndSerializeFileData(readPath, writePath string) error { + cReadPath := C.CString(readPath) + cWritePath := C.CString(writePath) + status := C.arbitrator_deserialize_and_serialize_file_data(cReadPath, cWritePath) + C.free(unsafe.Pointer(cReadPath)) + C.free(unsafe.Pointer(cWritePath)) + + if status != 0 { + return fmt.Errorf("failed to call arbitrator_deserialize_and_serialize_file_data. Error code: %d", status) + } else { + return nil + } +} + func (m *ArbitratorMachine) AddSequencerInboxMessage(index uint64, data []byte) error { defer runtime.KeepAlive(m) m.mutex.Lock() From fe2233ffb46f98a95398335c793c2725490da595 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 4 Oct 2024 18:55:49 +0200 Subject: [PATCH 328/489] avoid unncessary cloning of cache item data --- arbitrator/stylus/src/cache.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index 208f45e26c..6192a30eff 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -182,15 +182,14 @@ impl InitCache { // See if the item is in the LRU cache, promoting if so if let Some(item) = cache.lru.peek(&key).cloned() { cache.lru_counters.hits += 1; - let data = item.data(); if long_term_tag == Self::ARBOS_TAG { cache.long_term_size_bytes += item.entry_size_estimate_bytes; - cache.long_term.insert(key, item); + cache.long_term.insert(key, item.clone()); } else { // only calls get to move the key to the head of the LRU list cache.lru.get(&key); } - return Some(data); + return Some((item.module, Store::new(item.engine))); } cache.lru_counters.misses += 1; From 10436295450ca68103cd0c8a6bc51a2c0f3ceccc Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Mon, 7 Oct 2024 14:34:20 -0300 Subject: [PATCH 329/489] Bump go-ethereum --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 3686062434..72e4db5d66 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 3686062434f9030e5fccf9bdd9329b8d0f2a86a5 +Subproject commit 72e4db5d667da17ccb8b10e38e2e99e5b6b4c3d5 From 0e32d6b704a52f6f82e3cbd7e44a1b632fa81429 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Mon, 7 Oct 2024 15:10:19 -0300 Subject: [PATCH 330/489] Add nil-checks to arbitrum tracing hooks --- arbos/programs/api.go | 4 +++- arbos/util/tracing.go | 4 ++-- arbos/util/transfer.go | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/arbos/programs/api.go b/arbos/programs/api.go index 3e59031b2d..5ec8c97207 100644 --- a/arbos/programs/api.go +++ b/arbos/programs/api.go @@ -254,7 +254,9 @@ func newApiClosures( return memoryModel.GasCost(pages, open, ever) } captureHostio := func(name string, args, outs []byte, startInk, endInk uint64) { - tracingInfo.Tracer.CaptureStylusHostio(name, args, outs, startInk, endInk) + if tracingInfo.Tracer != nil && tracingInfo.Tracer.CaptureStylusHostio != nil { + tracingInfo.Tracer.CaptureStylusHostio(name, args, outs, startInk, endInk) + } tracingInfo.CaptureEVMTraceForHostio(name, args, outs, startInk, endInk) } diff --git a/arbos/util/tracing.go b/arbos/util/tracing.go index 64a8bcd6a2..8effee256c 100644 --- a/arbos/util/tracing.go +++ b/arbos/util/tracing.go @@ -63,7 +63,7 @@ func (info *TracingInfo) RecordStorageGet(key common.Hash) { if tracer.OnOpcode != nil { tracer.OnOpcode(0, byte(vm.SLOAD), 0, 0, scope, []byte{}, info.Depth, nil) } - } else { + } else if tracer.CaptureArbitrumStorageGet != nil { tracer.CaptureArbitrumStorageGet(key, info.Depth, info.Scenario == TracingBeforeEVM) } } @@ -79,7 +79,7 @@ func (info *TracingInfo) RecordStorageSet(key, value common.Hash) { if tracer.OnOpcode != nil { tracer.OnOpcode(0, byte(vm.SSTORE), 0, 0, scope, []byte{}, info.Depth, nil) } - } else { + } else if tracer.CaptureArbitrumStorageSet != nil { tracer.CaptureArbitrumStorageSet(key, value, info.Depth, info.Scenario == TracingBeforeEVM) } } diff --git a/arbos/util/transfer.go b/arbos/util/transfer.go index 32ef4d1bd0..e186dee0fa 100644 --- a/arbos/util/transfer.go +++ b/arbos/util/transfer.go @@ -36,7 +36,7 @@ func TransferBalance( return errors.New("tracing scenario mismatch") } - if scenario != TracingDuringEVM { + if scenario != TracingDuringEVM && tracer.CaptureArbitrumTransfer != nil { tracer.CaptureArbitrumTransfer(from, to, amount, scenario == TracingBeforeEVM, purpose) } else { fromCopy := from From e7139f5786bbd01a4415ab8fda2e15387558339f Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Mon, 7 Oct 2024 15:10:43 -0300 Subject: [PATCH 331/489] Fix OnExit tracing call --- arbos/util/tracing.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arbos/util/tracing.go b/arbos/util/tracing.go index 8effee256c..fb39460d44 100644 --- a/arbos/util/tracing.go +++ b/arbos/util/tracing.go @@ -122,7 +122,7 @@ func (info *TracingInfo) MockCall(input []byte, gas uint64, from, to common.Addr tracer.OnOpcode(0, byte(vm.RETURN), 0, 0, retScope, []byte{}, depth+1, nil) } if tracer.OnExit != nil { - tracer.OnExit(depth+1, nil, 0, nil, false) + tracer.OnExit(depth, nil, 0, nil, false) } popScope := &vm.ScopeContext{ From b06c136898556870cf59921ad7fbc182dcfbb8ce Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Mon, 7 Oct 2024 15:11:02 -0300 Subject: [PATCH 332/489] Fix stylus tracer --- execution/gethexec/stylus_tracer.go | 33 ++++++++--------------------- 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/execution/gethexec/stylus_tracer.go b/execution/gethexec/stylus_tracer.go index f054c5cb0b..dc7e5ae0a4 100644 --- a/execution/gethexec/stylus_tracer.go +++ b/execution/gethexec/stylus_tracer.go @@ -77,16 +77,9 @@ func newStylusTracer(ctx *tracers.Context, _ json.RawMessage) (*tracers.Tracer, return &tracers.Tracer{ Hooks: &tracing.Hooks{ - OnExit: t.OnExit, - OnEnter: t.OnEnter, - OnOpcode: t.OnOpcode, - OnFault: t.OnFault, - OnTxStart: t.OnTxStart, - OnTxEnd: t.OnTxEnd, - CaptureArbitrumTransfer: t.CaptureArbitrumTransfer, - CaptureArbitrumStorageGet: t.CaptureArbitrumStorageGet, - CaptureArbitrumStorageSet: t.CaptureArbitrumStorageSet, - CaptureStylusHostio: t.CaptureStylusHostio, + OnEnter: t.OnEnter, + OnExit: t.OnExit, + CaptureStylusHostio: t.CaptureStylusHostio, }, GetResult: t.GetResult, Stop: t.Stop, @@ -127,6 +120,9 @@ func (t *stylusTracer) OnEnter(depth int, typ byte, from common.Address, to comm if t.interrupt.Load() { return } + if depth == 0 { + return + } // This function adds the prefix evm_ because it assumes the opcode came from the EVM. // If the opcode comes from WASM, the CaptureStylusHostio function will remove the evm prefix. @@ -161,6 +157,9 @@ func (t *stylusTracer) OnExit(depth int, output []byte, gasUsed uint64, _ error, if t.interrupt.Load() { return } + if depth == 0 { + return + } var err error t.open, err = t.stack.Pop() if err != nil { @@ -199,17 +198,3 @@ func (t *stylusTracer) Stop(err error) { t.reason = err t.interrupt.Store(true) } - -// Unimplemented EVMLogger interface methods - -func (t *stylusTracer) CaptureArbitrumTransfer(from, to *common.Address, value *big.Int, before bool, purpose string) { -} -func (t *stylusTracer) CaptureArbitrumStorageGet(key common.Hash, depth int, before bool) {} -func (t *stylusTracer) CaptureArbitrumStorageSet(key, value common.Hash, depth int, before bool) {} -func (t *stylusTracer) OnOpcode(pc uint64, opcode byte, gas, cost uint64, scope tracing.OpContext, rData []byte, depth int, err error) { -} -func (t *stylusTracer) OnFault(pc uint64, op byte, gas, cost uint64, _ tracing.OpContext, depth int, err error) { -} -func (t *stylusTracer) OnTxStart(env *tracing.VMContext, tx *types.Transaction, from common.Address) { -} -func (t *stylusTracer) OnTxEnd(receipt *types.Receipt, err error) {} From 3a61eb3efc7bab2d64062c3504df571d32b47535 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Mon, 7 Oct 2024 15:13:14 -0300 Subject: [PATCH 333/489] Remove unused import --- execution/gethexec/stylus_tracer.go | 1 - 1 file changed, 1 deletion(-) diff --git a/execution/gethexec/stylus_tracer.go b/execution/gethexec/stylus_tracer.go index dc7e5ae0a4..cb4e858048 100644 --- a/execution/gethexec/stylus_tracer.go +++ b/execution/gethexec/stylus_tracer.go @@ -14,7 +14,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/tracing" - "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/log" From 74c82fe7d2a8782bd91550b91a9c45c54fd6614f Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Mon, 7 Oct 2024 15:26:28 -0300 Subject: [PATCH 334/489] Fix program gas tests --- system_tests/program_gas_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/system_tests/program_gas_test.go b/system_tests/program_gas_test.go index 119897cbfe..5fdbb147f5 100644 --- a/system_tests/program_gas_test.go +++ b/system_tests/program_gas_test.go @@ -400,14 +400,15 @@ func evmOpcodesGasUsage(ctx context.Context, rpcClient rpc.ClientInterface, tx * // in the caller's depth. Then, we subtract the gas before the call by the // gas after the call returned. var gasAfterCall uint64 + var found bool for j := i + 1; j < len(result.StructLogs); j++ { if result.StructLogs[j].Depth == result.StructLogs[i].Depth { // back to the original call gasAfterCall = result.StructLogs[j].Gas + result.StructLogs[j].GasCost - break + found = true } } - if gasAfterCall == 0 { + if !found { return nil, fmt.Errorf("malformed log: didn't get back to call original depth") } if i == 0 { From 1d643fde37ed978d6911038f93ca5b560342a3b1 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Mon, 7 Oct 2024 12:48:04 -0600 Subject: [PATCH 335/489] document the fact OCL does not support google cloud storage We are happy to merge community code, but we don't consider it tested or supported. --- das/google_cloud_storage_service.go | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/das/google_cloud_storage_service.go b/das/google_cloud_storage_service.go index 2c490f346c..829f4b5265 100644 --- a/das/google_cloud_storage_service.go +++ b/das/google_cloud_storage_service.go @@ -1,21 +1,24 @@ package das import ( - googlestorage "cloud.google.com/go/storage" "context" "fmt" + "io" + "math" + "sort" + "time" + + googlestorage "cloud.google.com/go/storage" + "github.com/google/go-cmp/cmp" + flag "github.com/spf13/pflag" + "google.golang.org/api/option" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" - "github.com/google/go-cmp/cmp" + "github.com/offchainlabs/nitro/arbstate/daprovider" "github.com/offchainlabs/nitro/das/dastree" "github.com/offchainlabs/nitro/util/pretty" - flag "github.com/spf13/pflag" - "google.golang.org/api/option" - "io" - "math" - "sort" - "time" ) type GoogleCloudStorageOperator interface { @@ -69,7 +72,7 @@ type GoogleCloudStorageServiceConfig struct { var DefaultGoogleCloudStorageServiceConfig = GoogleCloudStorageServiceConfig{} func GoogleCloudConfigAddOptions(prefix string, f *flag.FlagSet) { - f.Bool(prefix+".enable", DefaultGoogleCloudStorageServiceConfig.Enable, "enable storage/retrieval of sequencer batch data from an Google Cloud Storage bucket") + f.Bool(prefix+".enable", DefaultGoogleCloudStorageServiceConfig.Enable, "EXPERIMENTAL/unsupported - enable storage/retrieval of sequencer batch data from an Google Cloud Storage bucket") f.String(prefix+".access-token", DefaultGoogleCloudStorageServiceConfig.AccessToken, "Google Cloud Storage access token") f.String(prefix+".bucket", DefaultGoogleCloudStorageServiceConfig.Bucket, "Google Cloud Storage bucket") f.String(prefix+".object-prefix", DefaultGoogleCloudStorageServiceConfig.ObjectPrefix, "prefix to add to Google Cloud Storage objects") From 6c635821bdb214c244f35895f971428c1c436063 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Mon, 7 Oct 2024 16:59:36 -0300 Subject: [PATCH 336/489] Revert "Enable DAP fallback on test config" This reverts commit 143c68be28db76d12c71fa5f9c4d9c2e303b2f66. --- arbnode/batch_poster.go | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/arbnode/batch_poster.go b/arbnode/batch_poster.go index 8c80d65009..4e4a8d8572 100644 --- a/arbnode/batch_poster.go +++ b/arbnode/batch_poster.go @@ -267,27 +267,26 @@ var DefaultBatchPosterL1WalletConfig = genericconf.WalletConfig{ } var TestBatchPosterConfig = BatchPosterConfig{ - Enable: true, - EnableDapFallbackStoreDataOnChain: true, - MaxSize: 100000, - Max4844BatchSize: DefaultBatchPosterConfig.Max4844BatchSize, - PollInterval: time.Millisecond * 10, - ErrorDelay: time.Millisecond * 10, - MaxDelay: 0, - WaitForMaxDelay: false, - CompressionLevel: 2, - DASRetentionPeriod: daprovider.DefaultDASRetentionPeriod, - GasRefunderAddress: "", - ExtraBatchGas: 10_000, - Post4844Blobs: true, - IgnoreBlobPrice: false, - DataPoster: dataposter.TestDataPosterConfig, - ParentChainWallet: DefaultBatchPosterL1WalletConfig, - L1BlockBound: "", - L1BlockBoundBypass: time.Hour, - UseAccessLists: true, - GasEstimateBaseFeeMultipleBips: arbmath.OneInUBips * 3 / 2, - CheckBatchCorrectness: true, + Enable: true, + MaxSize: 100000, + Max4844BatchSize: DefaultBatchPosterConfig.Max4844BatchSize, + PollInterval: time.Millisecond * 10, + ErrorDelay: time.Millisecond * 10, + MaxDelay: 0, + WaitForMaxDelay: false, + CompressionLevel: 2, + DASRetentionPeriod: daprovider.DefaultDASRetentionPeriod, + GasRefunderAddress: "", + ExtraBatchGas: 10_000, + Post4844Blobs: true, + IgnoreBlobPrice: false, + DataPoster: dataposter.TestDataPosterConfig, + ParentChainWallet: DefaultBatchPosterL1WalletConfig, + L1BlockBound: "", + L1BlockBoundBypass: time.Hour, + UseAccessLists: true, + GasEstimateBaseFeeMultipleBips: arbmath.OneInUBips * 3 / 2, + CheckBatchCorrectness: true, } type BatchPosterOpts struct { From 2291972defd371b9cb94983104abc055b7bafe5b Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Mon, 7 Oct 2024 16:59:49 -0300 Subject: [PATCH 337/489] Revert "Use positive in config name (disable -> enable)" This reverts commit f0f5af5798d3bd6f4f6be5a1c16c3cba82567458. --- arbnode/batch_poster.go | 12 ++++++------ arbstate/daprovider/writer.go | 6 +++--- system_tests/das_test.go | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/arbnode/batch_poster.go b/arbnode/batch_poster.go index 4e4a8d8572..44b360e76e 100644 --- a/arbnode/batch_poster.go +++ b/arbnode/batch_poster.go @@ -141,8 +141,8 @@ type BatchPosterDangerousConfig struct { } type BatchPosterConfig struct { - Enable bool `koanf:"enable"` - EnableDapFallbackStoreDataOnChain bool `koanf:"enable-dap-fallback-store-data-on-chain" reload:"hot"` + Enable bool `koanf:"enable"` + DisableDapFallbackStoreDataOnChain bool `koanf:"disable-dap-fallback-store-data-on-chain" reload:"hot"` // Max batch size. MaxSize int `koanf:"max-size" reload:"hot"` // Maximum 4844 blob enabled batch size. @@ -205,7 +205,7 @@ type BatchPosterConfigFetcher func() *BatchPosterConfig func BatchPosterConfigAddOptions(prefix string, f *pflag.FlagSet) { f.Bool(prefix+".enable", DefaultBatchPosterConfig.Enable, "enable posting batches to l1") - f.Bool(prefix+".enable-dap-fallback-store-data-on-chain", DefaultBatchPosterConfig.EnableDapFallbackStoreDataOnChain, "If unable to batch to DA provider, enable fallback storing data on chain") + f.Bool(prefix+".disable-dap-fallback-store-data-on-chain", DefaultBatchPosterConfig.DisableDapFallbackStoreDataOnChain, "If unable to batch to DA provider, disable fallback storing data on chain") f.Int(prefix+".max-size", DefaultBatchPosterConfig.MaxSize, "maximum batch size") f.Int(prefix+".max-4844-batch-size", DefaultBatchPosterConfig.Max4844BatchSize, "maximum 4844 blob enabled batch size") f.Duration(prefix+".max-delay", DefaultBatchPosterConfig.MaxDelay, "maximum batch posting delay") @@ -231,8 +231,8 @@ func BatchPosterConfigAddOptions(prefix string, f *pflag.FlagSet) { } var DefaultBatchPosterConfig = BatchPosterConfig{ - Enable: false, - EnableDapFallbackStoreDataOnChain: true, + Enable: false, + DisableDapFallbackStoreDataOnChain: false, // This default is overridden for L3 chains in applyChainParameters in cmd/nitro/nitro.go MaxSize: 100000, // Try to fill 3 blobs per batch @@ -1366,7 +1366,7 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) return false, fmt.Errorf("%w: nonce changed from %d to %d while creating batch", storage.ErrStorageRace, nonce, gotNonce) } // #nosec G115 - sequencerMsg, err = b.dapWriter.Store(ctx, sequencerMsg, uint64(time.Now().Add(config.DASRetentionPeriod).Unix()), config.EnableDapFallbackStoreDataOnChain) + sequencerMsg, err = b.dapWriter.Store(ctx, sequencerMsg, uint64(time.Now().Add(config.DASRetentionPeriod).Unix()), config.DisableDapFallbackStoreDataOnChain) if err != nil { batchPosterDAFailureCounter.Inc(1) return false, err diff --git a/arbstate/daprovider/writer.go b/arbstate/daprovider/writer.go index 1d83c0348f..a26e53c94d 100644 --- a/arbstate/daprovider/writer.go +++ b/arbstate/daprovider/writer.go @@ -17,7 +17,7 @@ type Writer interface { ctx context.Context, message []byte, timeout uint64, - enableFallbackStoreDataOnChain bool, + disableFallbackStoreDataOnChain bool, ) ([]byte, error) } @@ -31,10 +31,10 @@ type writerForDAS struct { dasWriter DASWriter } -func (d *writerForDAS) Store(ctx context.Context, message []byte, timeout uint64, enableFallbackStoreDataOnChain bool) ([]byte, error) { +func (d *writerForDAS) Store(ctx context.Context, message []byte, timeout uint64, disableFallbackStoreDataOnChain bool) ([]byte, error) { cert, err := d.dasWriter.Store(ctx, message, timeout) if errors.Is(err, ErrBatchToDasFailed) { - if !enableFallbackStoreDataOnChain { + if disableFallbackStoreDataOnChain { return nil, errors.New("unable to batch to DAS and fallback storing data on chain is disabled") } log.Warn("Falling back to storing data on chain", "err", err) diff --git a/system_tests/das_test.go b/system_tests/das_test.go index 09779b80e5..ed3844d528 100644 --- a/system_tests/das_test.go +++ b/system_tests/das_test.go @@ -347,7 +347,7 @@ func TestDASBatchPosterFallback(t *testing.T) { builder.nodeConfig.DataAvailability.RestAggregator.Enable = true builder.nodeConfig.DataAvailability.RestAggregator.Urls = []string{restServerUrl} builder.nodeConfig.DataAvailability.ParentChainNodeURL = "none" - builder.nodeConfig.BatchPoster.EnableDapFallbackStoreDataOnChain = false // Disable DAS fallback + builder.nodeConfig.BatchPoster.DisableDapFallbackStoreDataOnChain = true // Disable DAS fallback builder.nodeConfig.BatchPoster.ErrorDelay = time.Millisecond * 250 // Increase error delay because we expect errors builder.L2Info = NewArbTestInfo(t, builder.chainConfig.ChainID) builder.L2Info.GenerateAccount("User2") @@ -387,7 +387,7 @@ func TestDASBatchPosterFallback(t *testing.T) { // Enable the DAP fallback and check the transaction on the second node. // (We don't need to restart the node because of the hot-reload.) - builder.nodeConfig.BatchPoster.EnableDapFallbackStoreDataOnChain = true + builder.nodeConfig.BatchPoster.DisableDapFallbackStoreDataOnChain = false _, err = WaitForTx(ctx, l2B.Client, tx.Hash(), time.Second*3) Require(t, err) l2balance, err := l2B.Client.BalanceAt(ctx, l2info.GetAddress("User2"), nil) From 8dce92c331f6267e3337ca65adb62217bfcd3102 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Mon, 7 Oct 2024 17:59:44 -0300 Subject: [PATCH 338/489] Fix sum gas comparison --- system_tests/program_gas_test.go | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/system_tests/program_gas_test.go b/system_tests/program_gas_test.go index 5fdbb147f5..64d2585f89 100644 --- a/system_tests/program_gas_test.go +++ b/system_tests/program_gas_test.go @@ -122,20 +122,23 @@ func TestProgramStorageCost(t *testing.T) { writeZeroData = multicallAppendStore(writeZeroData, slot, common.Hash{}, false) } + writePair := compareGasPair{vm.SSTORE, "storage_flush_cache"} + readPair := compareGasPair{vm.SLOAD, "storage_load_bytes32"} + for _, tc := range []struct { name string data []byte + pair compareGasPair }{ - {"initialWrite", writeRandAData}, - {"read", readData}, - {"writeAgain", writeRandBData}, - {"delete", writeZeroData}, - {"readZeros", readData}, - {"writeAgainAgain", writeRandAData}, + {"initialWrite", writeRandAData, writePair}, + {"read", readData, readPair}, + {"writeAgain", writeRandBData, writePair}, + {"delete", writeZeroData, writePair}, + {"readZeros", readData, readPair}, + {"writeAgainAgain", writeRandAData, writePair}, } { t.Run(tc.name, func(t *testing.T) { - compareGasUsage(t, builder, evmMulticall, stylusMulticall, tc.data, nil, compareGasSum, 0, - compareGasPair{vm.SSTORE, "storage_flush_cache"}, compareGasPair{vm.SLOAD, "storage_load_bytes32"}) + compareGasUsage(t, builder, evmMulticall, stylusMulticall, tc.data, nil, compareGasSum, 0, tc.pair) }) } } @@ -360,10 +363,12 @@ func compareGasUsage( } case compareGasSum: evmSum := float64(0) + for _, v := range evmGasUsage[opcode] { + evmSum += float64(v) + } stylusSum := float64(0) - for i := range evmGasUsage[opcode] { - evmSum += float64(evmGasUsage[opcode][i]) - stylusSum += stylusGasUsage[hostio][i] + for _, v := range stylusGasUsage[hostio] { + stylusSum += v } t.Logf("evm %v usage: %v - stylus %v usage: %v", opcode, evmSum, hostio, stylusSum) checkPercentDiff(t, evmSum, stylusSum, maxAllowedDifference) From 9e6cdc77de4f4adeea0f4f83fdd1a4731c8d400a Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Mon, 7 Oct 2024 18:39:35 -0300 Subject: [PATCH 339/489] Fix call program gas-cost tests --- system_tests/program_gas_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/system_tests/program_gas_test.go b/system_tests/program_gas_test.go index 64d2585f89..03f63ff39e 100644 --- a/system_tests/program_gas_test.go +++ b/system_tests/program_gas_test.go @@ -390,6 +390,10 @@ func evmOpcodesGasUsage(ctx context.Context, rpcClient rpc.ClientInterface, tx * op := vm.StringToOp(result.StructLogs[i].Op) gasUsed := uint64(0) if op == vm.CALL || op == vm.STATICCALL || op == vm.DELEGATECALL || op == vm.CREATE || op == vm.CREATE2 { + if result.StructLogs[i].GasCost == 0 { + // ignore mock call emitted by arbos + continue + } // For the CALL* opcodes, the GasCost in the tracer represents the gas sent // to the callee contract, which is 63/64 of the remaining gas. This happens // because the tracer is evaluated before the call is executed, so the EVM @@ -411,6 +415,7 @@ func evmOpcodesGasUsage(ctx context.Context, rpcClient rpc.ClientInterface, tx * // back to the original call gasAfterCall = result.StructLogs[j].Gas + result.StructLogs[j].GasCost found = true + break } } if !found { From 8c655e68db19eccdd959b3814ee236dc0836c0e9 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Tue, 8 Oct 2024 09:46:40 +0530 Subject: [PATCH 340/489] merge master and address PR comments --- .github/workflows/arbitrator-ci.yml | 6 +- .github/workflows/ci.yml | 60 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/gotestsum.sh | 83 +++ .github/workflows/submodule-pin-check.yml | 3 +- Dockerfile | 4 +- Makefile | 8 +- arbitrator/Cargo.lock | 78 +-- arbitrator/arbutil/src/types.rs | 101 +++ arbitrator/bench/Cargo.toml | 5 - arbitrator/bench/src/bin.rs | 6 +- arbitrator/bench/src/lib.rs | 2 - arbitrator/bench/src/parse_input.rs | 76 --- arbitrator/prover/Cargo.toml | 2 +- arbitrator/prover/src/lib.rs | 2 + arbitrator/prover/src/main.rs | 174 ++--- arbitrator/prover/src/parse_input.rs | 112 +++ arbitrator/{bench => prover}/src/prepare.rs | 13 +- .../stylus/tests/hostio-test/Cargo.lock | 636 ++++++++++++++++++ .../stylus/tests/hostio-test/Cargo.toml | 17 + .../stylus/tests/hostio-test/src/main.rs | 207 ++++++ arbitrator/wasm-libraries/Cargo.lock | 276 ++++++-- arbnode/api.go | 7 + arbnode/dataposter/data_poster.go | 4 +- arbnode/dataposter/dataposter_test.go | 63 +- arbnode/delayed.go | 5 +- arbnode/inbox_reader.go | 5 +- arbnode/inbox_tracker.go | 5 +- arbnode/node.go | 5 +- arbnode/sequencer_inbox.go | 9 +- arbnode/transaction_streamer.go | 4 +- arbos/tx_processor.go | 18 +- arbos/util/storage_cache.go | 5 + arbos/util/storage_cache_test.go | 3 +- arbutil/correspondingl1blocknumber.go | 6 +- arbutil/transaction_data.go | 5 +- arbutil/wait_for_l1.go | 24 +- broadcastclient/broadcastclient.go | 12 + cmd/nitro/init.go | 4 +- cmd/pruning/pruning.go | 5 +- contracts | 2 +- das/aggregator.go | 4 +- das/chain_fetch_das.go | 4 +- das/das.go | 8 +- das/dasRpcClient.go | 31 +- das/factory.go | 13 +- das/google_cloud_storage_service.go | 202 ++++++ das/google_cloud_storage_service_test.go | 84 +++ das/rpc_aggregator.go | 4 +- das/sign_after_store_das_writer.go | 1 + das/syncing_fallback_storage.go | 3 +- execution/gethexec/node.go | 3 +- go-ethereum | 2 +- go.mod | 98 +-- go.sum | 234 ++++--- pubsub/producer.go | 1 + staker/block_validator.go | 32 +- staker/l1_validator.go | 5 +- staker/rollup_watcher.go | 44 +- staker/staker.go | 18 +- staker/stateless_block_validator.go | 15 + staker/txbuilder/builder.go | 14 +- staker/validatorwallet/contract.go | 4 +- staker/validatorwallet/eoa.go | 8 +- staker/validatorwallet/noop.go | 8 +- system_tests/common_test.go | 73 +- system_tests/das_test.go | 100 ++- system_tests/eth_sync_test.go | 2 +- system_tests/full_challenge_impl_test.go | 5 +- system_tests/program_gas_test.go | 458 +++++++++++++ system_tests/program_test.go | 7 +- system_tests/validation_mock_test.go | 4 - system_tests/wrap_transaction_test.go | 13 +- util/headerreader/blob_client.go | 6 +- util/headerreader/header_reader.go | 7 +- validator/client/validation_client.go | 13 - validator/inputs/writer.go | 141 ++++ validator/inputs/writer_test.go | 92 +++ validator/interface.go | 1 - validator/server_api/json.go | 13 +- validator/server_arb/validator_spawner.go | 139 +--- validator/server_jit/jit_machine.go | 2 +- validator/valnode/validation_api.go | 9 - 83 files changed, 3149 insertions(+), 840 deletions(-) create mode 100755 .github/workflows/gotestsum.sh delete mode 100644 arbitrator/bench/src/lib.rs delete mode 100644 arbitrator/bench/src/parse_input.rs create mode 100644 arbitrator/prover/src/parse_input.rs rename arbitrator/{bench => prover}/src/prepare.rs (85%) create mode 100644 arbitrator/stylus/tests/hostio-test/Cargo.lock create mode 100644 arbitrator/stylus/tests/hostio-test/Cargo.toml create mode 100644 arbitrator/stylus/tests/hostio-test/src/main.rs create mode 100644 das/google_cloud_storage_service.go create mode 100644 das/google_cloud_storage_service_test.go create mode 100644 system_tests/program_gas_test.go create mode 100644 validator/inputs/writer.go create mode 100644 validator/inputs/writer_test.go diff --git a/.github/workflows/arbitrator-ci.yml b/.github/workflows/arbitrator-ci.yml index 392eb876c0..47646017ac 100644 --- a/.github/workflows/arbitrator-ci.yml +++ b/.github/workflows/arbitrator-ci.yml @@ -50,15 +50,13 @@ jobs: - name: Install go uses: actions/setup-go@v4 with: - go-version: 1.21.x + go-version: 1.23.x - name: Install custom go-ethereum run: | cd /tmp - git clone --branch v1.13.8 --depth 1 https://github.com/ethereum/go-ethereum.git + git clone --branch v1.14.11 --depth 1 https://github.com/ethereum/go-ethereum.git cd go-ethereum - # Enable KZG point evaluation precompile early - sed -i 's#var PrecompiledContractsBerlin = map\[common.Address\]PrecompiledContract{#\0 common.BytesToAddress([]byte{0x0a}): \&kzgPointEvaluation{},#g' core/vm/contracts.go go build -o /usr/local/bin/geth ./cmd/geth - name: Setup nodejs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b439fe4aec..a944f08f40 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,7 +46,7 @@ jobs: - name: Install go uses: actions/setup-go@v4 with: - go-version: 1.21.x + go-version: 1.23.x - name: Install wasm-ld run: | @@ -145,76 +145,42 @@ jobs: env: TEST_STATE_SCHEME: path run: | - packages=`go list ./...` - for package in $packages; do - echo running tests for $package - if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -timeout 20m -tags=cionly > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then - exit 1 - fi - done + echo "Running tests with Path Scheme" >> full.log + ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags cionly --timeout 20m --cover - name: run tests without race detection and hash state scheme if: matrix.test-mode == 'defaults' env: TEST_STATE_SCHEME: hash run: | - packages=`go list ./...` - for package in $packages; do - echo running tests for $package - if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -timeout 20m -tags=cionly; then - exit 1 - fi - done + echo "Running tests with Hash Scheme" >> full.log + ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags cionly --timeout 20m - name: run tests with race detection and hash state scheme if: matrix.test-mode == 'race' env: TEST_STATE_SCHEME: hash run: | - packages=`go list ./...` - for package in $packages; do - echo running tests for $package - if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -race -timeout 30m; then - exit 1 - fi - done + echo "Running tests with Hash Scheme" >> full.log + ${{ github.workspace }}/.github/workflows/gotestsum.sh --race --timeout 30m - name: run redis tests if: matrix.test-mode == 'defaults' - run: TEST_REDIS=redis://localhost:6379/0 gotestsum --format short-verbose -- -p 1 -run TestRedis ./arbnode/... ./system_tests/... -coverprofile=coverage-redis.txt -covermode=atomic -coverpkg=./... + run: | + echo "Running redis tests" >> full.log + TEST_REDIS=redis://localhost:6379/0 gotestsum --format short-verbose -- -p 1 -run TestRedis ./arbnode/... ./system_tests/... -coverprofile=coverage-redis.txt -covermode=atomic -coverpkg=./... - name: run challenge tests if: matrix.test-mode == 'challenge' - run: | - packages=`go list ./...` - for package in $packages; do - echo running tests for $package - if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -tags=challengetest -run=TestChallenge > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then - exit 1 - fi - done + run: ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags challengetest --run TestChallenge --cover - name: run stylus tests if: matrix.test-mode == 'stylus' - run: | - packages=`go list ./...` - for package in $packages; do - echo running tests for $package - if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -timeout 60m -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -tags=stylustest -run="TestProgramArbitrator" > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then - exit 1 - fi - done + run: ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags stylustest --run TestProgramArbitrator --timeout 60m --cover - name: run long stylus tests if: matrix.test-mode == 'long' - run: | - packages=`go list ./...` - for package in $packages; do - echo running tests for $package - if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -timeout 60m -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -tags=stylustest -run="TestProgramLong" > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then - exit 1 - fi - done + run: ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags stylustest --run TestProgramLong --timeout 60m --cover - name: Archive detailed run log uses: actions/upload-artifact@v3 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 1cde8f06b9..26447947d4 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -73,7 +73,7 @@ jobs: - name: Install go uses: actions/setup-go@v4 with: - go-version: 1.21.x + go-version: 1.23.x - name: Install rust stable uses: dtolnay/rust-toolchain@stable diff --git a/.github/workflows/gotestsum.sh b/.github/workflows/gotestsum.sh new file mode 100755 index 0000000000..ed631847b7 --- /dev/null +++ b/.github/workflows/gotestsum.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +check_missing_value() { + if [[ $1 -eq 0 || $2 == -* ]]; then + echo "missing $3 argument value" + exit 1 + fi +} + +timeout="" +tags="" +run="" +race=false +cover=false +while [[ $# -gt 0 ]]; do + case $1 in + --timeout) + shift + check_missing_value $# "$1" "--timeout" + timeout=$1 + shift + ;; + --tags) + shift + check_missing_value $# "$1" "--tags" + tags=$1 + shift + ;; + --run) + shift + check_missing_value $# "$1" "--run" + run=$1 + shift + ;; + --race) + race=true + shift + ;; + --cover) + cover=true + shift + ;; + *) + echo "Invalid argument: $1" + exit 1 + ;; + esac +done + +packages=$(go list ./...) +for package in $packages; do + cmd="stdbuf -oL gotestsum --format short-verbose --packages=\"$package\" --rerun-fails=2 --no-color=false --" + + if [ "$timeout" != "" ]; then + cmd="$cmd -timeout $timeout" + fi + + if [ "$tags" != "" ]; then + cmd="$cmd -tags=$tags" + fi + + if [ "$run" != "" ]; then + cmd="$cmd -run=$run" + fi + + if [ "$race" == true ]; then + cmd="$cmd -race" + fi + + if [ "$cover" == true ]; then + cmd="$cmd -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/..." + fi + + cmd="$cmd > >(stdbuf -oL tee -a full.log | grep -vE \"INFO|seal\")" + + echo "" + echo running tests for "$package" + echo "$cmd" + + if ! eval "$cmd"; then + exit 1 + fi +done diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index 90419b530e..60dd8ad827 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -18,7 +18,8 @@ jobs: with: fetch-depth: 0 submodules: true - ref: "${{ github.event.pull_request.merge_commit_sha }}" + persist-credentials: false + ref: "${{ github.event.pull_request.head.sha }}" - name: Check all submodules are ancestors of origin/HEAD or configured branch run: | diff --git a/Dockerfile b/Dockerfile index 9138ed30ad..aba5432254 100644 --- a/Dockerfile +++ b/Dockerfile @@ -66,7 +66,7 @@ COPY --from=wasm-libs-builder /workspace/ / FROM wasm-base AS wasm-bin-builder # pinned go version -RUN curl -L https://golang.org/dl/go1.21.10.linux-`dpkg --print-architecture`.tar.gz | tar -C /usr/local -xzf - +RUN curl -L https://golang.org/dl/go1.23.1.linux-`dpkg --print-architecture`.tar.gz | tar -C /usr/local -xzf - COPY ./Makefile ./go.mod ./go.sum ./ COPY ./arbcompress ./arbcompress COPY ./arbos ./arbos @@ -220,7 +220,7 @@ RUN ./download-machine.sh consensus-v30 0xb0de9cb89e4d944ae6023a3b62276e54804c24 RUN ./download-machine.sh consensus-v31 0x260f5fa5c3176a856893642e149cf128b5a8de9f828afec8d11184415dd8dc69 RUN ./download-machine.sh consensus-v32 0x184884e1eb9fefdc158f6c8ac912bb183bf3cf83f0090317e0bc4ac5860baa39 -FROM golang:1.21.10-bookworm AS node-builder +FROM golang:1.23.1-bookworm AS node-builder WORKDIR /workspace ARG version="" ARG datetime="" diff --git a/Makefile b/Makefile index c3cf1a5144..88bbd8dabe 100644 --- a/Makefile +++ b/Makefile @@ -149,8 +149,10 @@ stylus_test_erc20_wasm = $(call get_stylus_test_wasm,erc20) stylus_test_erc20_src = $(call get_stylus_test_rust,erc20) stylus_test_read-return-data_wasm = $(call get_stylus_test_wasm,read-return-data) stylus_test_read-return-data_src = $(call get_stylus_test_rust,read-return-data) +stylus_test_hostio-test_wasm = $(call get_stylus_test_wasm,hostio-test) +stylus_test_hostio-test_src = $(call get_stylus_test_rust,hostio-test) -stylus_test_wasms = $(stylus_test_keccak_wasm) $(stylus_test_keccak-100_wasm) $(stylus_test_fallible_wasm) $(stylus_test_storage_wasm) $(stylus_test_multicall_wasm) $(stylus_test_log_wasm) $(stylus_test_create_wasm) $(stylus_test_math_wasm) $(stylus_test_sdk-storage_wasm) $(stylus_test_erc20_wasm) $(stylus_test_read-return-data_wasm) $(stylus_test_evm-data_wasm) $(stylus_test_bfs:.b=.wasm) +stylus_test_wasms = $(stylus_test_keccak_wasm) $(stylus_test_keccak-100_wasm) $(stylus_test_fallible_wasm) $(stylus_test_storage_wasm) $(stylus_test_multicall_wasm) $(stylus_test_log_wasm) $(stylus_test_create_wasm) $(stylus_test_math_wasm) $(stylus_test_sdk-storage_wasm) $(stylus_test_erc20_wasm) $(stylus_test_read-return-data_wasm) $(stylus_test_evm-data_wasm) $(stylus_test_hostio-test_wasm) $(stylus_test_bfs:.b=.wasm) stylus_benchmarks = $(wildcard $(stylus_dir)/*.toml $(stylus_dir)/src/*.rs) $(stylus_test_wasms) # user targets @@ -482,6 +484,10 @@ $(stylus_test_erc20_wasm): $(stylus_test_erc20_src) $(cargo_nightly) --manifest-path $< --release --config $(stylus_cargo) @touch -c $@ # cargo might decide to not rebuild the binary +$(stylus_test_hostio-test_wasm): $(stylus_test_hostio-test_src) + $(cargo_nightly) --manifest-path $< --release --config $(stylus_cargo) + @touch -c $@ # cargo might decide to not rebuild the binary + contracts/test/prover/proofs/float%.json: $(arbitrator_cases)/float%.wasm $(prover_bin) $(output_latest)/soft-float.wasm $(prover_bin) $< -l $(output_latest)/soft-float.wasm -o $@ -b --allow-hostapi --require-success diff --git a/arbitrator/Cargo.lock b/arbitrator/Cargo.lock index 6048733acb..2b437968fa 100644 --- a/arbitrator/Cargo.lock +++ b/arbitrator/Cargo.lock @@ -215,7 +215,6 @@ dependencies = [ "prover", "serde", "serde_json", - "serde_with 3.9.0", ] [[package]] @@ -711,38 +710,14 @@ dependencies = [ "typenum", ] -[[package]] -name = "darling" -version = "0.13.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" -dependencies = [ - "darling_core 0.13.4", - "darling_macro 0.13.4", -] - [[package]] name = "darling" version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" dependencies = [ - "darling_core 0.20.10", - "darling_macro 0.20.10", -] - -[[package]] -name = "darling_core" -version = "0.13.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim 0.10.0", - "syn 1.0.109", + "darling_core", + "darling_macro", ] [[package]] @@ -759,24 +734,13 @@ dependencies = [ "syn 2.0.72", ] -[[package]] -name = "darling_macro" -version = "0.13.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" -dependencies = [ - "darling_core 0.13.4", - "quote", - "syn 1.0.109", -] - [[package]] name = "darling_macro" version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ - "darling_core 0.20.10", + "darling_core", "quote", "syn 2.0.72", ] @@ -934,7 +898,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59c3b24c345d8c314966bdc1832f6c2635bfcce8e7cf363bd115987bba2ee242" dependencies = [ - "darling 0.20.10", + "darling", "proc-macro2", "quote", "syn 2.0.72", @@ -1756,7 +1720,7 @@ dependencies = [ "rustc-demangle", "serde", "serde_json", - "serde_with 1.14.0", + "serde_with", "sha2 0.9.9", "sha3 0.9.1", "smallvec", @@ -2079,16 +2043,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_with" -version = "1.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678b5a069e50bf00ecd22d0cd8ddf7c236f68581b03db652061ed5eb13a312ff" -dependencies = [ - "serde", - "serde_with_macros 1.5.2", -] - [[package]] name = "serde_with" version = "3.9.0" @@ -2103,29 +2057,17 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "serde_with_macros 3.9.0", + "serde_with_macros", "time", ] -[[package]] -name = "serde_with_macros" -version = "1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" -dependencies = [ - "darling 0.13.4", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "serde_with_macros" version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8fee4991ef4f274617a51ad4af30519438dacb2f56ac773b08a1922ff743350" dependencies = [ - "darling 0.20.10", + "darling", "proc-macro2", "quote", "syn 2.0.72", @@ -2232,12 +2174,6 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" -[[package]] -name = "strsim" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" - [[package]] name = "strsim" version = "0.11.1" diff --git a/arbitrator/arbutil/src/types.rs b/arbitrator/arbutil/src/types.rs index 6cf1d6cdf7..722a89b81e 100644 --- a/arbitrator/arbutil/src/types.rs +++ b/arbitrator/arbutil/src/types.rs @@ -8,6 +8,7 @@ use std::{ borrow::Borrow, fmt, ops::{Deref, DerefMut}, + str::FromStr, }; // These values must be kept in sync with `arbutil/preimage_type.go`, @@ -83,6 +84,32 @@ impl From for Bytes32 { } } +impl FromStr for Bytes32 { + type Err = &'static str; + + fn from_str(s: &str) -> Result { + // Remove the "0x" prefix if present + let s = s.strip_prefix("0x").unwrap_or(s); + + // Pad with leading zeros if the string is shorter than 64 characters (32 bytes) + let padded = format!("{:0>64}", s); + + // Decode the hex string using the hex crate + let decoded_bytes = hex::decode(padded).map_err(|_| "Invalid hex string")?; + + // Ensure the decoded bytes is exactly 32 bytes + if decoded_bytes.len() != 32 { + return Err("Hex string too long for Bytes32"); + } + + // Create a 32-byte array and fill it with the decoded bytes. + let mut b = [0u8; 32]; + b.copy_from_slice(&decoded_bytes); + + Ok(Bytes32(b)) + } +} + impl TryFrom<&[u8]> for Bytes32 { type Error = std::array::TryFromSliceError; @@ -249,3 +276,77 @@ impl From for Bytes20 { <[u8; 20]>::from(x).into() } } + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_bytes32() { + let b = Bytes32::from(0x12345678u32); + let expected = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0x12, 0x34, 0x56, 0x78, + ]; + assert_eq!(b, Bytes32(expected)); + } + + #[test] + fn test_from_str_short() { + // Short hex string + let b = Bytes32::from_str("0x12345678").unwrap(); + let expected = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0x12, 0x34, 0x56, 0x78, + ]; + assert_eq!(b, Bytes32(expected)); + } + + #[test] + fn test_from_str_very_short() { + // Short hex string + let b = Bytes32::from_str("0x1").unwrap(); + let expected = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0x1, + ]; + assert_eq!(b, Bytes32(expected)); + } + + #[test] + fn test_from_str_no_prefix() { + // Short hex string + let b = Bytes32::from_str("12345678").unwrap(); + let expected = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0x12, 0x34, 0x56, 0x78, + ]; + assert_eq!(b, Bytes32(expected)); + } + + #[test] + fn test_from_str_full() { + // Full-length hex string + let b = + Bytes32::from_str("0x0000000000000000000000000000000000000000000000000000000012345678") + .unwrap(); + let expected = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0x12, 0x34, 0x56, 0x78, + ]; + assert_eq!(b, Bytes32(expected)); + } + + #[test] + fn test_from_str_invalid_non_hex() { + let s = "0x123g5678"; // Invalid character 'g' + assert!(Bytes32::from_str(s).is_err()); + } + + #[test] + fn test_from_str_too_big() { + let s = + "0123456789ABCDEF0123456789ABCDEF01234567890123456789ABCDEF01234567890123456789ABCDEF0"; // 65 characters + assert!(Bytes32::from_str(s).is_err()); + } +} diff --git a/arbitrator/bench/Cargo.toml b/arbitrator/bench/Cargo.toml index 3ab5b99b08..74b948aca8 100644 --- a/arbitrator/bench/Cargo.toml +++ b/arbitrator/bench/Cargo.toml @@ -3,10 +3,6 @@ name = "bench" version = "0.1.0" edition = "2021" -[lib] -name = "bench" -path = "src/lib.rs" - [[bin]] name = "benchbin" path = "src/bin.rs" @@ -20,7 +16,6 @@ clap = { version = "4.4.8", features = ["derive"] } gperftools = { version = "0.2.0", optional = true } serde = { version = "1.0.130", features = ["derive", "rc"] } serde_json = "1.0.67" -serde_with = { version = "3.8.1", features = ["base64"] } [features] counters = [] diff --git a/arbitrator/bench/src/bin.rs b/arbitrator/bench/src/bin.rs index f7e69f5373..60a7036e2b 100644 --- a/arbitrator/bench/src/bin.rs +++ b/arbitrator/bench/src/bin.rs @@ -1,6 +1,5 @@ use std::{path::PathBuf, time::Duration}; -use bench::prepare::*; use clap::Parser; use eyre::bail; @@ -10,11 +9,12 @@ use gperftools::profiler::PROFILER; #[cfg(feature = "heapprof")] use gperftools::heap_profiler::HEAP_PROFILER; -use prover::machine::MachineStatus; - #[cfg(feature = "counters")] use prover::{machine, memory, merkle}; +use prover::machine::MachineStatus; +use prover::prepare::prepare_machine; + #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] struct Args { diff --git a/arbitrator/bench/src/lib.rs b/arbitrator/bench/src/lib.rs deleted file mode 100644 index 5f7c024094..0000000000 --- a/arbitrator/bench/src/lib.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod parse_input; -pub mod prepare; diff --git a/arbitrator/bench/src/parse_input.rs b/arbitrator/bench/src/parse_input.rs deleted file mode 100644 index decc67372a..0000000000 --- a/arbitrator/bench/src/parse_input.rs +++ /dev/null @@ -1,76 +0,0 @@ -use arbutil::Bytes32; -use serde::{Deserialize, Serialize}; -use serde_json; -use serde_with::base64::Base64; -use serde_with::As; -use serde_with::DisplayFromStr; -use std::{ - collections::HashMap, - io::{self, BufRead}, -}; - -mod prefixed_hex { - use serde::{self, Deserialize, Deserializer, Serializer}; - - pub fn serialize(bytes: &Vec, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_str(&format!("0x{}", hex::encode(bytes))) - } - - pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> - where - D: Deserializer<'de>, - { - let s = String::deserialize(deserializer)?; - if let Some(s) = s.strip_prefix("0x") { - hex::decode(s).map_err(serde::de::Error::custom) - } else { - Err(serde::de::Error::custom("missing 0x prefix")) - } - } -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct PreimageMap(HashMap>); - -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(rename_all = "PascalCase")] -pub struct BatchInfo { - pub number: u64, - #[serde(with = "As::")] - pub data_b64: Vec, -} - -#[derive(Debug, Deserialize, Serialize)] -#[serde(rename_all = "PascalCase")] -pub struct StartState { - #[serde(with = "prefixed_hex")] - pub block_hash: Vec, - #[serde(with = "prefixed_hex")] - pub send_root: Vec, - pub batch: u64, - pub pos_in_batch: u64, -} - -#[derive(Debug, Deserialize, Serialize)] -#[serde(rename_all = "PascalCase")] -pub struct FileData { - pub id: u64, - pub has_delayed_msg: bool, - pub delayed_msg_nr: u64, - #[serde(with = "As::>>")] - pub preimages_b64: HashMap>>, - pub batch_info: Vec, - #[serde(with = "As::")] - pub delayed_msg_b64: Vec, - pub start_state: StartState, -} - -impl FileData { - pub fn from_reader(mut reader: R) -> io::Result { - let data = serde_json::from_reader(&mut reader)?; - Ok(data) - } -} diff --git a/arbitrator/prover/Cargo.toml b/arbitrator/prover/Cargo.toml index 5475647765..da329b1cb5 100644 --- a/arbitrator/prover/Cargo.toml +++ b/arbitrator/prover/Cargo.toml @@ -19,10 +19,10 @@ num = "0.4" rustc-demangle = "0.1.21" serde = { version = "1.0.130", features = ["derive", "rc"] } serde_json = "1.0.67" +serde_with = { version = "3.8.1", features = ["base64"] } sha3 = "0.9.1" static_assertions = "1.1.0" structopt = "0.3.23" -serde_with = "1.12.1" parking_lot = "0.12.1" lazy_static.workspace = true itertools = "0.10.5" diff --git a/arbitrator/prover/src/lib.rs b/arbitrator/prover/src/lib.rs index 0f537478eb..08473c2598 100644 --- a/arbitrator/prover/src/lib.rs +++ b/arbitrator/prover/src/lib.rs @@ -11,6 +11,8 @@ pub mod machine; /// cbindgen:ignore pub mod memory; pub mod merkle; +pub mod parse_input; +pub mod prepare; mod print; pub mod programs; mod reinterpret; diff --git a/arbitrator/prover/src/main.rs b/arbitrator/prover/src/main.rs index dba32e0e72..a889cc60f3 100644 --- a/arbitrator/prover/src/main.rs +++ b/arbitrator/prover/src/main.rs @@ -8,6 +8,7 @@ use eyre::{eyre, Context, Result}; use fnv::{FnvHashMap as HashMap, FnvHashSet as HashSet}; use prover::{ machine::{GlobalState, InboxIdentifier, Machine, MachineStatus, PreimageResolver, ProofInfo}, + prepare::prepare_machine, utils::{file_bytes, hash_preimage, CBytes}, wavm::Opcode, }; @@ -86,6 +87,10 @@ struct Opts { skip_until_host_io: bool, #[structopt(long)] max_steps: Option, + // JSON inputs supercede any of the command-line inputs which could + // be specified in the JSON file. + #[structopt(long)] + json_inputs: Option, } fn file_with_stub_header(path: &Path, headerlength: usize) -> Result> { @@ -135,83 +140,8 @@ fn main() -> Result<()> { } } } - let mut inbox_contents = HashMap::default(); - let mut inbox_position = opts.inbox_position; - let mut delayed_position = opts.delayed_inbox_position; - let inbox_header_len; - let delayed_header_len; - if opts.inbox_add_stub_headers { - inbox_header_len = INBOX_HEADER_LEN; - delayed_header_len = DELAYED_HEADER_LEN + 1; - } else { - inbox_header_len = 0; - delayed_header_len = 0; - } - - for path in opts.inbox { - inbox_contents.insert( - (InboxIdentifier::Sequencer, inbox_position), - file_with_stub_header(&path, inbox_header_len)?, - ); - println!("read file {:?} to seq. inbox {}", &path, inbox_position); - inbox_position += 1; - } - for path in opts.delayed_inbox { - inbox_contents.insert( - (InboxIdentifier::Delayed, delayed_position), - file_with_stub_header(&path, delayed_header_len)?, - ); - delayed_position += 1; - } - let mut preimages: HashMap> = HashMap::default(); - if let Some(path) = opts.preimages { - let mut file = BufReader::new(File::open(path)?); - loop { - let mut ty_buf = [0u8; 1]; - match file.read_exact(&mut ty_buf) { - Ok(()) => {} - Err(e) if e.kind() == ErrorKind::UnexpectedEof => break, - Err(e) => return Err(e.into()), - } - let preimage_ty: PreimageType = ty_buf[0].try_into()?; - - let mut size_buf = [0u8; 8]; - file.read_exact(&mut size_buf)?; - let size = u64::from_le_bytes(size_buf) as usize; - let mut buf = vec![0u8; size]; - file.read_exact(&mut buf)?; - - let hash = hash_preimage(&buf, preimage_ty)?; - preimages - .entry(preimage_ty) - .or_default() - .insert(hash.into(), buf.as_slice().into()); - } - } - let preimage_resolver = - Arc::new(move |_, ty, hash| preimages.get(&ty).and_then(|m| m.get(&hash)).cloned()) - as PreimageResolver; - - let last_block_hash = decode_hex_arg(&opts.last_block_hash, "--last-block-hash")?; - let last_send_root = decode_hex_arg(&opts.last_send_root, "--last-send-root")?; - - let global_state = GlobalState { - u64_vals: [opts.inbox_position, opts.position_within_message], - bytes32_vals: [last_block_hash, last_send_root], - }; - - let mut mach = Machine::from_paths( - &opts.libraries, - &opts.binary, - true, - opts.allow_hostapi, - opts.debug_funcs, - true, - global_state, - inbox_contents, - preimage_resolver, - )?; + let mut mach = initialize_machine(&opts)?; for path in &opts.stylus_modules { let err = || eyre!("failed to read module at {}", path.to_string_lossy().red()); @@ -414,6 +344,13 @@ fn main() -> Result<()> { }); } + println!( + "End GlobalState:\n BlockHash: {:?}\n SendRoot: {:?}\n Batch: {}\n PosInBatch: {}", + mach.get_global_state().bytes32_vals[0], + mach.get_global_state().bytes32_vals[1], + mach.get_global_state().u64_vals[0], + mach.get_global_state().u64_vals[1] + ); println!("End machine status: {:?}", mach.get_status()); println!("End machine hash: {}", mach.hash()); println!("End machine stack: {:?}", mach.get_data_stack()); @@ -462,7 +399,6 @@ fn main() -> Result<()> { } } } - let opts_binary = opts.binary; let opts_libraries = opts.libraries; let format_pc = |module_num: usize, func_num: usize| -> (String, String) { @@ -543,3 +479,87 @@ fn main() -> Result<()> { } Ok(()) } + +fn initialize_machine(opts: &Opts) -> eyre::Result { + if let Some(json_inputs) = opts.json_inputs.clone() { + prepare_machine(json_inputs, opts.binary.clone()) + } else { + let mut inbox_contents = HashMap::default(); + let mut inbox_position = opts.inbox_position; + let mut delayed_position = opts.delayed_inbox_position; + let inbox_header_len; + let delayed_header_len; + if opts.inbox_add_stub_headers { + inbox_header_len = INBOX_HEADER_LEN; + delayed_header_len = DELAYED_HEADER_LEN + 1; + } else { + inbox_header_len = 0; + delayed_header_len = 0; + } + + for path in opts.inbox.clone() { + inbox_contents.insert( + (InboxIdentifier::Sequencer, inbox_position), + file_with_stub_header(&path, inbox_header_len)?, + ); + println!("read file {:?} to seq. inbox {}", &path, inbox_position); + inbox_position += 1; + } + for path in opts.delayed_inbox.clone() { + inbox_contents.insert( + (InboxIdentifier::Delayed, delayed_position), + file_with_stub_header(&path, delayed_header_len)?, + ); + delayed_position += 1; + } + + let mut preimages: HashMap> = HashMap::default(); + if let Some(path) = opts.preimages.clone() { + let mut file = BufReader::new(File::open(path)?); + loop { + let mut ty_buf = [0u8; 1]; + match file.read_exact(&mut ty_buf) { + Ok(()) => {} + Err(e) if e.kind() == ErrorKind::UnexpectedEof => break, + Err(e) => return Err(e.into()), + } + let preimage_ty: PreimageType = ty_buf[0].try_into()?; + + let mut size_buf = [0u8; 8]; + file.read_exact(&mut size_buf)?; + let size = u64::from_le_bytes(size_buf) as usize; + let mut buf = vec![0u8; size]; + file.read_exact(&mut buf)?; + + let hash = hash_preimage(&buf, preimage_ty)?; + preimages + .entry(preimage_ty) + .or_default() + .insert(hash.into(), buf.as_slice().into()); + } + } + let preimage_resolver = + Arc::new(move |_, ty, hash| preimages.get(&ty).and_then(|m| m.get(&hash)).cloned()) + as PreimageResolver; + + let last_block_hash = decode_hex_arg(&opts.last_block_hash, "--last-block-hash")?; + let last_send_root = decode_hex_arg(&opts.last_send_root, "--last-send-root")?; + + let global_state = GlobalState { + u64_vals: [opts.inbox_position, opts.position_within_message], + bytes32_vals: [last_block_hash, last_send_root], + }; + + Machine::from_paths( + &opts.libraries, + &opts.binary, + true, + opts.allow_hostapi, + opts.debug_funcs, + true, + global_state, + inbox_contents, + preimage_resolver, + ) + } +} diff --git a/arbitrator/prover/src/parse_input.rs b/arbitrator/prover/src/parse_input.rs new file mode 100644 index 0000000000..fa7adb4c41 --- /dev/null +++ b/arbitrator/prover/src/parse_input.rs @@ -0,0 +1,112 @@ +use arbutil::Bytes32; +use serde::Deserialize; +use serde_json; +use serde_with::base64::Base64; +use serde_with::As; +use serde_with::DisplayFromStr; +use std::{ + collections::HashMap, + io::{self, BufRead}, +}; + +/// prefixed_hex deserializes hex strings which are prefixed with `0x` +/// +/// The default hex deserializer does not support prefixed hex strings. +/// +/// It is an error to use this deserializer on a string that does not +/// begin with `0x`. +mod prefixed_hex { + use serde::{self, Deserialize, Deserializer}; + + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + if let Some(s) = s.strip_prefix("0x") { + hex::decode(s).map_err(serde::de::Error::custom) + } else { + Err(serde::de::Error::custom("missing 0x prefix")) + } + } +} + +#[derive(Debug)] +pub struct UserWasm(Vec); + +/// UserWasm is a wrapper around Vec +/// +/// It is useful for decompressing a brotli-compressed wasm module. +/// +/// Note: The wrapped Vec is already Base64 decoded before +/// from(Vec) is called by serde. +impl UserWasm { + /// as_vec returns the decompressed wasm module as a Vec + pub fn as_vec(&self) -> Vec { + self.0.clone() + } +} + +impl AsRef<[u8]> for UserWasm { + fn as_ref(&self) -> &[u8] { + &self.0 + } +} + +/// The Vec is compressed using brotli, and must be decompressed before use. +impl From> for UserWasm { + fn from(data: Vec) -> Self { + let decompressed = brotli::decompress(&data, brotli::Dictionary::Empty).unwrap(); + Self(decompressed) + } +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "PascalCase")] +pub struct BatchInfo { + pub number: u64, + #[serde(with = "As::")] + pub data_b64: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "PascalCase")] +pub struct StartState { + #[serde(with = "prefixed_hex")] + pub block_hash: Vec, + #[serde(with = "prefixed_hex")] + pub send_root: Vec, + pub batch: u64, + pub pos_in_batch: u64, +} + +/// FileData is the deserialized form of the input JSON file. +/// +/// The go JSON library in json.go uses some custom serialization and +/// compression logic that needs to be reversed when deserializing the +/// JSON in rust. +/// +/// Note: It is important to change this file whenever the go JSON +/// serialization changes. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "PascalCase")] +pub struct FileData { + pub id: u64, + pub has_delayed_msg: bool, + pub delayed_msg_nr: u64, + #[serde(with = "As::>>")] + pub preimages_b64: HashMap>>, + pub batch_info: Vec, + #[serde(with = "As::")] + pub delayed_msg_b64: Vec, + pub start_state: StartState, + #[serde(with = "As::>>")] + pub user_wasms: HashMap>, +} + +impl FileData { + pub fn from_reader(mut reader: R) -> io::Result { + let data = serde_json::from_reader(&mut reader)?; + Ok(data) + } +} diff --git a/arbitrator/bench/src/prepare.rs b/arbitrator/prover/src/prepare.rs similarity index 85% rename from arbitrator/bench/src/prepare.rs rename to arbitrator/prover/src/prepare.rs index 741a7350ac..a485267f39 100644 --- a/arbitrator/bench/src/prepare.rs +++ b/arbitrator/prover/src/prepare.rs @@ -1,13 +1,13 @@ use arbutil::{Bytes32, PreimageType}; -use prover::machine::{argument_data_to_inbox, GlobalState, Machine}; -use prover::utils::CBytes; use std::collections::HashMap; use std::fs::File; use std::io::BufReader; use std::path::{Path, PathBuf}; use std::sync::Arc; +use crate::machine::{argument_data_to_inbox, GlobalState, Machine}; use crate::parse_input::*; +use crate::utils::CBytes; pub fn prepare_machine(preimages: PathBuf, machines: PathBuf) -> eyre::Result { let file = File::open(preimages)?; @@ -40,6 +40,15 @@ pub fn prepare_machine(preimages: PathBuf, machines: PathBuf) -> eyre::Result = std::result::Result>; + +// These are not available as hostios in the sdk, so we import them directly. +#[link(wasm_import_module = "vm_hooks")] +extern "C" { + fn math_div(value: *mut u8, divisor: *const u8); + fn math_mod(value: *mut u8, modulus: *const u8); + fn math_pow(value: *mut u8, exponent: *const u8); + fn math_add_mod(value: *mut u8, addend: *const u8, modulus: *const u8); + fn math_mul_mod(value: *mut u8, multiplier: *const u8, modulus: *const u8); + fn transient_load_bytes32(key: *const u8, dest: *mut u8); + fn transient_store_bytes32(key: *const u8, value: *const u8); + fn exit_early(status: u32); +} + +#[external] +impl HostioTest { + fn exit_early() -> Result<()> { + unsafe { + exit_early(0); + } + Ok(()) + } + + fn transient_load_bytes32(key: B256) -> Result { + let mut result = B256::ZERO; + unsafe { + transient_load_bytes32(key.as_ptr(), result.as_mut_ptr()); + } + Ok(result) + } + + fn transient_store_bytes32(key: B256, value: B256) { + unsafe { + transient_store_bytes32(key.as_ptr(), value.as_ptr()); + } + } + + fn return_data_size() -> Result { + unsafe { Ok(hostio::return_data_size().try_into().unwrap()) } + } + + fn emit_log(data: Bytes, n: i8, t1: B256, t2: B256, t3: B256, t4: B256) -> Result<()> { + let topics = &[t1, t2, t3, t4]; + evm::raw_log(&topics[0..n as usize], data.as_slice())?; + Ok(()) + } + + fn account_balance(account: Address) -> Result { + Ok(account.balance()) + } + + fn account_code(account: Address) -> Result> { + let mut size = 10000; + let mut code = vec![0; size]; + unsafe { + size = hostio::account_code(account.as_ptr(), 0, size, code.as_mut_ptr()); + } + code.resize(size, 0); + Ok(code) + } + + fn account_code_size(account: Address) -> Result { + Ok(account.code_size().try_into().unwrap()) + } + + fn account_codehash(account: Address) -> Result { + Ok(account.codehash()) + } + + fn evm_gas_left() -> Result { + Ok(evm::gas_left().try_into().unwrap()) + } + + fn evm_ink_left() -> Result { + Ok(tx::ink_to_gas(evm::ink_left()).try_into().unwrap()) + } + + fn block_basefee() -> Result { + Ok(block::basefee()) + } + + fn chainid() -> Result { + Ok(block::chainid().try_into().unwrap()) + } + + fn block_coinbase() -> Result
{ + Ok(block::coinbase()) + } + + fn block_gas_limit() -> Result { + Ok(block::gas_limit().try_into().unwrap()) + } + + fn block_number() -> Result { + Ok(block::number().try_into().unwrap()) + } + + fn block_timestamp() -> Result { + Ok(block::timestamp().try_into().unwrap()) + } + + fn contract_address() -> Result
{ + Ok(contract::address()) + } + + fn math_div(a: U256, b: U256) -> Result { + let mut a_bytes: B256 = a.into(); + let b_bytes: B256 = b.into(); + unsafe { + math_div(a_bytes.as_mut_ptr(), b_bytes.as_ptr()); + } + Ok(a_bytes.into()) + } + + fn math_mod(a: U256, b: U256) -> Result { + let mut a_bytes: B256 = a.into(); + let b_bytes: B256 = b.into(); + unsafe { + math_mod(a_bytes.as_mut_ptr(), b_bytes.as_ptr()); + } + Ok(a_bytes.into()) + } + + fn math_pow(a: U256, b: U256) -> Result { + let mut a_bytes: B256 = a.into(); + let b_bytes: B256 = b.into(); + unsafe { + math_pow(a_bytes.as_mut_ptr(), b_bytes.as_ptr()); + } + Ok(a_bytes.into()) + } + + fn math_add_mod(a: U256, b: U256, c: U256) -> Result { + let mut a_bytes: B256 = a.into(); + let b_bytes: B256 = b.into(); + let c_bytes: B256 = c.into(); + unsafe { + math_add_mod(a_bytes.as_mut_ptr(), b_bytes.as_ptr(), c_bytes.as_ptr()); + } + Ok(a_bytes.into()) + } + + fn math_mul_mod(a: U256, b: U256, c: U256) -> Result { + let mut a_bytes: B256 = a.into(); + let b_bytes: B256 = b.into(); + let c_bytes: B256 = c.into(); + unsafe { + math_mul_mod(a_bytes.as_mut_ptr(), b_bytes.as_ptr(), c_bytes.as_ptr()); + } + Ok(a_bytes.into()) + } + + fn msg_sender() -> Result
{ + Ok(msg::sender()) + } + + fn msg_value() -> Result { + Ok(msg::value()) + } + + fn keccak(preimage: Bytes) -> Result { + let mut result = B256::ZERO; + unsafe { + hostio::native_keccak256(preimage.as_ptr(), preimage.len(), result.as_mut_ptr()); + } + Ok(result) + } + + fn tx_gas_price() -> Result { + Ok(tx::gas_price()) + } + + fn tx_ink_price() -> Result { + Ok(tx::ink_to_gas(tx::ink_price().into()).try_into().unwrap()) + } + + fn tx_origin() -> Result
{ + Ok(tx::origin()) + } +} diff --git a/arbitrator/wasm-libraries/Cargo.lock b/arbitrator/wasm-libraries/Cargo.lock index 7620ff538b..a5a066e5c9 100644 --- a/arbitrator/wasm-libraries/Cargo.lock +++ b/arbitrator/wasm-libraries/Cargo.lock @@ -31,6 +31,21 @@ version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "ansi_term" version = "0.12.1" @@ -91,6 +106,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bincode" version = "1.3.3" @@ -203,6 +224,15 @@ dependencies = [ "rand_pcg", ] +[[package]] +name = "cc" +version = "1.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9540e661f81799159abee814118cc139a2004b3a3aa3ea37724a1b66530b90e0" +dependencies = [ + "shlex", +] + [[package]] name = "cfg-if" version = "0.1.10" @@ -215,6 +245,19 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "num-traits", + "serde", + "windows-targets", +] + [[package]] name = "clap" version = "2.34.0" @@ -236,6 +279,12 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "cpufeatures" version = "0.2.12" @@ -261,38 +310,14 @@ dependencies = [ "typenum", ] -[[package]] -name = "darling" -version = "0.13.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" -dependencies = [ - "darling_core 0.13.4", - "darling_macro 0.13.4", -] - [[package]] name = "darling" version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" dependencies = [ - "darling_core 0.20.10", - "darling_macro 0.20.10", -] - -[[package]] -name = "darling_core" -version = "0.13.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim 0.10.0", - "syn 1.0.109", + "darling_core", + "darling_macro", ] [[package]] @@ -305,29 +330,29 @@ dependencies = [ "ident_case", "proc-macro2", "quote", + "strsim 0.11.1", "syn 2.0.72", ] [[package]] name = "darling_macro" -version = "0.13.4" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ - "darling_core 0.13.4", + "darling_core", "quote", - "syn 1.0.109", + "syn 2.0.72", ] [[package]] -name = "darling_macro" -version = "0.20.10" +name = "deranged" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" dependencies = [ - "darling_core 0.20.10", - "quote", - "syn 2.0.72", + "powerfmt", + "serde", ] [[package]] @@ -434,7 +459,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59c3b24c345d8c314966bdc1832f6c2635bfcce8e7cf363bd115987bba2ee242" dependencies = [ - "darling 0.20.10", + "darling", "proc-macro2", "quote", "syn 2.0.72", @@ -548,6 +573,29 @@ dependencies = [ "caller-env", ] +[[package]] +name = "iana-time-zone" +version = "0.1.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -568,6 +616,7 @@ checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", "hashbrown 0.12.3", + "serde", ] [[package]] @@ -578,6 +627,7 @@ checksum = "de3fc2e30ba82dd1b3911c8de1ffc143c74a914a14e99514d7637e3099df5ea0" dependencies = [ "equivalent", "hashbrown 0.14.5", + "serde", ] [[package]] @@ -595,6 +645,15 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +[[package]] +name = "js-sys" +version = "0.3.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "keccak" version = "0.1.5" @@ -632,6 +691,12 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + [[package]] name = "lru" version = "0.12.4" @@ -719,6 +784,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-derive" version = "0.4.2" @@ -832,6 +903,12 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "proc-macro-crate" version = "3.1.0" @@ -1115,24 +1192,32 @@ dependencies = [ [[package]] name = "serde_with" -version = "1.14.0" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678b5a069e50bf00ecd22d0cd8ddf7c236f68581b03db652061ed5eb13a312ff" +checksum = "69cecfa94848272156ea67b2b1a53f20fc7bc638c4a46d2f8abde08f05f4b857" dependencies = [ + "base64", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.3.0", "serde", + "serde_derive", + "serde_json", "serde_with_macros", + "time", ] [[package]] name = "serde_with_macros" -version = "1.5.2" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" +checksum = "a8fee4991ef4f274617a51ad4af30519438dacb2f56ac773b08a1922ff743350" dependencies = [ - "darling 0.13.4", + "darling", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.72", ] [[package]] @@ -1181,6 +1266,12 @@ dependencies = [ "keccak", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "simdutf8" version = "0.1.4" @@ -1216,9 +1307,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "strsim" -version = "0.10.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "structopt" @@ -1307,6 +1398,37 @@ dependencies = [ "syn 2.0.72", ] +[[package]] +name = "time" +version = "0.3.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tiny-keccak" version = "2.0.2" @@ -1445,6 +1567,61 @@ dependencies = [ "wee_alloc", ] +[[package]] +name = "wasm-bindgen" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +dependencies = [ + "cfg-if 1.0.0", + "once_cell", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.72", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" + [[package]] name = "wasm-encoder" version = "0.215.0" @@ -1535,6 +1712,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-targets" version = "0.52.6" diff --git a/arbnode/api.go b/arbnode/api.go index 228ad51cf8..2dabd41bff 100644 --- a/arbnode/api.go +++ b/arbnode/api.go @@ -7,9 +7,11 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/staker" "github.com/offchainlabs/nitro/validator" + "github.com/offchainlabs/nitro/validator/server_api" ) type BlockValidatorAPI struct { @@ -54,3 +56,8 @@ func (a *BlockValidatorDebugAPI) ValidateMessageNumber( result.Valid = valid return result, err } + +func (a *BlockValidatorDebugAPI) ValidationInputsAt(ctx context.Context, msgNum hexutil.Uint64, target ethdb.WasmTarget, +) (server_api.InputJSON, error) { + return a.val.ValidationInputsAt(ctx, arbutil.MessageIndex(msgNum), target) +} diff --git a/arbnode/dataposter/data_poster.go b/arbnode/dataposter/data_poster.go index c2c94b8c10..373d247696 100644 --- a/arbnode/dataposter/data_poster.go +++ b/arbnode/dataposter/data_poster.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/consensus/misc/eip4844" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto/kzg4844" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" @@ -38,7 +39,6 @@ import ( "github.com/offchainlabs/nitro/arbnode/dataposter/noop" "github.com/offchainlabs/nitro/arbnode/dataposter/slice" "github.com/offchainlabs/nitro/arbnode/dataposter/storage" - "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/util/arbmath" "github.com/offchainlabs/nitro/util/blobs" "github.com/offchainlabs/nitro/util/headerreader" @@ -69,7 +69,7 @@ var ( type DataPoster struct { stopwaiter.StopWaiter headerReader *headerreader.HeaderReader - client arbutil.L1Interface + client *ethclient.Client auth *bind.TransactOpts signer signerFn config ConfigFetcher diff --git a/arbnode/dataposter/dataposter_test.go b/arbnode/dataposter/dataposter_test.go index d2c49427be..7bf0f86e6f 100644 --- a/arbnode/dataposter/dataposter_test.go +++ b/arbnode/dataposter/dataposter_test.go @@ -2,17 +2,18 @@ package dataposter import ( "context" + "errors" "fmt" "math/big" "testing" "time" "github.com/Knetic/govaluate" - "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi/bind" - "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" @@ -152,46 +153,36 @@ func TestMaxFeeCapFormulaCalculation(t *testing.T) { } } -type stubL1Client struct { +type stubL1ClientInner struct { senderNonce uint64 suggestedGasTipCap *big.Int - - // Define most of the required methods that aren't used by feeAndTipCaps - backends.SimulatedBackend -} - -func (c *stubL1Client) NonceAt(ctx context.Context, account common.Address, blockNumber *big.Int) (uint64, error) { - return c.senderNonce, nil -} - -func (c *stubL1Client) SuggestGasTipCap(ctx context.Context) (*big.Int, error) { - return c.suggestedGasTipCap, nil -} - -// Not used but we need to define -func (c *stubL1Client) BlockNumber(ctx context.Context) (uint64, error) { - return 0, nil -} - -func (c *stubL1Client) CallContractAtHash(ctx context.Context, msg ethereum.CallMsg, blockHash common.Hash) ([]byte, error) { - return []byte{}, nil } -func (c *stubL1Client) CodeAtHash(ctx context.Context, address common.Address, blockHash common.Hash) ([]byte, error) { - return []byte{}, nil +func (c *stubL1ClientInner) CallContext(ctx_in context.Context, result interface{}, method string, args ...interface{}) error { + switch method { + case "eth_getTransactionCount": + ptr, ok := result.(*hexutil.Uint64) + if !ok { + return errors.New("result is not a *hexutil.Uint64") + } + *ptr = hexutil.Uint64(c.senderNonce) + case "eth_maxPriorityFeePerGas": + ptr, ok := result.(*hexutil.Big) + if !ok { + return errors.New("result is not a *hexutil.Big") + } + *ptr = hexutil.Big(*c.suggestedGasTipCap) + } + return nil } -func (c *stubL1Client) ChainID(ctx context.Context) (*big.Int, error) { +func (c *stubL1ClientInner) EthSubscribe(ctx context.Context, channel interface{}, args ...interface{}) (*rpc.ClientSubscription, error) { return nil, nil } - -func (c *stubL1Client) Client() rpc.ClientInterface { +func (c *stubL1ClientInner) BatchCallContext(ctx context.Context, b []rpc.BatchElem) error { return nil } - -func (c *stubL1Client) TransactionSender(ctx context.Context, tx *types.Transaction, block common.Hash, index uint) (common.Address, error) { - return common.Address{}, nil -} +func (c *stubL1ClientInner) Close() {} func TestFeeAndTipCaps_EnoughBalance_NoBacklog_NoUnconfirmed_BlobTx(t *testing.T) { conf := func() *DataPosterConfig { @@ -223,10 +214,10 @@ func TestFeeAndTipCaps_EnoughBalance_NoBacklog_NoUnconfirmed_BlobTx(t *testing.T extraBacklog: func() uint64 { return 0 }, balance: big.NewInt(0).Mul(big.NewInt(params.Ether), big.NewInt(10)), usingNoOpStorage: false, - client: &stubL1Client{ + client: ethclient.NewClient(&stubL1ClientInner{ senderNonce: 1, suggestedGasTipCap: big.NewInt(2 * params.GWei), - }, + }), auth: &bind.TransactOpts{ From: common.Address{}, }, @@ -354,10 +345,10 @@ func TestFeeAndTipCaps_RBF_RisingBlobFee_FallingBaseFee(t *testing.T) { extraBacklog: func() uint64 { return 0 }, balance: big.NewInt(0).Mul(big.NewInt(params.Ether), big.NewInt(10)), usingNoOpStorage: false, - client: &stubL1Client{ + client: ethclient.NewClient(&stubL1ClientInner{ senderNonce: 1, suggestedGasTipCap: big.NewInt(2 * params.GWei), - }, + }), auth: &bind.TransactOpts{ From: common.Address{}, }, diff --git a/arbnode/delayed.go b/arbnode/delayed.go index 082f0ecf9d..354fa671b3 100644 --- a/arbnode/delayed.go +++ b/arbnode/delayed.go @@ -19,6 +19,7 @@ import ( "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethclient" "github.com/offchainlabs/nitro/arbos/arbostypes" "github.com/offchainlabs/nitro/arbutil" @@ -58,11 +59,11 @@ type DelayedBridge struct { con *bridgegen.IBridge address common.Address fromBlock uint64 - client arbutil.L1Interface + client *ethclient.Client messageProviders map[common.Address]*bridgegen.IDelayedMessageProvider } -func NewDelayedBridge(client arbutil.L1Interface, addr common.Address, fromBlock uint64) (*DelayedBridge, error) { +func NewDelayedBridge(client *ethclient.Client, addr common.Address, fromBlock uint64) (*DelayedBridge, error) { con, err := bridgegen.NewIBridge(addr, client) if err != nil { return nil, err diff --git a/arbnode/inbox_reader.go b/arbnode/inbox_reader.go index c596cfa9b0..98104b2ea7 100644 --- a/arbnode/inbox_reader.go +++ b/arbnode/inbox_reader.go @@ -14,6 +14,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" flag "github.com/spf13/pflag" @@ -93,7 +94,7 @@ type InboxReader struct { delayedBridge *DelayedBridge sequencerInbox *SequencerInbox caughtUpChan chan struct{} - client arbutil.L1Interface + client *ethclient.Client l1Reader *headerreader.HeaderReader // Atomic @@ -101,7 +102,7 @@ type InboxReader struct { lastReadBatchCount atomic.Uint64 } -func NewInboxReader(tracker *InboxTracker, client arbutil.L1Interface, l1Reader *headerreader.HeaderReader, firstMessageBlock *big.Int, delayedBridge *DelayedBridge, sequencerInbox *SequencerInbox, config InboxReaderConfigFetcher) (*InboxReader, error) { +func NewInboxReader(tracker *InboxTracker, client *ethclient.Client, l1Reader *headerreader.HeaderReader, firstMessageBlock *big.Int, delayedBridge *DelayedBridge, sequencerInbox *SequencerInbox, config InboxReaderConfigFetcher) (*InboxReader, error) { err := config().Validate() if err != nil { return nil, err diff --git a/arbnode/inbox_tracker.go b/arbnode/inbox_tracker.go index fe4149c80e..7686fe413f 100644 --- a/arbnode/inbox_tracker.go +++ b/arbnode/inbox_tracker.go @@ -13,6 +13,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" @@ -599,7 +600,7 @@ type multiplexerBackend struct { positionWithinMessage uint64 ctx context.Context - client arbutil.L1Interface + client *ethclient.Client inbox *InboxTracker } @@ -639,7 +640,7 @@ func (b *multiplexerBackend) ReadDelayedInbox(seqNum uint64) (*arbostypes.L1Inco var delayedMessagesMismatch = errors.New("sequencer batch delayed messages missing or different") -func (t *InboxTracker) AddSequencerBatches(ctx context.Context, client arbutil.L1Interface, batches []*SequencerInboxBatch) error { +func (t *InboxTracker) AddSequencerBatches(ctx context.Context, client *ethclient.Client, batches []*SequencerInboxBatch) error { var nextAcc common.Hash var prevbatchmeta BatchMetadata sequenceNumberToKeep := uint64(0) diff --git a/arbnode/node.go b/arbnode/node.go index a9da4ea24b..c5b3bbe071 100644 --- a/arbnode/node.go +++ b/arbnode/node.go @@ -18,6 +18,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" @@ -407,7 +408,7 @@ func createNodeImpl( arbDb ethdb.Database, configFetcher ConfigFetcher, l2Config *params.ChainConfig, - l1client arbutil.L1Interface, + l1client *ethclient.Client, deployInfo *chaininfo.RollupAddresses, txOptsValidator *bind.TransactOpts, txOptsBatchPoster *bind.TransactOpts, @@ -781,7 +782,7 @@ func CreateNode( arbDb ethdb.Database, configFetcher ConfigFetcher, l2Config *params.ChainConfig, - l1client arbutil.L1Interface, + l1client *ethclient.Client, deployInfo *chaininfo.RollupAddresses, txOptsValidator *bind.TransactOpts, txOptsBatchPoster *bind.TransactOpts, diff --git a/arbnode/sequencer_inbox.go b/arbnode/sequencer_inbox.go index 73e52ded53..81146ed46e 100644 --- a/arbnode/sequencer_inbox.go +++ b/arbnode/sequencer_inbox.go @@ -15,6 +15,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" "github.com/offchainlabs/nitro/arbstate/daprovider" "github.com/offchainlabs/nitro/arbutil" @@ -52,10 +53,10 @@ type SequencerInbox struct { con *bridgegen.SequencerInbox address common.Address fromBlock int64 - client arbutil.L1Interface + client *ethclient.Client } -func NewSequencerInbox(client arbutil.L1Interface, addr common.Address, fromBlock int64) (*SequencerInbox, error) { +func NewSequencerInbox(client *ethclient.Client, addr common.Address, fromBlock int64) (*SequencerInbox, error) { con, err := bridgegen.NewSequencerInbox(addr, client) if err != nil { return nil, err @@ -111,7 +112,7 @@ type SequencerInboxBatch struct { serialized []byte // nil if serialization isn't cached yet } -func (m *SequencerInboxBatch) getSequencerData(ctx context.Context, client arbutil.L1Interface) ([]byte, error) { +func (m *SequencerInboxBatch) getSequencerData(ctx context.Context, client *ethclient.Client) ([]byte, error) { switch m.dataLocation { case batchDataTxInput: data, err := arbutil.GetLogEmitterTxData(ctx, client, m.rawLog) @@ -169,7 +170,7 @@ func (m *SequencerInboxBatch) getSequencerData(ctx context.Context, client arbut } } -func (m *SequencerInboxBatch) Serialize(ctx context.Context, client arbutil.L1Interface) ([]byte, error) { +func (m *SequencerInboxBatch) Serialize(ctx context.Context, client *ethclient.Client) ([]byte, error) { if m.serialized != nil { return m.serialized, nil } diff --git a/arbnode/transaction_streamer.go b/arbnode/transaction_streamer.go index 24a0564b97..38b1c003db 100644 --- a/arbnode/transaction_streamer.go +++ b/arbnode/transaction_streamer.go @@ -1140,7 +1140,7 @@ func (s *TransactionStreamer) storeResult( // exposed for testing // return value: true if should be called again immediately -func (s *TransactionStreamer) ExecuteNextMsg(ctx context.Context, exec execution.ExecutionSequencer) bool { +func (s *TransactionStreamer) ExecuteNextMsg(ctx context.Context) bool { if ctx.Err() != nil { return false } @@ -1212,7 +1212,7 @@ func (s *TransactionStreamer) ExecuteNextMsg(ctx context.Context, exec execution } func (s *TransactionStreamer) executeMessages(ctx context.Context, ignored struct{}) time.Duration { - if s.ExecuteNextMsg(ctx, s.exec) { + if s.ExecuteNextMsg(ctx) { return 0 } return s.config().ExecuteMessageLoopDelay diff --git a/arbos/tx_processor.go b/arbos/tx_processor.go index b08c7c5d30..d6c35339f6 100644 --- a/arbos/tx_processor.go +++ b/arbos/tx_processor.go @@ -532,6 +532,20 @@ func (p *TxProcessor) EndTxHook(gasLeft uint64, success bool) { refund := func(refundFrom common.Address, amount *big.Int) { const errLog = "fee address doesn't have enough funds to give user refund" + logMissingRefund := func(err error) { + if !errors.Is(err, vm.ErrInsufficientBalance) { + log.Error("unexpected error refunding balance", "err", err, "feeAddress", refundFrom) + return + } + logLevel := log.Error + isContract := p.evm.StateDB.GetCodeSize(refundFrom) > 0 + if isContract { + // It's expected that the balance might not still be in this address if it's a contract. + logLevel = log.Debug + } + logLevel(errLog, "err", err, "feeAddress", refundFrom) + } + // Refund funds to the fee refund address without overdrafting the L1 deposit. toRefundAddr := takeFunds(maxRefund, amount) err = util.TransferBalance(&refundFrom, &inner.RefundTo, toRefundAddr, p.evm, scenario, "refund") @@ -539,13 +553,13 @@ func (p *TxProcessor) EndTxHook(gasLeft uint64, success bool) { // Normally the network fee address should be holding any collected fees. // However, in theory, they could've been transferred out during the redeem attempt. // If the network fee address doesn't have the necessary balance, log an error and don't give a refund. - log.Error(errLog, "err", err, "feeAddress", refundFrom) + logMissingRefund(err) } // Any extra refund can't be given to the fee refund address if it didn't come from the L1 deposit. // Instead, give the refund to the retryable from address. err = util.TransferBalance(&refundFrom, &inner.From, arbmath.BigSub(amount, toRefundAddr), p.evm, scenario, "refund") if err != nil { - log.Error(errLog, "err", err, "feeAddress", refundFrom) + logMissingRefund(err) } } diff --git a/arbos/util/storage_cache.go b/arbos/util/storage_cache.go index bf05a5824d..9573d1ffc7 100644 --- a/arbos/util/storage_cache.go +++ b/arbos/util/storage_cache.go @@ -5,6 +5,7 @@ package util import ( "github.com/ethereum/go-ethereum/common" + "slices" ) type storageCacheEntry struct { @@ -67,6 +68,10 @@ func (s *storageCache) Flush() []storageCacheStores { }) } } + sortFunc := func(a, b storageCacheStores) int { + return a.Key.Cmp(b.Key) + } + slices.SortFunc(stores, sortFunc) return stores } diff --git a/arbos/util/storage_cache_test.go b/arbos/util/storage_cache_test.go index 1cc4ea14ec..9fd452851d 100644 --- a/arbos/util/storage_cache_test.go +++ b/arbos/util/storage_cache_test.go @@ -4,7 +4,6 @@ package util import ( - "bytes" "slices" "testing" @@ -76,7 +75,7 @@ func TestStorageCache(t *testing.T) { {Key: keys[2], Value: values[2]}, } sortFunc := func(a, b storageCacheStores) int { - return bytes.Compare(a.Key.Bytes(), b.Key.Bytes()) + return a.Key.Cmp(b.Key) } slices.SortFunc(stores, sortFunc) slices.SortFunc(expected, sortFunc) diff --git a/arbutil/correspondingl1blocknumber.go b/arbutil/correspondingl1blocknumber.go index d654e471e2..c8770e2034 100644 --- a/arbutil/correspondingl1blocknumber.go +++ b/arbutil/correspondingl1blocknumber.go @@ -19,7 +19,11 @@ func ParentHeaderToL1BlockNumber(header *types.Header) uint64 { return header.Number.Uint64() } -func CorrespondingL1BlockNumber(ctx context.Context, client L1Interface, parentBlockNumber uint64) (uint64, error) { +type ParentHeaderFetcher interface { + HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) +} + +func CorrespondingL1BlockNumber(ctx context.Context, client ParentHeaderFetcher, parentBlockNumber uint64) (uint64, error) { // #nosec G115 header, err := client.HeaderByNumber(ctx, big.NewInt(int64(parentBlockNumber))) if err != nil { diff --git a/arbutil/transaction_data.go b/arbutil/transaction_data.go index 8270a628bd..c5728967c7 100644 --- a/arbutil/transaction_data.go +++ b/arbutil/transaction_data.go @@ -8,9 +8,10 @@ import ( "fmt" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" ) -func GetLogTransaction(ctx context.Context, client L1Interface, log types.Log) (*types.Transaction, error) { +func GetLogTransaction(ctx context.Context, client *ethclient.Client, log types.Log) (*types.Transaction, error) { tx, err := client.TransactionInBlock(ctx, log.BlockHash, log.TxIndex) if err != nil { return nil, err @@ -22,7 +23,7 @@ func GetLogTransaction(ctx context.Context, client L1Interface, log types.Log) ( } // GetLogEmitterTxData requires that the tx's data is at least 4 bytes long -func GetLogEmitterTxData(ctx context.Context, client L1Interface, log types.Log) ([]byte, error) { +func GetLogEmitterTxData(ctx context.Context, client *ethclient.Client, log types.Log) ([]byte, error) { tx, err := GetLogTransaction(ctx, client, log) if err != nil { return nil, err diff --git a/arbutil/wait_for_l1.go b/arbutil/wait_for_l1.go index 4b4819156d..80dd356b24 100644 --- a/arbutil/wait_for_l1.go +++ b/arbutil/wait_for_l1.go @@ -10,27 +10,13 @@ import ( "math/big" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/ethclient" ) -type L1Interface interface { - bind.ContractBackend - bind.BlockHashContractCaller - ethereum.ChainReader - ethereum.ChainStateReader - ethereum.TransactionReader - TransactionSender(ctx context.Context, tx *types.Transaction, block common.Hash, index uint) (common.Address, error) - BlockNumber(ctx context.Context) (uint64, error) - PendingCallContract(ctx context.Context, msg ethereum.CallMsg) ([]byte, error) - ChainID(ctx context.Context) (*big.Int, error) - Client() rpc.ClientInterface -} - -func SendTxAsCall(ctx context.Context, client L1Interface, tx *types.Transaction, from common.Address, blockNum *big.Int, unlimitedGas bool) ([]byte, error) { +func SendTxAsCall(ctx context.Context, client *ethclient.Client, tx *types.Transaction, from common.Address, blockNum *big.Int, unlimitedGas bool) ([]byte, error) { var gas uint64 if unlimitedGas { gas = 0 @@ -50,7 +36,7 @@ func SendTxAsCall(ctx context.Context, client L1Interface, tx *types.Transaction return client.CallContract(ctx, callMsg, blockNum) } -func GetPendingCallBlockNumber(ctx context.Context, client L1Interface) (*big.Int, error) { +func GetPendingCallBlockNumber(ctx context.Context, client *ethclient.Client) (*big.Int, error) { msg := ethereum.CallMsg{ // Pretend to be a contract deployment to execute EVM code without calling a contract. To: nil, @@ -70,7 +56,7 @@ func GetPendingCallBlockNumber(ctx context.Context, client L1Interface) (*big.In return new(big.Int).SetBytes(callRes), nil } -func DetailTxError(ctx context.Context, client L1Interface, tx *types.Transaction, txRes *types.Receipt) error { +func DetailTxError(ctx context.Context, client *ethclient.Client, tx *types.Transaction, txRes *types.Receipt) error { // Re-execute the transaction as a call to get a better error if ctx.Err() != nil { return ctx.Err() @@ -96,7 +82,7 @@ func DetailTxError(ctx context.Context, client L1Interface, tx *types.Transactio return fmt.Errorf("SendTxAsCall got: %w for tx hash %v", err, tx.Hash()) } -func DetailTxErrorUsingCallMsg(ctx context.Context, client L1Interface, txHash common.Hash, txRes *types.Receipt, callMsg ethereum.CallMsg) error { +func DetailTxErrorUsingCallMsg(ctx context.Context, client *ethclient.Client, txHash common.Hash, txRes *types.Receipt, callMsg ethereum.CallMsg) error { // Re-execute the transaction as a call to get a better error if ctx.Err() != nil { return ctx.Err() diff --git a/broadcastclient/broadcastclient.go b/broadcastclient/broadcastclient.go index 7d27c57fe9..4e97ca8cd0 100644 --- a/broadcastclient/broadcastclient.go +++ b/broadcastclient/broadcastclient.go @@ -280,6 +280,18 @@ func (bc *BroadcastClient) connect(ctx context.Context, nextSeqNum arbutil.Messa MinVersion: tls.VersionTLS12, }, Extensions: extensions, + NetDial: func(ctx context.Context, network, addr string) (net.Conn, error) { + var netDialer net.Dialer + // For tcp connections, prefer IPv4 over IPv6 to avoid rate limiting issues + if network == "tcp" { + conn, err := netDialer.DialContext(ctx, "tcp4", addr) + if err == nil { + return conn, nil + } + return netDialer.DialContext(ctx, "tcp6", addr) + } + return netDialer.DialContext(ctx, network, addr) + }, } if bc.isShuttingDown() { diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index 07c74cb802..9e3ecec747 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" @@ -37,7 +38,6 @@ import ( "github.com/offchainlabs/nitro/arbnode" "github.com/offchainlabs/nitro/arbos/arbosState" "github.com/offchainlabs/nitro/arbos/arbostypes" - "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/cmd/chaininfo" "github.com/offchainlabs/nitro/cmd/conf" "github.com/offchainlabs/nitro/cmd/ipfshelper" @@ -560,7 +560,7 @@ func rebuildLocalWasm(ctx context.Context, config *gethexec.Config, l2BlockChain return chainDb, l2BlockChain, nil } -func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeConfig, chainId *big.Int, cacheConfig *core.CacheConfig, targetConfig *gethexec.StylusTargetConfig, persistentConfig *conf.PersistentConfig, l1Client arbutil.L1Interface, rollupAddrs chaininfo.RollupAddresses) (ethdb.Database, *core.BlockChain, error) { +func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeConfig, chainId *big.Int, cacheConfig *core.CacheConfig, targetConfig *gethexec.StylusTargetConfig, persistentConfig *conf.PersistentConfig, l1Client *ethclient.Client, rollupAddrs chaininfo.RollupAddresses) (ethdb.Database, *core.BlockChain, error) { if !config.Init.Force { if readOnlyDb, err := stack.OpenDatabaseWithFreezerWithExtraOptions("l2chaindata", 0, 0, config.Persistent.Ancient, "l2chaindata/", true, persistentConfig.Pebble.ExtraOptions("l2chaindata")); err == nil { if chainConfig := gethexec.TryReadStoredChainConfig(readOnlyDb); chainConfig != nil { diff --git a/cmd/pruning/pruning.go b/cmd/pruning/pruning.go index 6fc7741478..0755f5ff9e 100644 --- a/cmd/pruning/pruning.go +++ b/cmd/pruning/pruning.go @@ -15,6 +15,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/pruner" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" @@ -80,7 +81,7 @@ func (r *importantRoots) addHeader(header *types.Header, overwrite bool) error { var hashListRegex = regexp.MustCompile("^(0x)?[0-9a-fA-F]{64}(,(0x)?[0-9a-fA-F]{64})*$") // Finds important roots to retain while proving -func findImportantRoots(ctx context.Context, chainDb ethdb.Database, stack *node.Node, initConfig *conf.InitConfig, cacheConfig *core.CacheConfig, persistentConfig *conf.PersistentConfig, l1Client arbutil.L1Interface, rollupAddrs chaininfo.RollupAddresses, validatorRequired bool) ([]common.Hash, error) { +func findImportantRoots(ctx context.Context, chainDb ethdb.Database, stack *node.Node, initConfig *conf.InitConfig, cacheConfig *core.CacheConfig, persistentConfig *conf.PersistentConfig, l1Client *ethclient.Client, rollupAddrs chaininfo.RollupAddresses, validatorRequired bool) ([]common.Hash, error) { chainConfig := gethexec.TryReadStoredChainConfig(chainDb) if chainConfig == nil { return nil, errors.New("database doesn't have a chain config (was this node initialized?)") @@ -233,7 +234,7 @@ func findImportantRoots(ctx context.Context, chainDb ethdb.Database, stack *node return roots.roots, nil } -func PruneChainDb(ctx context.Context, chainDb ethdb.Database, stack *node.Node, initConfig *conf.InitConfig, cacheConfig *core.CacheConfig, persistentConfig *conf.PersistentConfig, l1Client arbutil.L1Interface, rollupAddrs chaininfo.RollupAddresses, validatorRequired bool) error { +func PruneChainDb(ctx context.Context, chainDb ethdb.Database, stack *node.Node, initConfig *conf.InitConfig, cacheConfig *core.CacheConfig, persistentConfig *conf.PersistentConfig, l1Client *ethclient.Client, rollupAddrs chaininfo.RollupAddresses, validatorRequired bool) error { if cacheConfig.StateScheme == rawdb.PathScheme { return nil } diff --git a/contracts b/contracts index 7396313311..b140ed63ac 160000 --- a/contracts +++ b/contracts @@ -1 +1 @@ -Subproject commit 7396313311ab17cb30e2eef27cccf96f0a9e8f7f +Subproject commit b140ed63acdb53cb906ffd1fa3c36fdbd474364e diff --git a/das/aggregator.go b/das/aggregator.go index e7460fa371..372e448e76 100644 --- a/das/aggregator.go +++ b/das/aggregator.go @@ -15,11 +15,11 @@ import ( flag "github.com/spf13/pflag" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/offchainlabs/nitro/arbstate/daprovider" - "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/blsSignatures" "github.com/offchainlabs/nitro/das/dastree" "github.com/offchainlabs/nitro/solgen/go/bridgegen" @@ -114,7 +114,7 @@ func NewAggregator(ctx context.Context, config DataAvailabilityConfig, services func NewAggregatorWithL1Info( config DataAvailabilityConfig, services []ServiceDetails, - l1client arbutil.L1Interface, + l1client *ethclient.Client, seqInboxAddress common.Address, ) (*Aggregator, error) { seqInboxCaller, err := bridgegen.NewSequencerInboxCaller(seqInboxAddress, l1client) diff --git a/das/chain_fetch_das.go b/das/chain_fetch_das.go index 465b54f400..4de6c981cf 100644 --- a/das/chain_fetch_das.go +++ b/das/chain_fetch_das.go @@ -12,8 +12,8 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" - "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/das/dastree" "github.com/offchainlabs/nitro/solgen/go/bridgegen" ) @@ -42,7 +42,7 @@ type KeysetFetcher struct { keysetCache syncedKeysetCache } -func NewKeysetFetcher(l1client arbutil.L1Interface, seqInboxAddr common.Address) (*KeysetFetcher, error) { +func NewKeysetFetcher(l1client *ethclient.Client, seqInboxAddr common.Address) (*KeysetFetcher, error) { seqInbox, err := bridgegen.NewSequencerInbox(seqInboxAddr, l1client) if err != nil { return nil, err diff --git a/das/das.go b/das/das.go index 6bd02fbc75..0b03c05ad6 100644 --- a/das/das.go +++ b/das/das.go @@ -41,9 +41,10 @@ type DataAvailabilityConfig struct { LocalCache CacheConfig `koanf:"local-cache"` RedisCache RedisConfig `koanf:"redis-cache"` - LocalDBStorage LocalDBStorageConfig `koanf:"local-db-storage"` - LocalFileStorage LocalFileStorageConfig `koanf:"local-file-storage"` - S3Storage S3StorageServiceConfig `koanf:"s3-storage"` + LocalDBStorage LocalDBStorageConfig `koanf:"local-db-storage"` + LocalFileStorage LocalFileStorageConfig `koanf:"local-file-storage"` + S3Storage S3StorageServiceConfig `koanf:"s3-storage"` + GoogleCloudStorage GoogleCloudStorageServiceConfig `koanf:"google-cloud-storage"` MigrateLocalDBToFileStorage bool `koanf:"migrate-local-db-to-file-storage"` @@ -114,6 +115,7 @@ func dataAvailabilityConfigAddOptions(prefix string, f *flag.FlagSet, r role) { LocalDBStorageConfigAddOptions(prefix+".local-db-storage", f) LocalFileStorageConfigAddOptions(prefix+".local-file-storage", f) S3ConfigAddOptions(prefix+".s3-storage", f) + GoogleCloudConfigAddOptions(prefix+".google-cloud-storage", f) f.Bool(prefix+".migrate-local-db-to-file-storage", DefaultDataAvailabilityConfig.MigrateLocalDBToFileStorage, "daserver will migrate all data on startup from local-db-storage to local-file-storage, then mark local-db-storage as unusable") // Key config for storage diff --git a/das/dasRpcClient.go b/das/dasRpcClient.go index 7d48ed796d..241f2196b1 100644 --- a/das/dasRpcClient.go +++ b/das/dasRpcClient.go @@ -12,6 +12,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "golang.org/x/sync/errgroup" "github.com/ethereum/go-ethereum/rpc" @@ -21,6 +22,17 @@ import ( "github.com/offchainlabs/nitro/util/signature" ) +var ( + rpcClientStoreRequestGauge = metrics.NewRegisteredGauge("arb/das/rpcclient/store/requests", nil) + rpcClientStoreSuccessGauge = metrics.NewRegisteredGauge("arb/das/rpcclient/store/success", nil) + rpcClientStoreFailureGauge = metrics.NewRegisteredGauge("arb/das/rpcclient/store/failure", nil) + rpcClientStoreStoredBytesGauge = metrics.NewRegisteredGauge("arb/das/rpcclient/store/bytes", nil) + rpcClientStoreDurationHistogram = metrics.NewRegisteredHistogram("arb/das/rpcclient/store/duration", nil, metrics.NewBoundedHistogramSample()) + + rpcClientSendChunkSuccessGauge = metrics.NewRegisteredGauge("arb/das/rpcclient/sendchunk/success", nil) + rpcClientSendChunkFailureGauge = metrics.NewRegisteredGauge("arb/das/rpcclient/sendchunk/failure", nil) +) + type DASRPCClient struct { // implements DataAvailabilityService clnt *rpc.Client url string @@ -58,8 +70,20 @@ func NewDASRPCClient(target string, signer signature.DataSignerFunc, maxStoreChu } func (c *DASRPCClient) Store(ctx context.Context, message []byte, timeout uint64) (*daprovider.DataAvailabilityCertificate, error) { + rpcClientStoreRequestGauge.Inc(1) + start := time.Now() + success := false + defer func() { + if success { + rpcClientStoreSuccessGauge.Inc(1) + } else { + rpcClientStoreFailureGauge.Inc(1) + } + rpcClientStoreDurationHistogram.Update(time.Since(start).Nanoseconds()) + }() + // #nosec G115 - timestamp := uint64(time.Now().Unix()) + timestamp := uint64(start.Unix()) nChunks := uint64(len(message)) / c.chunkSize lastChunkSize := uint64(len(message)) % c.chunkSize if lastChunkSize > 0 { @@ -116,6 +140,9 @@ func (c *DASRPCClient) Store(ctx context.Context, message []byte, timeout uint64 return nil, err } + rpcClientStoreStoredBytesGauge.Inc(int64(len(message))) + success = true + return &daprovider.DataAvailabilityCertificate{ DataHash: common.BytesToHash(storeResult.DataHash), Timeout: uint64(storeResult.Timeout), @@ -133,8 +160,10 @@ func (c *DASRPCClient) sendChunk(ctx context.Context, batchId, i uint64, chunk [ } if err := c.clnt.CallContext(ctx, nil, "das_sendChunk", hexutil.Uint64(batchId), hexutil.Uint64(i), hexutil.Bytes(chunk), hexutil.Bytes(chunkReqSig)); err != nil { + rpcClientSendChunkFailureGauge.Inc(1) return err } + rpcClientSendChunkSuccessGauge.Inc(1) return nil } diff --git a/das/factory.go b/das/factory.go index 7f696912b3..3e9771f932 100644 --- a/das/factory.go +++ b/das/factory.go @@ -9,8 +9,8 @@ import ( "fmt" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethclient" - "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/solgen/go/bridgegen" "github.com/offchainlabs/nitro/util/headerreader" "github.com/offchainlabs/nitro/util/signature" @@ -65,6 +65,15 @@ func CreatePersistentStorageService( storageServices = append(storageServices, s) } + if config.GoogleCloudStorage.Enable { + s, err := NewGoogleCloudStorageService(config.GoogleCloudStorage) + if err != nil { + return nil, nil, err + } + lifecycleManager.Register(s) + storageServices = append(storageServices, s) + } + if len(storageServices) > 1 { s, err := NewRedundantStorageService(ctx, storageServices) if err != nil { @@ -112,7 +121,7 @@ func CreateBatchPosterDAS( ctx context.Context, config *DataAvailabilityConfig, dataSigner signature.DataSignerFunc, - l1Reader arbutil.L1Interface, + l1Reader *ethclient.Client, sequencerInboxAddr common.Address, ) (DataAvailabilityServiceWriter, DataAvailabilityServiceReader, *KeysetFetcher, *LifecycleManager, error) { if !config.Enable { diff --git a/das/google_cloud_storage_service.go b/das/google_cloud_storage_service.go new file mode 100644 index 0000000000..2c490f346c --- /dev/null +++ b/das/google_cloud_storage_service.go @@ -0,0 +1,202 @@ +package das + +import ( + googlestorage "cloud.google.com/go/storage" + "context" + "fmt" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "github.com/google/go-cmp/cmp" + "github.com/offchainlabs/nitro/arbstate/daprovider" + "github.com/offchainlabs/nitro/das/dastree" + "github.com/offchainlabs/nitro/util/pretty" + flag "github.com/spf13/pflag" + "google.golang.org/api/option" + "io" + "math" + "sort" + "time" +) + +type GoogleCloudStorageOperator interface { + Bucket(name string) *googlestorage.BucketHandle + Upload(ctx context.Context, bucket, objectPrefix string, value []byte) error + Download(ctx context.Context, bucket, objectPrefix string, key common.Hash) ([]byte, error) + Close(ctx context.Context) error +} + +type GoogleCloudStorageClient struct { + client *googlestorage.Client +} + +func (g *GoogleCloudStorageClient) Bucket(name string) *googlestorage.BucketHandle { + return g.client.Bucket(name) +} + +func (g *GoogleCloudStorageClient) Upload(ctx context.Context, bucket, objectPrefix string, value []byte) error { + obj := g.client.Bucket(bucket).Object(objectPrefix + EncodeStorageServiceKey(dastree.Hash(value))) + w := obj.NewWriter(ctx) + + if _, err := fmt.Fprintln(w, value); err != nil { + return err + } + return w.Close() + +} + +func (g *GoogleCloudStorageClient) Download(ctx context.Context, bucket, objectPrefix string, key common.Hash) ([]byte, error) { + obj := g.client.Bucket(bucket).Object(objectPrefix + EncodeStorageServiceKey(key)) + reader, err := obj.NewReader(ctx) + if err != nil { + return nil, err + } + return io.ReadAll(reader) +} + +func (g *GoogleCloudStorageClient) Close(ctx context.Context) error { + return g.client.Close() +} + +type GoogleCloudStorageServiceConfig struct { + Enable bool `koanf:"enable"` + AccessToken string `koanf:"access-token"` + Bucket string `koanf:"bucket"` + ObjectPrefix string `koanf:"object-prefix"` + EnableExpiry bool `koanf:"enable-expiry"` + MaxRetention time.Duration `koanf:"max-retention"` +} + +var DefaultGoogleCloudStorageServiceConfig = GoogleCloudStorageServiceConfig{} + +func GoogleCloudConfigAddOptions(prefix string, f *flag.FlagSet) { + f.Bool(prefix+".enable", DefaultGoogleCloudStorageServiceConfig.Enable, "enable storage/retrieval of sequencer batch data from an Google Cloud Storage bucket") + f.String(prefix+".access-token", DefaultGoogleCloudStorageServiceConfig.AccessToken, "Google Cloud Storage access token") + f.String(prefix+".bucket", DefaultGoogleCloudStorageServiceConfig.Bucket, "Google Cloud Storage bucket") + f.String(prefix+".object-prefix", DefaultGoogleCloudStorageServiceConfig.ObjectPrefix, "prefix to add to Google Cloud Storage objects") + f.Bool(prefix+".enable-expiry", DefaultLocalFileStorageConfig.EnableExpiry, "enable expiry of batches") + f.Duration(prefix+".max-retention", DefaultLocalFileStorageConfig.MaxRetention, "store requests with expiry times farther in the future than max-retention will be rejected") + +} + +type GoogleCloudStorageService struct { + operator GoogleCloudStorageOperator + bucket string + objectPrefix string + enableExpiry bool + maxRetention time.Duration +} + +func NewGoogleCloudStorageService(config GoogleCloudStorageServiceConfig) (StorageService, error) { + var client *googlestorage.Client + var err error + // Note that if the credentials are not specified, the client library will find credentials using ADC(Application Default Credentials) + // https://cloud.google.com/docs/authentication/provide-credentials-adc. + if config.AccessToken == "" { + client, err = googlestorage.NewClient(context.Background()) + } else { + client, err = googlestorage.NewClient(context.Background(), option.WithCredentialsJSON([]byte(config.AccessToken))) + } + if err != nil { + return nil, fmt.Errorf("error creating Google Cloud Storage client: %w", err) + } + service := &GoogleCloudStorageService{ + operator: &GoogleCloudStorageClient{client: client}, + bucket: config.Bucket, + objectPrefix: config.ObjectPrefix, + enableExpiry: config.EnableExpiry, + maxRetention: config.MaxRetention, + } + if config.EnableExpiry { + lifecycleRule := googlestorage.LifecycleRule{ + Action: googlestorage.LifecycleAction{Type: "Delete"}, + Condition: googlestorage.LifecycleCondition{AgeInDays: int64(config.MaxRetention.Hours() / 24)}, // Objects older than 30 days + } + ctx := context.Background() + bucket := service.operator.Bucket(service.bucket) + // check if bucket exists (and others), and update expiration policy if enabled + attrs, err := bucket.Attrs(ctx) + if err != nil { + return nil, fmt.Errorf("error getting bucket attributes: %w", err) + } + attrs.Lifecycle.Rules = append(attrs.Lifecycle.Rules, lifecycleRule) + + bucketAttrsToUpdate := googlestorage.BucketAttrsToUpdate{ + Lifecycle: &attrs.Lifecycle, + } + if _, err := bucket.Update(ctx, bucketAttrsToUpdate); err != nil { + return nil, fmt.Errorf("failed to update bucket lifecycle: %w", err) + } + } + return service, nil +} + +func (gcs *GoogleCloudStorageService) Put(ctx context.Context, data []byte, expiry uint64) error { + logPut("das.GoogleCloudStorageService.Store", data, expiry, gcs) + if expiry > math.MaxInt64 { + return fmt.Errorf("request expiry time (%v) exceeds max int64", expiry) + } + // #nosec G115 + expiryTime := time.Unix(int64(expiry), 0) + currentTimePlusRetention := time.Now().Add(gcs.maxRetention) + if expiryTime.After(currentTimePlusRetention) { + return fmt.Errorf("requested expiry time (%v) exceeds current time plus maximum allowed retention period(%v)", expiryTime, currentTimePlusRetention) + } + if err := gcs.operator.Upload(ctx, gcs.bucket, gcs.objectPrefix, data); err != nil { + log.Error("das.GoogleCloudStorageService.Store", "err", err) + return err + } + return nil +} + +func (gcs *GoogleCloudStorageService) GetByHash(ctx context.Context, key common.Hash) ([]byte, error) { + log.Trace("das.GoogleCloudStorageService.GetByHash", "key", pretty.PrettyHash(key), "this", gcs) + buf, err := gcs.operator.Download(ctx, gcs.bucket, gcs.objectPrefix, key) + if err != nil { + log.Error("das.GoogleCloudStorageService.GetByHash", "err", err) + return nil, err + } + return buf, nil +} + +func (gcs *GoogleCloudStorageService) ExpirationPolicy(ctx context.Context) (daprovider.ExpirationPolicy, error) { + if gcs.enableExpiry { + return daprovider.KeepForever, nil + } + return daprovider.DiscardAfterDataTimeout, nil +} + +func (gcs *GoogleCloudStorageService) Sync(ctx context.Context) error { + return nil +} + +func (gcs *GoogleCloudStorageService) Close(ctx context.Context) error { + return gcs.operator.Close(ctx) +} + +func (gcs *GoogleCloudStorageService) String() string { + return fmt.Sprintf("GoogleCloudStorageService(:%s)", gcs.bucket) +} + +func (gcs *GoogleCloudStorageService) HealthCheck(ctx context.Context) error { + bucket := gcs.operator.Bucket(gcs.bucket) + // check if we have bucket permissions + permissions := []string{ + "storage.buckets.get", + "storage.buckets.list", + "storage.objects.create", + "storage.objects.delete", + "storage.objects.list", + "storage.objects.get", + } + perms, err := bucket.IAM().TestPermissions(ctx, permissions) + if err != nil { + return fmt.Errorf("could not check permissions: %w", err) + } + sort.Strings(permissions) + sort.Strings(perms) + if !cmp.Equal(perms, permissions) { + return fmt.Errorf("permissions mismatch (-want +got):\n%s", cmp.Diff(permissions, perms)) + } + + return nil +} diff --git a/das/google_cloud_storage_service_test.go b/das/google_cloud_storage_service_test.go new file mode 100644 index 0000000000..799d999bad --- /dev/null +++ b/das/google_cloud_storage_service_test.go @@ -0,0 +1,84 @@ +package das + +import ( + "bytes" + googlestorage "cloud.google.com/go/storage" + "context" + "errors" + "github.com/ethereum/go-ethereum/common" + "github.com/offchainlabs/nitro/das/dastree" + "testing" + "time" +) + +type mockGCSClient struct { + storage map[string][]byte +} + +func (c *mockGCSClient) Bucket(name string) *googlestorage.BucketHandle { + return nil +} + +func (c *mockGCSClient) Download(ctx context.Context, bucket, objectPrefix string, key common.Hash) ([]byte, error) { + value, ok := c.storage[objectPrefix+EncodeStorageServiceKey(key)] + if !ok { + return nil, ErrNotFound + } + return value, nil +} + +func (c *mockGCSClient) Close(ctx context.Context) error { + return nil +} + +func (c *mockGCSClient) Upload(ctx context.Context, bucket, objectPrefix string, value []byte) error { + key := objectPrefix + EncodeStorageServiceKey(dastree.Hash(value)) + c.storage[key] = value + return nil +} + +func NewTestGoogleCloudStorageService(ctx context.Context, googleCloudStorageConfig GoogleCloudStorageServiceConfig) (StorageService, error) { + return &GoogleCloudStorageService{ + bucket: googleCloudStorageConfig.Bucket, + objectPrefix: googleCloudStorageConfig.ObjectPrefix, + operator: &mockGCSClient{ + storage: make(map[string][]byte), + }, + maxRetention: googleCloudStorageConfig.MaxRetention, + }, nil +} + +func TestNewGoogleCloudStorageService(t *testing.T) { + ctx := context.Background() + // #nosec G115 + expiry := uint64(time.Now().Add(time.Hour).Unix()) + googleCloudStorageServiceConfig := DefaultGoogleCloudStorageServiceConfig + googleCloudStorageServiceConfig.Enable = true + googleCloudStorageServiceConfig.MaxRetention = time.Hour * 24 + googleCloudService, err := NewTestGoogleCloudStorageService(ctx, googleCloudStorageServiceConfig) + Require(t, err) + + val1 := []byte("The first value") + val1CorrectKey := dastree.Hash(val1) + val2IncorrectKey := dastree.Hash(append(val1, 0)) + + _, err = googleCloudService.GetByHash(ctx, val1CorrectKey) + if !errors.Is(err, ErrNotFound) { + t.Fatal(err) + } + + err = googleCloudService.Put(ctx, val1, expiry) + Require(t, err) + + _, err = googleCloudService.GetByHash(ctx, val2IncorrectKey) + if !errors.Is(err, ErrNotFound) { + t.Fatal(err) + } + + val, err := googleCloudService.GetByHash(ctx, val1CorrectKey) + Require(t, err) + if !bytes.Equal(val, val1) { + t.Fatal(val, val1) + } + +} diff --git a/das/rpc_aggregator.go b/das/rpc_aggregator.go index 1b3e2b8f44..9cf481e015 100644 --- a/das/rpc_aggregator.go +++ b/das/rpc_aggregator.go @@ -21,7 +21,7 @@ import ( "github.com/offchainlabs/nitro/util/signature" "github.com/ethereum/go-ethereum/common" - "github.com/offchainlabs/nitro/arbutil" + "github.com/ethereum/go-ethereum/ethclient" ) type BackendConfig struct { @@ -83,7 +83,7 @@ func NewRPCAggregator(ctx context.Context, config DataAvailabilityConfig, signer return NewAggregator(ctx, config, services) } -func NewRPCAggregatorWithL1Info(config DataAvailabilityConfig, l1client arbutil.L1Interface, seqInboxAddress common.Address, signer signature.DataSignerFunc) (*Aggregator, error) { +func NewRPCAggregatorWithL1Info(config DataAvailabilityConfig, l1client *ethclient.Client, seqInboxAddress common.Address, signer signature.DataSignerFunc) (*Aggregator, error) { services, err := ParseServices(config.RPCAggregator, signer) if err != nil { return nil, err diff --git a/das/sign_after_store_das_writer.go b/das/sign_after_store_das_writer.go index 40b03847d8..b09ed091cc 100644 --- a/das/sign_after_store_das_writer.go +++ b/das/sign_after_store_das_writer.go @@ -81,6 +81,7 @@ func NewSignAfterStoreDASWriter(ctx context.Context, config DataAvailabilityConf if err != nil { return nil, err } + log.Info("DAS public key used for signing", "key", hexutil.Encode(blsSignatures.PublicKeyToBytes(publicKey))) keyset := &daprovider.DataAvailabilityKeyset{ AssumedHonest: 1, diff --git a/das/syncing_fallback_storage.go b/das/syncing_fallback_storage.go index 43ae6160d7..0670a29c73 100644 --- a/das/syncing_fallback_storage.go +++ b/das/syncing_fallback_storage.go @@ -17,6 +17,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" "github.com/offchainlabs/nitro/arbstate/daprovider" "github.com/offchainlabs/nitro/arbutil" @@ -243,7 +244,7 @@ func FindDASDataFromLog( inboxContract *bridgegen.SequencerInbox, deliveredEvent *bridgegen.SequencerInboxSequencerBatchDelivered, inboxAddr common.Address, - l1Client arbutil.L1Interface, + l1Client *ethclient.Client, batchDeliveredLog types.Log) ([]byte, error) { data := []byte{} if deliveredEvent.DataLocation == uint8(batchDataSeparateEvent) { diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index 5a1efc6d08..cb06a58e74 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -16,6 +16,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/filters" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" @@ -179,7 +180,7 @@ func CreateExecutionNode( stack *node.Node, chainDB ethdb.Database, l2BlockChain *core.BlockChain, - l1client arbutil.L1Interface, + l1client *ethclient.Client, configFetcher ConfigFetcher, ) (*ExecutionNode, error) { config := configFetcher() diff --git a/go-ethereum b/go-ethereum index 17cd001675..b068464bf5 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 17cd00167543a5a2b0b083e32820051100154c2f +Subproject commit b068464bf59ab5414f72c2d4aba855b8af5edc17 diff --git a/go.mod b/go.mod index 3826016948..488d455f44 100644 --- a/go.mod +++ b/go.mod @@ -1,21 +1,22 @@ module github.com/offchainlabs/nitro -go 1.21 +go 1.23 replace github.com/VictoriaMetrics/fastcache => ./fastcache replace github.com/ethereum/go-ethereum => ./go-ethereum require ( + cloud.google.com/go/storage v1.43.0 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible github.com/Shopify/toxiproxy v2.1.4+incompatible github.com/alicebob/miniredis/v2 v2.32.1 github.com/andybalholm/brotli v1.0.4 - github.com/aws/aws-sdk-go-v2 v1.21.2 - github.com/aws/aws-sdk-go-v2/config v1.18.45 - github.com/aws/aws-sdk-go-v2/credentials v1.13.43 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.10 - github.com/aws/aws-sdk-go-v2/service/s3 v1.26.9 + github.com/aws/aws-sdk-go-v2 v1.31.0 + github.com/aws/aws-sdk-go-v2/config v1.27.40 + github.com/aws/aws-sdk-go-v2/credentials v1.17.38 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.27 + github.com/aws/aws-sdk-go-v2/service/s3 v1.64.1 github.com/cavaliergopher/grab/v3 v3.0.1 github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 github.com/codeclysm/extract/v3 v3.0.2 @@ -29,7 +30,7 @@ require ( github.com/gobwas/ws-examples v0.0.0-20190625122829-a9e8908d9484 github.com/google/btree v1.1.2 github.com/google/go-cmp v0.6.0 - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.6.0 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/holiman/uint256 v1.2.4 github.com/knadh/koanf v1.4.0 @@ -42,18 +43,37 @@ require ( github.com/spf13/pflag v1.0.5 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/wealdtech/go-merkletree v1.0.0 - golang.org/x/crypto v0.21.0 + golang.org/x/crypto v0.24.0 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa - golang.org/x/sys v0.18.0 - golang.org/x/term v0.18.0 - golang.org/x/tools v0.16.0 + golang.org/x/sys v0.21.0 + golang.org/x/term v0.21.0 + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d + google.golang.org/api v0.187.0 gopkg.in/natefinch/lumberjack.v2 v2.0.0 ) require ( + cloud.google.com/go v0.115.0 // indirect + cloud.google.com/go/auth v0.6.1 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect + cloud.google.com/go/compute/metadata v0.3.0 // indirect + cloud.google.com/go/iam v1.1.8 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/google/go-querystring v1.1.0 // indirect - github.com/onsi/ginkgo v1.16.5 // indirect - github.com/onsi/gomega v1.18.1 // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/gax-go/v2 v2.12.5 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.opentelemetry.io/otel/trace v1.24.0 // indirect + google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d // indirect + google.golang.org/grpc v1.64.0 // indirect ) require ( @@ -62,24 +82,24 @@ require ( github.com/StackExchange/wmi v1.2.1 // indirect github.com/VictoriaMetrics/fastcache v1.12.1 // indirect github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.1 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.1 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.1 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.5 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.4 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 // indirect - github.com/aws/smithy-go v1.15.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.5 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.14 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.18 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.18 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.18 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.20 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.20 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.18 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.23.4 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.27.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.31.4 // indirect + github.com/aws/smithy-go v1.22.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.10.0 // indirect github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect - github.com/cespare/xxhash/v2 v2.3.0 + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cockroachdb/errors v1.9.1 // indirect github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect github.com/cockroachdb/redact v1.1.3 // indirect @@ -98,7 +118,6 @@ require ( github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/ethereum/c-kzg-4844 v0.4.0 // indirect - github.com/fjl/memsize v0.0.2 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/gammazero/deque v0.2.1 // indirect github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect @@ -111,9 +130,9 @@ require ( github.com/gofrs/flock v0.8.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect - github.com/golang/glog v1.0.0 // indirect - github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/glog v1.2.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect github.com/google/flatbuffers v1.12.1 // indirect github.com/google/go-github/v62 v62.0.0 @@ -126,7 +145,6 @@ require ( github.com/holiman/bloomfilter/v2 v2.0.3 // indirect github.com/huin/goupnp v1.3.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5 // indirect github.com/juju/loggo v0.0.0-20180524022052-584905176618 // indirect github.com/klauspost/compress v1.17.2 // indirect @@ -163,13 +181,13 @@ require ( github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/yuin/gopher-lua v1.1.1 // indirect - go.opencensus.io v0.22.5 // indirect - golang.org/x/mod v0.14.0 // indirect - golang.org/x/net v0.23.0 // indirect + go.opencensus.io v0.24.0 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/net v0.26.0 // indirect golang.org/x/oauth2 v0.22.0 - golang.org/x/sync v0.5.0 - golang.org/x/text v0.14.0 // indirect - golang.org/x/time v0.3.0 // indirect - google.golang.org/protobuf v1.33.0 // indirect + golang.org/x/sync v0.7.0 + golang.org/x/text v0.16.0 // indirect + golang.org/x/time v0.5.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) diff --git a/go.sum b/go.sum index 2fceedb715..d11610724e 100644 --- a/go.sum +++ b/go.sum @@ -13,14 +13,26 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.115.0 h1:CnFSK6Xo3lDYRoBKEcAtia6VSC837/ZkJuRduSFnr14= +cloud.google.com/go v0.115.0/go.mod h1:8jIM5vVgoAEoiVxQ/O4BFTfHqulPZgs/ufEzMcFMdWU= +cloud.google.com/go/auth v0.6.1 h1:T0Zw1XM5c1GlpN2HYr2s+m3vr1p2wy+8VN+Z1FKxW38= +cloud.google.com/go/auth v0.6.1/go.mod h1:eFHG7zDzbXHKmjJddFG/rBlcGp6t25SwRUiEQSlO4x4= +cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4= +cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/iam v1.1.8 h1:r7umDwhj+BQyz0ScZMp4QrGXjSTI3ZINnpgU2nlB/K0= +cloud.google.com/go/iam v1.1.8/go.mod h1:GvE6lyMmfxXauzNq8NbgJbeVQNspG+tcdL/W8QO1+zE= +cloud.google.com/go/longrunning v0.5.7 h1:WLbHekDbjK1fVFD3ibpFFVoyizlLRl73I7YKuAKilhU= +cloud.google.com/go/longrunning v0.5.7/go.mod h1:8GClkudohy1Fxm3owmBGid8W0pSgodEMwEAztp38Xng= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -30,6 +42,8 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs= +cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -70,64 +84,53 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go-v2 v1.9.2/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= -github.com/aws/aws-sdk-go-v2 v1.16.3/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU= -github.com/aws/aws-sdk-go-v2 v1.21.2 h1:+LXZ0sgo8quN9UOKXXzAWRT3FWd4NxeXWOZom9pE7GA= -github.com/aws/aws-sdk-go-v2 v1.21.2/go.mod h1:ErQhvNuEMhJjweavOYhxVkn2RUx7kQXVATHrjKtxIpM= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.1 h1:SdK4Ppk5IzLs64ZMvr6MrSficMtjY2oS0WOORXTlxwU= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.1/go.mod h1:n8Bs1ElDD2wJ9kCRTczA83gYbBmjSwZp3umc6zF4EeM= +github.com/aws/aws-sdk-go-v2 v1.31.0 h1:3V05LbxTSItI5kUqNwhJrrrY1BAXxXt0sN0l72QmG5U= +github.com/aws/aws-sdk-go-v2 v1.31.0/go.mod h1:ztolYtaEUtdpf9Wftr31CJfLVjOnD/CVRkKOOYgF8hA= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.5 h1:xDAuZTn4IMm8o1LnBZvmrL8JA1io4o3YWNXgohbf20g= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.5/go.mod h1:wYSv6iDS621sEFLfKvpPE2ugjTuGlAG7iROg0hLOkfc= github.com/aws/aws-sdk-go-v2/config v1.8.3/go.mod h1:4AEiLtAb8kLs7vgw2ZV3p2VZ1+hBavOc84hqxVNpCyw= -github.com/aws/aws-sdk-go-v2/config v1.15.5/go.mod h1:ZijHHh0xd/A+ZY53az0qzC5tT46kt4JVCePf2NX9Lk4= -github.com/aws/aws-sdk-go-v2/config v1.18.45 h1:Aka9bI7n8ysuwPeFdm77nfbyHCAKQ3z9ghB3S/38zes= -github.com/aws/aws-sdk-go-v2/config v1.18.45/go.mod h1:ZwDUgFnQgsazQTnWfeLWk5GjeqTQTL8lMkoE1UXzxdE= +github.com/aws/aws-sdk-go-v2/config v1.27.40 h1:sie4mPBGFOO+Z27+yHzvyN31G20h/bf2xb5mCbpLv2Q= +github.com/aws/aws-sdk-go-v2/config v1.27.40/go.mod h1:4KW7Aa5tNo+0VHnuLnnE1vPHtwMurlNZNS65IdcewHA= github.com/aws/aws-sdk-go-v2/credentials v1.4.3/go.mod h1:FNNC6nQZQUuyhq5aE5c7ata8o9e4ECGmS4lAXC7o1mQ= -github.com/aws/aws-sdk-go-v2/credentials v1.12.0/go.mod h1:9YWk7VW+eyKsoIL6/CljkTrNVWBSK9pkqOPUuijid4A= -github.com/aws/aws-sdk-go-v2/credentials v1.13.43 h1:LU8vo40zBlo3R7bAvBVy/ku4nxGEyZe9N8MqAeFTzF8= -github.com/aws/aws-sdk-go-v2/credentials v1.13.43/go.mod h1:zWJBz1Yf1ZtX5NGax9ZdNjhhI4rgjfgsyk6vTY1yfVg= +github.com/aws/aws-sdk-go-v2/credentials v1.17.38 h1:iM90eRhCeZtlkzCNCG1JysOzJXGYf5rx80aD1lUgNDU= +github.com/aws/aws-sdk-go-v2/credentials v1.17.38/go.mod h1:TCVYPZeQuLaYNEkf/TVn6k5k/zdVZZ7xH9po548VNNg= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.6.0/go.mod h1:gqlclDEZp4aqJOancXK6TN24aKhT0W0Ae9MHk3wzTMM= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.4/go.mod h1:u/s5/Z+ohUQOPXl00m2yJVyioWDECsbpXTQlaqSlufc= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 h1:PIktER+hwIG286DqXyvVENjgLTAwGgoeriLDD5C+YlQ= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13/go.mod h1:f/Ib/qYjhV2/qdsf79H3QP/eRE4AkVyEf6sk7XfZ1tg= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.10 h1:JL7cY85hyjlgfA29MMyAlItX+JYIH9XsxgMBS7jtlqA= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.10/go.mod h1:p+ul5bLZSDRRXCZ/vePvfmZBH9akozXBJA5oMshWa5U= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.10/go.mod h1:F+EZtuIwjlv35kRJPyBGcsA4f7bnSoz15zOQ2lJq1Z4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 h1:nFBQlGtkbPzp/NjZLuFxRqmT91rLJkgvsEQs68h962Y= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43/go.mod h1:auo+PiyLl0n1l8A0e8RIeR8tOzYPfZZH/JNlrJ8igTQ= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.4/go.mod h1:8glyUqVIM4AmeenIsPo0oVh3+NUwnsQml2OFupfQW+0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 h1:JRVhO25+r3ar2mKGP7E0LDl8K9/G36gjlqca5iQbaqc= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37/go.mod h1:Qe+2KtKml+FEsQF/DHmDV+xjtche/hwoF75EG4UlHW8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.14 h1:C/d03NAmh8C4BZXhuRNboF/DqhBkBCeDiJDcaqIT5pA= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.14/go.mod h1:7I0Ju7p9mCIdlrfS+JCgqcYD0VXz/N4yozsox+0o078= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.27 h1:1oLpQSTuqbizOUEYdxAwH+Eveg+FOCOkg84Yijba6Kc= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.27/go.mod h1:afo0vF9P3pjy1ny+cb45lzBjtKeEb5t5MPRxeTXpujw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.18 h1:kYQ3H1u0ANr9KEKlGs/jTLrBFPo8P8NaH/w7A01NeeM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.18/go.mod h1:r506HmK5JDUh9+Mw4CfGJGSSoqIiLCndAuqXuhbv67Y= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.18 h1:Z7IdFUONvTcvS7YuhtVxN99v2cCoHRXOS4mTr0B/pUc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.18/go.mod h1:DkKMmksZVVyat+Y+r1dEOgJEfUeA7UngIHWeKsi0yNc= github.com/aws/aws-sdk-go-v2/internal/ini v1.2.4/go.mod h1:ZcBrrI3zBKlhGFNYWvju0I3TR93I7YIgAfy82Fh4lcQ= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.11/go.mod h1:0MR+sS1b/yxsfAPvAESrw8NfwUoxMinDyw6EYR9BS2U= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 h1:hze8YsjSh8Wl1rYa1CJpRmXP21BvOBuc76YhW0HsuQ4= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45/go.mod h1:lD5M20o09/LCuQ2mE62Mb/iSdSlCNuj6H5ci7tW7OsE= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.1 h1:C21IDZCm9Yu5xqjb3fKmxDoYvJXtw1DNlOmLZEIlY1M= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.1/go.mod h1:l/BbcfqDCT3hePawhy4ZRtewjtdkl6GWtd9/U+1penQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.18 h1:OWYvKL53l1rbsUmW7bQyJVsYU/Ii3bbAAQIIFNbM0Tk= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.18/go.mod h1:CUx0G1v3wG6l01tUB+j7Y8kclA8NSqK4ef0YG79a4cg= github.com/aws/aws-sdk-go-v2/service/appconfig v1.4.2/go.mod h1:FZ3HkCe+b10uFZZkFdvf98LHW21k49W8o8J366lqVKY= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.1 h1:T4pFel53bkHjL2mMo+4DKE6r6AuoZnM0fg7k1/ratr4= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.1/go.mod h1:GeUru+8VzrTXV/83XyMJ80KpH8xO89VPoUileyNQ+tc= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.5 h1:9LSZqt4v1JiehyZTrQnRFf2mY/awmyYNNY/b7zqtduU= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.5/go.mod h1:S8TVP66AAkMMdYYCNZGvrdEq9YRm+qLXjio4FqRnrEE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.5 h1:QFASJGfT8wMXtuP3D5CRmMjARHv9ZmzFUMJznHDOY3w= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.5/go.mod h1:QdZ3OmoIjSX+8D1OPAzPxDfjXASbBMDsz9qvtyIhtik= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.20 h1:rTWjG6AvWekO2B1LHeM3ktU7MqyX9rzWQ7hgzneZW7E= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.20/go.mod h1:RGW2DDpVc8hu6Y6yG8G5CHVmVOAn1oV8rNKOHRJyswg= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.2/go.mod h1:72HRZDLMtmVQiLG2tLfQcaWLCssELvGl+Zf2WVxMmR8= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.4/go.mod h1:uKkN7qmSIsNJVyMtxNQoCEYMvFEXbOg9fwCJPdfp2u8= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37 h1:WWZA/I2K4ptBS1kg0kV1JbBtG/umed0vwHRrmcr9z7k= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37/go.mod h1:vBmDnwWXWxNPFRMmG2m/3MKOe+xEcMDo1tanpaWCcck= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.4 h1:RE/DlZLYrz1OOmq8F28IXHLksuuvlpzUbvJ+SESCZBI= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.4/go.mod h1:oudbsSdDtazNj47z1ut1n37re9hDsKpk2ZI3v7KSxq0= -github.com/aws/aws-sdk-go-v2/service/s3 v1.26.9 h1:LCQKnopq2t4oQS3VKivlYTzAHCTJZZoQICM9fny7KHY= -github.com/aws/aws-sdk-go-v2/service/s3 v1.26.9/go.mod h1:iMYipLPXlWpBJ0KFX7QJHZ84rBydHBY8as2aQICTPWk= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.20 h1:Xbwbmk44URTiHNx6PNo0ujDE6ERlsCKJD3u1zfnzAPg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.20/go.mod h1:oAfOFzUB14ltPZj1rWwRc3d/6OgD76R8KlvU3EqM9Fg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.18 h1:eb+tFOIl9ZsUe2259/BKPeniKuz4/02zZFH/i4Nf8Rg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.18/go.mod h1:GVCC2IJNJTmdlyEsSmofEy7EfJncP7DNnXDzRjJ5Keg= +github.com/aws/aws-sdk-go-v2/service/s3 v1.64.1 h1:jjHf+M6vCp/WzbyFEroY4/Nx8dJac520A0EPwlYk0Do= +github.com/aws/aws-sdk-go-v2/service/s3 v1.64.1/go.mod h1:NLTqRLe3pUNu3nTEHI6XlHLKYmc8fbHUdMxAB6+s41Q= github.com/aws/aws-sdk-go-v2/service/sso v1.4.2/go.mod h1:NBvT9R1MEF+Ud6ApJKM0G+IkPchKS7p7c2YPKwHmBOk= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.4/go.mod h1:cPDwJwsP4Kff9mldCXAmddjJL6JGQqtA3Mzer2zyr88= -github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 h1:JuPGc7IkOP4AaqcZSIcyqLpFSqBWK32rM9+a1g6u73k= -github.com/aws/aws-sdk-go-v2/service/sso v1.15.2/go.mod h1:gsL4keucRCgW+xA85ALBpRFfdSLH4kHOVSnLMSuBECo= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3 h1:HFiiRkf1SdaAmV3/BHOFZ9DjFynPHj8G/UIO1lQS+fk= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3/go.mod h1:a7bHA82fyUXOm+ZSWKU6PIoBxrjSprdLoM8xPYvzYVg= +github.com/aws/aws-sdk-go-v2/service/sso v1.23.4 h1:ck/Y8XWNR1gHa4BFkwE3oSu7XDJGwl+8TI7E/RB2EcQ= +github.com/aws/aws-sdk-go-v2/service/sso v1.23.4/go.mod h1:XRlMvmad0ZNL+75C5FYdMvbbLkd6qiqz6foR1nA1PXY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.27.4 h1:4f2/JKYZHAZbQ7koBpZ012bKi32NHPY0m7TDuJgsbug= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.27.4/go.mod h1:FnvDM4sfa+isJ3kDXIzAB9GAwVSzFzSy97uZ3IsHo4E= github.com/aws/aws-sdk-go-v2/service/sts v1.7.2/go.mod h1:8EzeIqfWt2wWT4rJVu3f21TfrhJ8AEMzVybRNSb/b4g= -github.com/aws/aws-sdk-go-v2/service/sts v1.16.4/go.mod h1:lfSYenAXtavyX2A1LsViglqlG9eEFYxNryTZS5rn3QE= -github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 h1:0BkLfgeDjfZnZ+MhB3ONb01u9pwFYTCZVhlsSSBvlbU= -github.com/aws/aws-sdk-go-v2/service/sts v1.23.2/go.mod h1:Eows6e1uQEsc4ZaHANmsPRzAKcVDrcmjjWiih2+HUUQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.31.4 h1:uK6dUUdJtqutK1XO/tmNaQMJiPLCJY/eAeOOmqQ6ygY= +github.com/aws/aws-sdk-go-v2/service/sts v1.31.4/go.mod h1:yMWe0F+XG0DkRZK5ODZhG7BEFYhLXi2dqGsv6tX0cgI= github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= -github.com/aws/smithy-go v1.11.2/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM= -github.com/aws/smithy-go v1.15.0 h1:PS/durmlzvAFpQHDs4wi4sNNP9ExsqZh6IlfdHXgKK8= -github.com/aws/smithy-go v1.15.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM= +github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -237,8 +240,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= -github.com/fjl/memsize v0.0.2 h1:27txuSD9or+NZlnOWdKUxeBzTAUkWCVh+4Gf2dWFOzA= -github.com/fjl/memsize v0.0.2/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -273,6 +276,11 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= @@ -280,7 +288,6 @@ github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= @@ -306,12 +313,13 @@ github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzq github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= -github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= +github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -335,8 +343,8 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= @@ -356,11 +364,10 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github/v62 v62.0.0 h1:/6mGCaRywZz9MuHyw9gD1CwsbmBX8GWsbFkwMmHdhl4= @@ -371,8 +378,11 @@ github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -380,17 +390,22 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b h1:RMpPgZTSApbPf7xaVel+QkoGPRLFLrwFO89uDUHEGf0= github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.12.5 h1:8gw9KZK8TiVKB6q3zHY3SBzLnrGp6HQjyfYBYGmXdxA= +github.com/googleapis/gax-go/v2 v2.12.5/go.mod h1:BUDKcWo+RaKq5SC9vVYL0wLADa3VcfswbOMMRmB9H3E= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= @@ -437,7 +452,6 @@ github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/hydrogen18/memlistener v0.0.0-20200120041712-dcc25e7acd91/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= @@ -448,9 +462,7 @@ github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0Gqw github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= @@ -572,25 +584,19 @@ github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxzi github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/npillmayer/nestext v0.1.3/go.mod h1:h2lrijH8jpicr25dFY+oAJLyzlya6jhnuG+zWp9L0Uk= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= -github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= @@ -680,14 +686,19 @@ github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobt github.com/status-im/keycard-go v0.2.0/go.mod h1:wlp8ZLbsmrF6g6WjugPAx+IzoLrkdf9+mHxBEeo3Hbg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= @@ -739,8 +750,20 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= +go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= golang.org/x/crypto v0.0.0-20180214000028-650f4a345ab4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -753,8 +776,8 @@ golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -791,8 +814,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180406214816-61147c48b25b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -829,17 +852,17 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -862,8 +885,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -913,7 +936,6 @@ golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -940,14 +962,14 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -959,14 +981,15 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1011,16 +1034,16 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM= -golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -1038,6 +1061,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.187.0 h1:Mxs7VATVC2v7CY+7Xwm4ndkX71hpElcvx0D1Ji/p1eo= +google.golang.org/api v0.187.0/go.mod h1:KIHlTc4x7N7gKKuVsdmfBXN13yEEWXWFURWY6SBp2gk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1076,6 +1101,12 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d h1:PksQg4dV6Sem3/HkBX+Ltq8T0ke0PKIRBNBatoDTVls= +google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d/go.mod h1:s7iA721uChleev562UJO2OYB0PPT9CMFjV+Ce7VJH5M= +google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 h1:MuYw1wJzT+ZkybKfaOXKp5hJiZDn2iHaXRw0mRYdHSc= +google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4/go.mod h1:px9SlOOZBg1wM1zdnr8jEL4CNGUBZ+ZKYtNPApNQc4c= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d h1:k3zyW3BYYR30e8v3x0bTDdE9vpYFjZHK+HcyqkrppWk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -1091,7 +1122,10 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1104,8 +1138,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pubsub/producer.go b/pubsub/producer.go index 5c87f4f722..dacaeba7d0 100644 --- a/pubsub/producer.go +++ b/pubsub/producer.go @@ -201,6 +201,7 @@ func (p *Producer[Request, Response]) clearMessages(ctx context.Context) time.Du } if _, err := p.client.XDel(ctx, p.redisStream, pelData.Lower).Result(); err != nil { log.Error("error deleting PEL's lower message thats past its TTL", "msgID", pelData.Lower, "err", err) + return 0 } } } diff --git a/staker/block_validator.go b/staker/block_validator.go index e1b2c75b84..5a1f123693 100644 --- a/staker/block_validator.go +++ b/staker/block_validator.go @@ -29,6 +29,8 @@ import ( "github.com/offchainlabs/nitro/util/stopwaiter" "github.com/offchainlabs/nitro/validator" "github.com/offchainlabs/nitro/validator/client/redis" + "github.com/offchainlabs/nitro/validator/inputs" + "github.com/offchainlabs/nitro/validator/server_api" "github.com/spf13/pflag" ) @@ -94,6 +96,9 @@ type BlockValidator struct { // for testing only testingProgressMadeChan chan struct{} + // For troubleshooting failed validations + validationInputsWriter *inputs.Writer + fatalErr chan<- error MemoryFreeLimitChecker resourcemanager.LimitChecker @@ -115,6 +120,9 @@ type BlockValidatorConfig struct { Dangerous BlockValidatorDangerousConfig `koanf:"dangerous"` MemoryFreeLimit string `koanf:"memory-free-limit" reload:"hot"` ValidationServerConfigsList string `koanf:"validation-server-configs-list"` + // The directory to which the BlockValidator will write the + // block_inputs_.json files when WriteToFile() is called. + BlockInputsFilePath string `koanf:"block-inputs-file-path"` memoryFreeLimit int } @@ -182,6 +190,7 @@ func BlockValidatorConfigAddOptions(prefix string, f *pflag.FlagSet) { f.Bool(prefix+".failure-is-fatal", DefaultBlockValidatorConfig.FailureIsFatal, "failing a validation is treated as a fatal error") BlockValidatorDangerousConfigAddOptions(prefix+".dangerous", f) f.String(prefix+".memory-free-limit", DefaultBlockValidatorConfig.MemoryFreeLimit, "minimum free-memory limit after reaching which the blockvalidator pauses validation. Enabled by default as 1GB, to disable provide empty string") + f.String(prefix+".block-inputs-file-path", DefaultBlockValidatorConfig.BlockInputsFilePath, "directory to write block validation inputs files") } func BlockValidatorDangerousConfigAddOptions(prefix string, f *pflag.FlagSet) { @@ -201,6 +210,7 @@ var DefaultBlockValidatorConfig = BlockValidatorConfig{ PendingUpgradeModuleRoot: "latest", FailureIsFatal: true, Dangerous: DefaultBlockValidatorDangerousConfig, + BlockInputsFilePath: "./target/validation_inputs", MemoryFreeLimit: "default", RecordingIterLimit: 20, } @@ -219,6 +229,7 @@ var TestBlockValidatorConfig = BlockValidatorConfig{ PendingUpgradeModuleRoot: "latest", FailureIsFatal: true, Dangerous: DefaultBlockValidatorDangerousConfig, + BlockInputsFilePath: "./target/validation_inputs", MemoryFreeLimit: "default", } @@ -277,6 +288,13 @@ func NewBlockValidator( fatalErr: fatalErr, prevBatchCache: make(map[uint64][]byte), } + valInputsWriter, err := inputs.NewWriter( + inputs.WithBaseDir(ret.stack.InstanceDir()), + inputs.WithSlug("BlockValidator")) + if err != nil { + return nil, err + } + ret.validationInputsWriter = valInputsWriter if !config().Dangerous.ResetBlockValidation { validated, err := ret.ReadLastValidatedInfo() if err != nil { @@ -508,18 +526,16 @@ func (v *BlockValidator) sendRecord(s *validationStatus) error { } //nolint:gosec -func (v *BlockValidator) writeToFile(validationEntry *validationEntry, moduleRoot common.Hash) error { +func (v *BlockValidator) writeToFile(validationEntry *validationEntry) error { input, err := validationEntry.ToInput([]ethdb.WasmTarget{rawdb.TargetWavm}) if err != nil { return err } - for _, spawner := range v.execSpawners { - if validator.SpawnerSupportsModule(spawner, moduleRoot) { - _, err = spawner.WriteToFile(input, validationEntry.End, moduleRoot).Await(v.GetContext()) - return err - } + inputJson := server_api.ValidationInputToJson(input) + if err := v.validationInputsWriter.Write(inputJson); err != nil { + return err } - return errors.New("did not find exec spawner for wasmModuleRoot") + return nil } func (v *BlockValidator) SetCurrentWasmModuleRoot(hash common.Hash) error { @@ -823,7 +839,7 @@ validationsLoop: runEnd, err := run.Current() if err == nil && runEnd != validationStatus.Entry.End { err = fmt.Errorf("validation failed: expected %v got %v", validationStatus.Entry.End, runEnd) - writeErr := v.writeToFile(validationStatus.Entry, run.WasmModuleRoot()) + writeErr := v.writeToFile(validationStatus.Entry) if writeErr != nil { log.Warn("failed to write debug results file", "err", writeErr) } diff --git a/staker/l1_validator.go b/staker/l1_validator.go index 6ea9fd8ded..5b0c211324 100644 --- a/staker/l1_validator.go +++ b/staker/l1_validator.go @@ -19,6 +19,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/solgen/go/rollupgen" @@ -45,7 +46,7 @@ type L1Validator struct { rollup *RollupWatcher rollupAddress common.Address validatorUtils *rollupgen.ValidatorUtils - client arbutil.L1Interface + client *ethclient.Client builder *txbuilder.Builder wallet ValidatorWalletInterface callOpts bind.CallOpts @@ -57,7 +58,7 @@ type L1Validator struct { } func NewL1Validator( - client arbutil.L1Interface, + client *ethclient.Client, wallet ValidatorWalletInterface, validatorUtilsAddress common.Address, callOpts bind.CallOpts, diff --git a/staker/rollup_watcher.go b/staker/rollup_watcher.go index 5ef28a49dc..4d7db52322 100644 --- a/staker/rollup_watcher.go +++ b/staker/rollup_watcher.go @@ -4,16 +4,19 @@ package staker import ( + "bytes" "context" "encoding/binary" "errors" "fmt" "math/big" + "strings" "sync/atomic" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rpc" "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/solgen/go/rollupgen" "github.com/offchainlabs/nitro/util/headerreader" @@ -48,12 +51,19 @@ type RollupWatcher struct { *rollupgen.RollupUserLogic address common.Address fromBlock *big.Int - client arbutil.L1Interface + client RollupWatcherL1Interface baseCallOpts bind.CallOpts unSupportedL3Method atomic.Bool + supportedL3Method atomic.Bool } -func NewRollupWatcher(address common.Address, client arbutil.L1Interface, callOpts bind.CallOpts) (*RollupWatcher, error) { +type RollupWatcherL1Interface interface { + bind.ContractBackend + HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) + FilterLogs(ctx context.Context, q ethereum.FilterQuery) ([]types.Log, error) +} + +func NewRollupWatcher(address common.Address, client RollupWatcherL1Interface, callOpts bind.CallOpts) (*RollupWatcher, error) { con, err := rollupgen.NewRollupUserLogic(address, client) if err != nil { return nil, err @@ -73,15 +83,41 @@ func (r *RollupWatcher) getCallOpts(ctx context.Context) *bind.CallOpts { return &opts } +const noNodeErr string = "NO_NODE" + +func looksLikeNoNodeError(err error) bool { + if err == nil { + return false + } + if strings.Contains(err.Error(), noNodeErr) { + return true + } + var errWithData rpc.DataError + ok := errors.As(err, &errWithData) + if !ok { + return false + } + dataString, ok := errWithData.ErrorData().(string) + if !ok { + return false + } + data := common.FromHex(dataString) + return bytes.Contains(data, []byte(noNodeErr)) +} + func (r *RollupWatcher) getNodeCreationBlock(ctx context.Context, nodeNum uint64) (*big.Int, error) { callOpts := r.getCallOpts(ctx) if !r.unSupportedL3Method.Load() { createdAtBlock, err := r.GetNodeCreationBlockForLogLookup(callOpts, nodeNum) if err == nil { + r.supportedL3Method.Store(true) return createdAtBlock, nil } - log.Trace("failed to call getNodeCreationBlockForLogLookup, falling back on node CreatedAtBlock field", "err", err) - if headerreader.ExecutionRevertedRegexp.MatchString(err.Error()) { + if headerreader.ExecutionRevertedRegexp.MatchString(err.Error()) && !looksLikeNoNodeError(err) { + if r.supportedL3Method.Load() { + return nil, fmt.Errorf("getNodeCreationBlockForLogLookup failed despite previously succeeding: %w", err) + } + log.Info("getNodeCreationBlockForLogLookup does not seem to exist, falling back on node CreatedAtBlock field", "err", err) r.unSupportedL3Method.Store(true) } else { return nil, err diff --git a/staker/staker.go b/staker/staker.go index 77ca93e02c..45e6f6f551 100644 --- a/staker/staker.go +++ b/staker/staker.go @@ -15,6 +15,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rpc" @@ -280,7 +281,7 @@ type ValidatorWalletInterface interface { TxSenderAddress() *common.Address RollupAddress() common.Address ChallengeManagerAddress() common.Address - L1Client() arbutil.L1Interface + L1Client() *ethclient.Client TestTransactions(context.Context, []*types.Transaction) error ExecuteTransactions(context.Context, *txbuilder.Builder, common.Address) (*types.Transaction, error) TimeoutChallenges(context.Context, []uint64) (*types.Transaction, error) @@ -304,7 +305,6 @@ func NewStaker( validatorUtilsAddress common.Address, fatalErr chan<- error, ) (*Staker, error) { - if err := config().Validate(); err != nil { return nil, err } @@ -511,7 +511,9 @@ func (s *Staker) Start(ctxIn context.Context) { } s.StopWaiter.Start(ctxIn, s) backoff := time.Second - ephemeralErrorHandler := util.NewEphemeralErrorHandler(10*time.Minute, "is ahead of on-chain nonce", 0) + isAheadOfOnChainNonceEphemeralErrorHandler := util.NewEphemeralErrorHandler(10*time.Minute, "is ahead of on-chain nonce", 0) + exceedsMaxMempoolSizeEphemeralErrorHandler := util.NewEphemeralErrorHandler(10*time.Minute, dataposter.ErrExceedsMaxMempoolSize.Error(), 0) + blockValidationPendingEphemeralErrorHandler := util.NewEphemeralErrorHandler(10*time.Minute, "block validation is still pending", 0) s.CallIteratively(func(ctx context.Context) (returningWait time.Duration) { defer func() { panicErr := recover() @@ -545,7 +547,9 @@ func (s *Staker) Start(ctxIn context.Context) { } } if err == nil { - ephemeralErrorHandler.Reset() + isAheadOfOnChainNonceEphemeralErrorHandler.Reset() + exceedsMaxMempoolSizeEphemeralErrorHandler.Reset() + blockValidationPendingEphemeralErrorHandler.Reset() backoff = time.Second stakerLastSuccessfulActionGauge.Update(time.Now().Unix()) stakerActionSuccessCounter.Inc(1) @@ -563,7 +567,9 @@ func (s *Staker) Start(ctxIn context.Context) { } else { logLevel = log.Warn } - logLevel = ephemeralErrorHandler.LogLevel(err, logLevel) + logLevel = isAheadOfOnChainNonceEphemeralErrorHandler.LogLevel(err, logLevel) + logLevel = exceedsMaxMempoolSizeEphemeralErrorHandler.LogLevel(err, logLevel) + logLevel = blockValidationPendingEphemeralErrorHandler.LogLevel(err, logLevel) logLevel("error acting as staker", "err", err) return backoff }) @@ -1220,7 +1226,7 @@ func (s *Staker) updateStakerBalanceMetric(ctx context.Context) { } balance, err := s.client.BalanceAt(ctx, *txSenderAddress, nil) if err != nil { - log.Error("error getting staker balance", "txSenderAddress", *txSenderAddress, "err", err) + log.Warn("error getting staker balance", "txSenderAddress", *txSenderAddress, "err", err) return } stakerBalanceGauge.Update(arbmath.BalancePerEther(balance)) diff --git a/staker/stateless_block_validator.go b/staker/stateless_block_validator.go index 60306d712f..9257c5582a 100644 --- a/staker/stateless_block_validator.go +++ b/staker/stateless_block_validator.go @@ -23,6 +23,7 @@ import ( "github.com/offchainlabs/nitro/util/rpcclient" "github.com/offchainlabs/nitro/validator" "github.com/offchainlabs/nitro/validator/client/redis" + "github.com/offchainlabs/nitro/validator/server_api" validatorclient "github.com/offchainlabs/nitro/validator/client" ) @@ -40,6 +41,7 @@ type StatelessBlockValidator struct { streamer TransactionStreamerInterface db ethdb.Database dapReaders []daprovider.Reader + stack *node.Node } type BlockValidatorRegistrer interface { @@ -264,6 +266,7 @@ func NewStatelessBlockValidator( db: arbdb, dapReaders: dapReaders, execSpawners: executionSpawners, + stack: stack, }, nil } @@ -508,6 +511,18 @@ func (v *StatelessBlockValidator) ValidateResult( return true, &entry.End, nil } +func (v *StatelessBlockValidator) ValidationInputsAt(ctx context.Context, pos arbutil.MessageIndex, target ethdb.WasmTarget) (server_api.InputJSON, error) { + entry, err := v.CreateReadyValidationEntry(ctx, pos) + if err != nil { + return server_api.InputJSON{}, err + } + input, err := entry.ToInput([]ethdb.WasmTarget{target}) + if err != nil { + return server_api.InputJSON{}, err + } + return *server_api.ValidationInputToJson(input), nil +} + func (v *StatelessBlockValidator) OverrideRecorder(t *testing.T, recorder execution.ExecutionRecorder) { v.recorder = recorder } diff --git a/staker/txbuilder/builder.go b/staker/txbuilder/builder.go index 9a5e9df2b5..f52b03a781 100644 --- a/staker/txbuilder/builder.go +++ b/staker/txbuilder/builder.go @@ -12,13 +12,13 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "github.com/offchainlabs/nitro/arbutil" + "github.com/ethereum/go-ethereum/ethclient" ) type ValidatorWalletInterface interface { // Address must be able to be called concurrently with other functions Address() *common.Address - L1Client() arbutil.L1Interface + L1Client() *ethclient.Client TestTransactions(context.Context, []*types.Transaction) error ExecuteTransactions(context.Context, *Builder, common.Address) (*types.Transaction, error) AuthIfEoa() *bind.TransactOpts @@ -27,10 +27,10 @@ type ValidatorWalletInterface interface { // Builder combines any transactions sent to it via SendTransaction into one batch, // which is then sent to the validator wallet. // This lets the validator make multiple atomic transactions. -// This inherits from an eth client so it can be used as an L1Interface, -// where it transparently intercepts calls to SendTransaction and queues them for the next batch. +// This inherits from an ethclient.Client so it can be used to transparently +// intercept calls to SendTransaction and queue them for the next batch. type Builder struct { - arbutil.L1Interface + *ethclient.Client transactions []*types.Transaction builderAuth *bind.TransactOpts isAuthFake bool @@ -55,7 +55,7 @@ func NewBuilder(wallet ValidatorWalletInterface) (*Builder, error) { return &Builder{ builderAuth: builderAuth, wallet: wallet, - L1Interface: wallet.L1Client(), + Client: wallet.L1Client(), isAuthFake: isAuthFake, }, nil } @@ -70,7 +70,7 @@ func (b *Builder) ClearTransactions() { func (b *Builder) EstimateGas(ctx context.Context, call ethereum.CallMsg) (gas uint64, err error) { if len(b.transactions) == 0 && !b.isAuthFake { - return b.L1Interface.EstimateGas(ctx, call) + return b.Client.EstimateGas(ctx, call) } return 0, nil } diff --git a/staker/validatorwallet/contract.go b/staker/validatorwallet/contract.go index 6346029c3a..3202d58569 100644 --- a/staker/validatorwallet/contract.go +++ b/staker/validatorwallet/contract.go @@ -16,10 +16,10 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/offchainlabs/nitro/arbnode/dataposter" - "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/solgen/go/rollupgen" "github.com/offchainlabs/nitro/staker/txbuilder" "github.com/offchainlabs/nitro/util/arbmath" @@ -384,7 +384,7 @@ func (v *Contract) TimeoutChallenges(ctx context.Context, challenges []uint64) ( return v.dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), *v.Address(), data, gas, auth.Value) } -func (v *Contract) L1Client() arbutil.L1Interface { +func (v *Contract) L1Client() *ethclient.Client { return v.l1Reader.Client() } diff --git a/staker/validatorwallet/eoa.go b/staker/validatorwallet/eoa.go index 3ae305b36c..7c7f472579 100644 --- a/staker/validatorwallet/eoa.go +++ b/staker/validatorwallet/eoa.go @@ -10,8 +10,8 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" "github.com/offchainlabs/nitro/arbnode/dataposter" - "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/solgen/go/challengegen" "github.com/offchainlabs/nitro/solgen/go/rollupgen" "github.com/offchainlabs/nitro/staker/txbuilder" @@ -19,7 +19,7 @@ import ( type EOA struct { auth *bind.TransactOpts - client arbutil.L1Interface + client *ethclient.Client rollupAddress common.Address challengeManager *challengegen.ChallengeManager challengeManagerAddress common.Address @@ -27,7 +27,7 @@ type EOA struct { getExtraGas func() uint64 } -func NewEOA(dataPoster *dataposter.DataPoster, rollupAddress common.Address, l1Client arbutil.L1Interface, getExtraGas func() uint64) (*EOA, error) { +func NewEOA(dataPoster *dataposter.DataPoster, rollupAddress common.Address, l1Client *ethclient.Client, getExtraGas func() uint64) (*EOA, error) { return &EOA{ auth: dataPoster.Auth(), client: l1Client, @@ -63,7 +63,7 @@ func (w *EOA) TxSenderAddress() *common.Address { return &w.auth.From } -func (w *EOA) L1Client() arbutil.L1Interface { +func (w *EOA) L1Client() *ethclient.Client { return w.client } diff --git a/staker/validatorwallet/noop.go b/staker/validatorwallet/noop.go index b050ebe861..fec39ac2b1 100644 --- a/staker/validatorwallet/noop.go +++ b/staker/validatorwallet/noop.go @@ -10,18 +10,18 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" "github.com/offchainlabs/nitro/arbnode/dataposter" - "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/staker/txbuilder" ) // NoOp validator wallet is used for watchtower mode. type NoOp struct { - l1Client arbutil.L1Interface + l1Client *ethclient.Client rollupAddress common.Address } -func NewNoOp(l1Client arbutil.L1Interface, rollupAddress common.Address) *NoOp { +func NewNoOp(l1Client *ethclient.Client, rollupAddress common.Address) *NoOp { return &NoOp{ l1Client: l1Client, rollupAddress: rollupAddress, @@ -46,7 +46,7 @@ func (*NoOp) TimeoutChallenges(ctx context.Context, challenges []uint64) (*types return nil, errors.New("no op validator wallet cannot timeout challenges") } -func (n *NoOp) L1Client() arbutil.L1Interface { return n.l1Client } +func (n *NoOp) L1Client() *ethclient.Client { return n.l1Client } func (n *NoOp) RollupAddress() common.Address { return n.rollupAddress } diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 7304e8c2e0..0a0bc895a6 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -24,6 +24,7 @@ import ( "github.com/offchainlabs/nitro/arbos/arbostypes" "github.com/offchainlabs/nitro/arbos/util" "github.com/offchainlabs/nitro/arbstate/daprovider" + "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/blsSignatures" "github.com/offchainlabs/nitro/cmd/chaininfo" "github.com/offchainlabs/nitro/cmd/conf" @@ -35,6 +36,7 @@ import ( "github.com/offchainlabs/nitro/util/headerreader" "github.com/offchainlabs/nitro/util/redisutil" "github.com/offchainlabs/nitro/util/signature" + "github.com/offchainlabs/nitro/validator/inputs" "github.com/offchainlabs/nitro/validator/server_api" "github.com/offchainlabs/nitro/validator/server_common" "github.com/offchainlabs/nitro/validator/valnode" @@ -69,7 +71,6 @@ import ( "github.com/ethereum/go-ethereum/rpc" "github.com/offchainlabs/nitro/arbnode" - "github.com/offchainlabs/nitro/arbutil" _ "github.com/offchainlabs/nitro/execution/nodeInterface" "github.com/offchainlabs/nitro/solgen/go/bridgegen" "github.com/offchainlabs/nitro/solgen/go/mocksgen" @@ -83,7 +84,6 @@ import ( ) type info = *BlockchainTestInfo -type client = arbutil.L1Interface type SecondNodeParams struct { nodeConfig *arbnode.Config @@ -138,8 +138,8 @@ func (tc *TestClient) GetBaseFeeAt(t *testing.T, blockNum *big.Int) *big.Int { return GetBaseFeeAt(t, tc.Client, tc.ctx, blockNum) } -func (tc *TestClient) SendWaitTestTransactions(t *testing.T, txs []*types.Transaction) { - SendWaitTestTransactions(t, tc.ctx, tc.Client, txs) +func (tc *TestClient) SendWaitTestTransactions(t *testing.T, txs []*types.Transaction) []*types.Receipt { + return SendWaitTestTransactions(t, tc.ctx, tc.Client, txs) } func (tc *TestClient) DeploySimple(t *testing.T, auth bind.TransactOpts) (common.Address, *mocksgen.Simple) { @@ -763,26 +763,29 @@ func (b *NodeBuilder) BridgeBalance(t *testing.T, account string, amount *big.In return BridgeBalance(t, account, amount, b.L1Info, b.L2Info, b.L1.Client, b.L2.Client, b.ctx) } -func SendWaitTestTransactions(t *testing.T, ctx context.Context, client client, txs []*types.Transaction) { +func SendWaitTestTransactions(t *testing.T, ctx context.Context, client *ethclient.Client, txs []*types.Transaction) []*types.Receipt { t.Helper() + receipts := make([]*types.Receipt, len(txs)) for _, tx := range txs { Require(t, client.SendTransaction(ctx, tx)) } - for _, tx := range txs { - _, err := EnsureTxSucceeded(ctx, client, tx) + for i, tx := range txs { + var err error + receipts[i], err = EnsureTxSucceeded(ctx, client, tx) Require(t, err) } + return receipts } func TransferBalance( - t *testing.T, from, to string, amount *big.Int, l2info info, client client, ctx context.Context, + t *testing.T, from, to string, amount *big.Int, l2info info, client *ethclient.Client, ctx context.Context, ) (*types.Transaction, *types.Receipt) { t.Helper() return TransferBalanceTo(t, from, l2info.GetAddress(to), amount, l2info, client, ctx) } func TransferBalanceTo( - t *testing.T, from string, to common.Address, amount *big.Int, l2info info, client client, ctx context.Context, + t *testing.T, from string, to common.Address, amount *big.Int, l2info info, client *ethclient.Client, ctx context.Context, ) (*types.Transaction, *types.Receipt) { t.Helper() tx := l2info.PrepareTxTo(from, &to, l2info.TransferGas, amount, nil) @@ -795,7 +798,7 @@ func TransferBalanceTo( // if l2client is not nil - will wait until balance appears in l2 func BridgeBalance( - t *testing.T, account string, amount *big.Int, l1info info, l2info info, l1client client, l2client client, ctx context.Context, + t *testing.T, account string, amount *big.Int, l1info info, l2info info, l1client *ethclient.Client, l2client *ethclient.Client, ctx context.Context, ) (*types.Transaction, *types.Receipt) { t.Helper() @@ -855,8 +858,8 @@ func SendSignedTxesInBatchViaL1( t *testing.T, ctx context.Context, l1info *BlockchainTestInfo, - l1client arbutil.L1Interface, - l2client arbutil.L1Interface, + l1client *ethclient.Client, + l2client *ethclient.Client, delayedTxes types.Transactions, ) types.Receipts { delayedInboxContract, err := bridgegen.NewInbox(l1info.GetAddress("Inbox"), l1client) @@ -906,8 +909,8 @@ func SendSignedTxViaL1( t *testing.T, ctx context.Context, l1info *BlockchainTestInfo, - l1client arbutil.L1Interface, - l2client arbutil.L1Interface, + l1client *ethclient.Client, + l2client *ethclient.Client, delayedTx *types.Transaction, ) *types.Receipt { delayedInboxContract, err := bridgegen.NewInbox(l1info.GetAddress("Inbox"), l1client) @@ -937,8 +940,8 @@ func SendUnsignedTxViaL1( t *testing.T, ctx context.Context, l1info *BlockchainTestInfo, - l1client arbutil.L1Interface, - l2client arbutil.L1Interface, + l1client *ethclient.Client, + l2client *ethclient.Client, templateTx *types.Transaction, ) *types.Receipt { delayedInboxContract, err := bridgegen.NewInbox(l1info.GetAddress("Inbox"), l1client) @@ -984,13 +987,13 @@ func SendUnsignedTxViaL1( return receipt } -func GetBaseFee(t *testing.T, client client, ctx context.Context) *big.Int { +func GetBaseFee(t *testing.T, client *ethclient.Client, ctx context.Context) *big.Int { header, err := client.HeaderByNumber(ctx, nil) Require(t, err) return header.BaseFee } -func GetBaseFeeAt(t *testing.T, client client, ctx context.Context, blockNum *big.Int) *big.Int { +func GetBaseFeeAt(t *testing.T, client *ethclient.Client, ctx context.Context, blockNum *big.Int) *big.Int { header, err := client.HeaderByNumber(ctx, blockNum) Require(t, err) return header.BaseFee @@ -1212,7 +1215,7 @@ func createTestL1BlockChain(t *testing.T, l1info info) (info, *ethclient.Client, return l1info, l1Client, l1backend, stack } -func getInitMessage(ctx context.Context, t *testing.T, parentChainClient client, addresses *chaininfo.RollupAddresses) *arbostypes.ParsedInitMessage { +func getInitMessage(ctx context.Context, t *testing.T, parentChainClient *ethclient.Client, addresses *chaininfo.RollupAddresses) *arbostypes.ParsedInitMessage { bridge, err := arbnode.NewDelayedBridge(parentChainClient, addresses.Bridge, addresses.DeployedAt) Require(t, err) deployedAtBig := arbmath.UintToBig(addresses.DeployedAt) @@ -1231,7 +1234,7 @@ func deployOnParentChain( t *testing.T, ctx context.Context, parentChainInfo info, - parentChainClient client, + parentChainClient *ethclient.Client, parentChainReaderConfig *headerreader.Config, chainConfig *params.ChainConfig, wasmModuleRoot common.Hash, @@ -1454,7 +1457,7 @@ func authorizeDASKeyset( ctx context.Context, dasSignerKey *blsSignatures.PublicKey, l1info info, - l1client arbutil.L1Interface, + l1client *ethclient.Client, ) { if dasSignerKey == nil { return @@ -1694,6 +1697,34 @@ func logParser[T any](t *testing.T, source string, name string) func(*types.Log) } } +// recordBlock writes a json file with all of the data needed to validate a block. +// +// This can be used as an input to the arbitrator prover to validate a block. +func recordBlock(t *testing.T, block uint64, builder *NodeBuilder) { + t.Helper() + ctx := builder.ctx + inboxPos := arbutil.MessageIndex(block) + for { + time.Sleep(250 * time.Millisecond) + batches, err := builder.L2.ConsensusNode.InboxTracker.GetBatchCount() + Require(t, err) + haveMessages, err := builder.L2.ConsensusNode.InboxTracker.GetBatchMessageCount(batches - 1) + Require(t, err) + if haveMessages >= inboxPos { + break + } + } + validationInputsWriter, err := inputs.NewWriter(inputs.WithSlug(t.Name())) + Require(t, err) + inputJson, err := builder.L2.ConsensusNode.StatelessBlockValidator.ValidationInputsAt(ctx, inboxPos, rawdb.TargetWavm) + if err != nil { + Fatal(t, "failed to get validation inputs", block, err) + } + if err := validationInputsWriter.Write(&inputJson); err != nil { + Fatal(t, "failed to write validation inputs", block, err) + } +} + func populateMachineDir(t *testing.T, cr *github.ConsensusRelease) string { baseDir := t.TempDir() machineDir := baseDir + "/machines" diff --git a/system_tests/das_test.go b/system_tests/das_test.go index 9f4d153b6f..ed3844d528 100644 --- a/system_tests/das_test.go +++ b/system_tests/das_test.go @@ -6,6 +6,7 @@ package arbtest import ( "context" "encoding/base64" + "errors" "io" "math/big" "net" @@ -22,7 +23,6 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/offchainlabs/nitro/arbnode" - "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/blsSignatures" "github.com/offchainlabs/nitro/cmd/genericconf" "github.com/offchainlabs/nitro/das" @@ -37,25 +37,20 @@ func startLocalDASServer( t *testing.T, ctx context.Context, dataDir string, - l1client arbutil.L1Interface, + l1client *ethclient.Client, seqInboxAddress common.Address, ) (*http.Server, *blsSignatures.PublicKey, das.BackendConfig, *das.RestfulDasServer, string) { keyDir := t.TempDir() pubkey, _, err := das.GenerateAndStoreKeys(keyDir) Require(t, err) - config := das.DataAvailabilityConfig{ - Enable: true, - Key: das.KeyConfig{ - KeyDir: keyDir, - }, - LocalFileStorage: das.LocalFileStorageConfig{ - Enable: true, - DataDir: dataDir, - }, - ParentChainNodeURL: "none", - RequestTimeout: 5 * time.Second, - } + config := das.DefaultDataAvailabilityConfig + config.Enable = true + config.Key = das.KeyConfig{KeyDir: keyDir} + config.ParentChainNodeURL = "none" + config.LocalFileStorage = das.DefaultLocalFileStorageConfig + config.LocalFileStorage.Enable = true + config.LocalFileStorage.DataDir = dataDir storageService, lifecycleManager, err := das.CreatePersistentStorageService(ctx, &config) defer lifecycleManager.StopAndWaitUntil(time.Second) @@ -327,3 +322,80 @@ func initTest(t *testing.T) { enableLogging(logLvl) } } + +func TestDASBatchPosterFallback(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Setup L1 + builder := NewNodeBuilder(ctx).DefaultConfig(t, true) + builder.chainConfig = params.ArbitrumDevTestDASChainConfig() + builder.BuildL1(t) + l1client := builder.L1.Client + l1info := builder.L1Info + + // Setup DAS server + dasDataDir := t.TempDir() + dasRpcServer, pubkey, backendConfig, _, restServerUrl := startLocalDASServer( + t, ctx, dasDataDir, l1client, builder.addresses.SequencerInbox) + authorizeDASKeyset(t, ctx, pubkey, l1info, l1client) + + // Setup sequence/batch-poster L2 node + builder.nodeConfig.DataAvailability.Enable = true + builder.nodeConfig.DataAvailability.RPCAggregator = aggConfigForBackend(backendConfig) + builder.nodeConfig.DataAvailability.RestAggregator = das.DefaultRestfulClientAggregatorConfig + builder.nodeConfig.DataAvailability.RestAggregator.Enable = true + builder.nodeConfig.DataAvailability.RestAggregator.Urls = []string{restServerUrl} + builder.nodeConfig.DataAvailability.ParentChainNodeURL = "none" + builder.nodeConfig.BatchPoster.DisableDapFallbackStoreDataOnChain = true // Disable DAS fallback + builder.nodeConfig.BatchPoster.ErrorDelay = time.Millisecond * 250 // Increase error delay because we expect errors + builder.L2Info = NewArbTestInfo(t, builder.chainConfig.ChainID) + builder.L2Info.GenerateAccount("User2") + cleanup := builder.BuildL2OnL1(t) + defer cleanup() + l2client := builder.L2.Client + l2info := builder.L2Info + + // Setup secondary L2 node + nodeConfigB := arbnode.ConfigDefaultL1NonSequencerTest() + nodeConfigB.BlockValidator.Enable = false + nodeConfigB.DataAvailability.Enable = true + nodeConfigB.DataAvailability.RestAggregator = das.DefaultRestfulClientAggregatorConfig + nodeConfigB.DataAvailability.RestAggregator.Enable = true + nodeConfigB.DataAvailability.RestAggregator.Urls = []string{restServerUrl} + nodeConfigB.DataAvailability.ParentChainNodeURL = "none" + nodeBParams := SecondNodeParams{ + nodeConfig: nodeConfigB, + initData: &l2info.ArbInitData, + } + l2B, cleanupB := builder.Build2ndNode(t, &nodeBParams) + defer cleanupB() + + // Check batch posting using the DAS + checkBatchPosting(t, ctx, l1client, l2client, l1info, l2info, big.NewInt(1e12), l2B.Client) + + // Shutdown the DAS + err := dasRpcServer.Shutdown(ctx) + Require(t, err) + + // Send 2nd transaction and check it doesn't arrive on second node + tx, _ := TransferBalanceTo(t, "Owner", l2info.GetAddress("User2"), big.NewInt(1e12), l2info, l2client, ctx) + _, err = WaitForTx(ctx, l2B.Client, tx.Hash(), time.Second*3) + if err == nil || !errors.Is(err, context.DeadlineExceeded) { + Fatal(t, "expected context-deadline exceeded error, but got:", err) + } + + // Enable the DAP fallback and check the transaction on the second node. + // (We don't need to restart the node because of the hot-reload.) + builder.nodeConfig.BatchPoster.DisableDapFallbackStoreDataOnChain = false + _, err = WaitForTx(ctx, l2B.Client, tx.Hash(), time.Second*3) + Require(t, err) + l2balance, err := l2B.Client.BalanceAt(ctx, l2info.GetAddress("User2"), nil) + Require(t, err) + if l2balance.Cmp(big.NewInt(2e12)) != 0 { + Fatal(t, "Unexpected balance:", l2balance) + } + + // Send another transaction with fallback on + checkBatchPosting(t, ctx, l1client, l2client, l1info, l2info, big.NewInt(3e12), l2B.Client) +} diff --git a/system_tests/eth_sync_test.go b/system_tests/eth_sync_test.go index 1f07f7c45f..ce9994fb1e 100644 --- a/system_tests/eth_sync_test.go +++ b/system_tests/eth_sync_test.go @@ -71,7 +71,7 @@ func TestEthSyncing(t *testing.T) { if progress == nil { Fatal(t, "eth_syncing returned nil but shouldn't have") } - for testClientB.ConsensusNode.TxStreamer.ExecuteNextMsg(ctx, testClientB.ExecNode) { + for testClientB.ConsensusNode.TxStreamer.ExecuteNextMsg(ctx) { } progress, err = testClientB.Client.SyncProgress(ctx) Require(t, err) diff --git a/system_tests/full_challenge_impl_test.go b/system_tests/full_challenge_impl_test.go index ddc229074c..bf30c928d8 100644 --- a/system_tests/full_challenge_impl_test.go +++ b/system_tests/full_challenge_impl_test.go @@ -27,7 +27,6 @@ import ( "github.com/offchainlabs/nitro/arbnode" "github.com/offchainlabs/nitro/arbos" "github.com/offchainlabs/nitro/arbstate" - "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/solgen/go/challengegen" "github.com/offchainlabs/nitro/solgen/go/mocksgen" "github.com/offchainlabs/nitro/solgen/go/ospgen" @@ -178,7 +177,7 @@ func makeBatch(t *testing.T, l2Node *arbnode.Node, l2Info *BlockchainTestInfo, b Require(t, err, "failed to get batch metadata after adding batch:") } -func confirmLatestBlock(ctx context.Context, t *testing.T, l1Info *BlockchainTestInfo, backend arbutil.L1Interface) { +func confirmLatestBlock(ctx context.Context, t *testing.T, l1Info *BlockchainTestInfo, backend *ethclient.Client) { t.Helper() // With SimulatedBeacon running in on-demand block production mode, the // finalized block is considered to be be the nearest multiple of 32 less @@ -190,7 +189,7 @@ func confirmLatestBlock(ctx context.Context, t *testing.T, l1Info *BlockchainTes } } -func setupSequencerInboxStub(ctx context.Context, t *testing.T, l1Info *BlockchainTestInfo, l1Client arbutil.L1Interface, chainConfig *params.ChainConfig) (common.Address, *mocksgen.SequencerInboxStub, common.Address) { +func setupSequencerInboxStub(ctx context.Context, t *testing.T, l1Info *BlockchainTestInfo, l1Client *ethclient.Client, chainConfig *params.ChainConfig) (common.Address, *mocksgen.SequencerInboxStub, common.Address) { txOpts := l1Info.GetDefaultTransactOpts("deployer", ctx) bridgeAddr, tx, bridge, err := mocksgen.DeployBridgeUnproxied(&txOpts, l1Client) Require(t, err) diff --git a/system_tests/program_gas_test.go b/system_tests/program_gas_test.go new file mode 100644 index 0000000000..119897cbfe --- /dev/null +++ b/system_tests/program_gas_test.go @@ -0,0 +1,458 @@ +package arbtest + +import ( + "context" + "fmt" + "math" + "math/big" + "regexp" + "strings" + "testing" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/eth/tracers/logger" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/rpc" + "github.com/offchainlabs/nitro/arbos/util" + "github.com/offchainlabs/nitro/execution/gethexec" + "github.com/offchainlabs/nitro/solgen/go/mocksgen" + "github.com/offchainlabs/nitro/solgen/go/precompilesgen" + "github.com/offchainlabs/nitro/util/testhelpers" +) + +func TestProgramSimpleCost(t *testing.T) { + builder := setupGasCostTest(t) + auth := builder.L2Info.GetDefaultTransactOpts("Owner", builder.ctx) + stylusProgram := deployWasm(t, builder.ctx, auth, builder.L2.Client, rustFile("hostio-test")) + evmProgram := deployEvmContract(t, builder.ctx, auth, builder.L2.Client, mocksgen.HostioTestMetaData) + otherProgram := deployWasm(t, builder.ctx, auth, builder.L2.Client, rustFile("storage")) + matchSnake := regexp.MustCompile("_[a-z]") + + for _, tc := range []struct { + hostio string + opcode vm.OpCode + params []any + maxDiff float64 + }{ + {hostio: "exit_early", opcode: vm.STOP}, + {hostio: "transient_load_bytes32", opcode: vm.TLOAD, params: []any{common.HexToHash("dead")}}, + {hostio: "transient_store_bytes32", opcode: vm.TSTORE, params: []any{common.HexToHash("dead"), common.HexToHash("beef")}}, + {hostio: "return_data_size", opcode: vm.RETURNDATASIZE, maxDiff: 1.5}, + {hostio: "account_balance", opcode: vm.BALANCE, params: []any{builder.L2Info.GetAddress("Owner")}}, + {hostio: "account_code", opcode: vm.EXTCODECOPY, params: []any{otherProgram}}, + {hostio: "account_code_size", opcode: vm.EXTCODESIZE, params: []any{otherProgram}, maxDiff: 0.3}, + {hostio: "account_codehash", opcode: vm.EXTCODEHASH, params: []any{otherProgram}}, + {hostio: "evm_gas_left", opcode: vm.GAS, maxDiff: 1.5}, + {hostio: "evm_ink_left", opcode: vm.GAS, maxDiff: 1.5}, + {hostio: "block_basefee", opcode: vm.BASEFEE, maxDiff: 0.5}, + {hostio: "chainid", opcode: vm.CHAINID, maxDiff: 1.5}, + {hostio: "block_coinbase", opcode: vm.COINBASE, maxDiff: 0.5}, + {hostio: "block_gas_limit", opcode: vm.GASLIMIT, maxDiff: 1.5}, + {hostio: "block_number", opcode: vm.NUMBER, maxDiff: 1.5}, + {hostio: "block_timestamp", opcode: vm.TIMESTAMP, maxDiff: 1.5}, + {hostio: "contract_address", opcode: vm.ADDRESS, maxDiff: 0.5}, + {hostio: "math_div", opcode: vm.DIV, params: []any{big.NewInt(1), big.NewInt(3)}}, + {hostio: "math_mod", opcode: vm.MOD, params: []any{big.NewInt(1), big.NewInt(3)}}, + {hostio: "math_add_mod", opcode: vm.ADDMOD, params: []any{big.NewInt(1), big.NewInt(3), big.NewInt(5)}, maxDiff: 0.7}, + {hostio: "math_mul_mod", opcode: vm.MULMOD, params: []any{big.NewInt(1), big.NewInt(3), big.NewInt(5)}, maxDiff: 0.7}, + {hostio: "msg_sender", opcode: vm.CALLER, maxDiff: 0.5}, + {hostio: "msg_value", opcode: vm.CALLVALUE, maxDiff: 0.5}, + {hostio: "tx_gas_price", opcode: vm.GASPRICE, maxDiff: 0.5}, + {hostio: "tx_ink_price", opcode: vm.GASPRICE, maxDiff: 1.5}, + {hostio: "tx_origin", opcode: vm.ORIGIN, maxDiff: 0.5}, + } { + t.Run(tc.hostio, func(t *testing.T) { + solFunc := matchSnake.ReplaceAllStringFunc(tc.hostio, func(s string) string { + return strings.ToUpper(strings.TrimPrefix(s, "_")) + }) + packer, _ := util.NewCallParser(mocksgen.HostioTestABI, solFunc) + data, err := packer(tc.params...) + Require(t, err) + compareGasUsage(t, builder, evmProgram, stylusProgram, data, nil, compareGasForEach, tc.maxDiff, compareGasPair{tc.opcode, tc.hostio}) + }) + } +} + +func TestProgramPowCost(t *testing.T) { + builder := setupGasCostTest(t) + auth := builder.L2Info.GetDefaultTransactOpts("Owner", builder.ctx) + stylusProgram := deployWasm(t, builder.ctx, auth, builder.L2.Client, rustFile("hostio-test")) + evmProgram := deployEvmContract(t, builder.ctx, auth, builder.L2.Client, mocksgen.HostioTestMetaData) + packer, _ := util.NewCallParser(mocksgen.HostioTestABI, "mathPow") + + for _, exponentNumBytes := range []uint{1, 2, 10, 32} { + name := fmt.Sprintf("exponentNumBytes%v", exponentNumBytes) + t.Run(name, func(t *testing.T) { + exponent := new(big.Int).Lsh(big.NewInt(1), exponentNumBytes*8-1) + params := []any{big.NewInt(1), exponent} + data, err := packer(params...) + Require(t, err) + evmGasUsage, stylusGasUsage := measureGasUsage(t, builder, evmProgram, stylusProgram, data, nil) + expectedGas := 2.652 + 1.75*float64(exponentNumBytes+1) + t.Logf("evm EXP usage: %v - stylus math_pow usage: %v - expected math_pow usage: %v", + evmGasUsage[vm.EXP][0], stylusGasUsage["math_pow"][0], expectedGas) + // The math_pow HostIO uses significally less gas than the EXP opcode. So, + // instead of comparing it to EVM, we compare it to the expected gas usage + // for each test case. + checkPercentDiff(t, stylusGasUsage["math_pow"][0], expectedGas, 0.001) + }) + } +} + +func TestProgramStorageCost(t *testing.T) { + builder := setupGasCostTest(t) + auth := builder.L2Info.GetDefaultTransactOpts("Owner", builder.ctx) + stylusMulticall := deployWasm(t, builder.ctx, auth, builder.L2.Client, rustFile("multicall")) + evmMulticall := deployEvmContract(t, builder.ctx, auth, builder.L2.Client, mocksgen.MultiCallTestMetaData) + + const numSlots = 42 + rander := testhelpers.NewPseudoRandomDataSource(t, 0) + readData := multicallEmptyArgs() + writeRandAData := multicallEmptyArgs() + writeRandBData := multicallEmptyArgs() + writeZeroData := multicallEmptyArgs() + for i := 0; i < numSlots; i++ { + slot := rander.GetHash() + readData = multicallAppendLoad(readData, slot, false) + writeRandAData = multicallAppendStore(writeRandAData, slot, rander.GetHash(), false) + writeRandBData = multicallAppendStore(writeRandBData, slot, rander.GetHash(), false) + writeZeroData = multicallAppendStore(writeZeroData, slot, common.Hash{}, false) + } + + for _, tc := range []struct { + name string + data []byte + }{ + {"initialWrite", writeRandAData}, + {"read", readData}, + {"writeAgain", writeRandBData}, + {"delete", writeZeroData}, + {"readZeros", readData}, + {"writeAgainAgain", writeRandAData}, + } { + t.Run(tc.name, func(t *testing.T) { + compareGasUsage(t, builder, evmMulticall, stylusMulticall, tc.data, nil, compareGasSum, 0, + compareGasPair{vm.SSTORE, "storage_flush_cache"}, compareGasPair{vm.SLOAD, "storage_load_bytes32"}) + }) + } +} + +func TestProgramLogCost(t *testing.T) { + builder := setupGasCostTest(t) + auth := builder.L2Info.GetDefaultTransactOpts("Owner", builder.ctx) + stylusProgram := deployWasm(t, builder.ctx, auth, builder.L2.Client, rustFile("hostio-test")) + evmProgram := deployEvmContract(t, builder.ctx, auth, builder.L2.Client, mocksgen.HostioTestMetaData) + packer, _ := util.NewCallParser(mocksgen.HostioTestABI, "emitLog") + + for ntopics := int8(0); ntopics < 5; ntopics++ { + for _, dataSize := range []uint64{10, 100, 1000} { + name := fmt.Sprintf("emitLog%dData%d", ntopics, dataSize) + t.Run(name, func(t *testing.T) { + args := []any{ + testhelpers.RandomSlice(dataSize), + ntopics, + } + for t := 0; t < 4; t++ { + args = append(args, testhelpers.RandomHash()) + } + data, err := packer(args...) + Require(t, err) + opcode := vm.LOG0 + vm.OpCode(ntopics) + compareGasUsage(t, builder, evmProgram, stylusProgram, data, nil, compareGasForEach, 0, compareGasPair{opcode, "emit_log"}) + }) + } + } + +} + +func TestProgramCallCost(t *testing.T) { + builder := setupGasCostTest(t) + auth := builder.L2Info.GetDefaultTransactOpts("Owner", builder.ctx) + stylusMulticall := deployWasm(t, builder.ctx, auth, builder.L2.Client, rustFile("multicall")) + evmMulticall := deployEvmContract(t, builder.ctx, auth, builder.L2.Client, mocksgen.MultiCallTestMetaData) + otherStylusProgram := deployWasm(t, builder.ctx, auth, builder.L2.Client, rustFile("hostio-test")) + otherEvmProgram := deployEvmContract(t, builder.ctx, auth, builder.L2.Client, mocksgen.HostioTestMetaData) + packer, _ := util.NewCallParser(mocksgen.HostioTestABI, "msgValue") + otherData, err := packer() + Require(t, err) + + for _, pair := range []compareGasPair{ + {vm.CALL, "call_contract"}, + {vm.DELEGATECALL, "delegate_call_contract"}, + {vm.STATICCALL, "static_call_contract"}, + } { + t.Run(pair.hostio+"/burnGas", func(t *testing.T) { + arbTest := common.HexToAddress("0x0000000000000000000000000000000000000069") + burnArbGas, _ := util.NewCallParser(precompilesgen.ArbosTestABI, "burnArbGas") + burnData, err := burnArbGas(big.NewInt(0)) + Require(t, err) + data := argsForMulticall(pair.opcode, arbTest, nil, burnData) + compareGasUsage(t, builder, evmMulticall, stylusMulticall, data, nil, compareGasForEach, 0, pair) + }) + + t.Run(pair.hostio+"/evmContract", func(t *testing.T) { + data := argsForMulticall(pair.opcode, otherEvmProgram, nil, otherData) + compareGasUsage(t, builder, evmMulticall, stylusMulticall, data, nil, compareGasForEach, 0, pair, + compareGasPair{vm.RETURNDATACOPY, "read_return_data"}) // also test read_return_data + }) + + t.Run(pair.hostio+"/stylusContract", func(t *testing.T) { + data := argsForMulticall(pair.opcode, otherStylusProgram, nil, otherData) + compareGasUsage(t, builder, evmMulticall, stylusMulticall, data, nil, compareGasForEach, 0, pair, + compareGasPair{vm.RETURNDATACOPY, "read_return_data"}) // also test read_return_data + }) + + t.Run(pair.hostio+"/multipleTimes", func(t *testing.T) { + data := multicallEmptyArgs() + for i := 0; i < 9; i++ { + data = multicallAppend(data, pair.opcode, otherEvmProgram, otherData) + } + compareGasUsage(t, builder, evmMulticall, stylusMulticall, data, nil, compareGasForEach, 0, pair) + }) + } + + t.Run("call_contract/evmContractWithValue", func(t *testing.T) { + value := big.NewInt(1000) + data := argsForMulticall(vm.CALL, otherEvmProgram, value, otherData) + compareGasUsage(t, builder, evmMulticall, stylusMulticall, data, value, compareGasForEach, 0, compareGasPair{vm.CALL, "call_contract"}) + }) +} + +func TestProgramCreateCost(t *testing.T) { + builder := setupGasCostTest(t) + auth := builder.L2Info.GetDefaultTransactOpts("Owner", builder.ctx) + stylusCreate := deployWasm(t, builder.ctx, auth, builder.L2.Client, rustFile("create")) + evmCreate := deployEvmContract(t, builder.ctx, auth, builder.L2.Client, mocksgen.CreateTestMetaData) + deployCode := common.FromHex(mocksgen.ProgramTestMetaData.Bin) + + t.Run("create1", func(t *testing.T) { + data := []byte{0x01} + data = append(data, (common.Hash{}).Bytes()...) // endowment + data = append(data, deployCode...) + compareGasUsage(t, builder, evmCreate, stylusCreate, data, nil, compareGasForEach, 0, compareGasPair{vm.CREATE, "create1"}) + }) + + t.Run("create2", func(t *testing.T) { + data := []byte{0x02} + data = append(data, (common.Hash{}).Bytes()...) // endowment + data = append(data, (common.HexToHash("beef")).Bytes()...) // salt + data = append(data, deployCode...) + compareGasUsage(t, builder, evmCreate, stylusCreate, data, nil, compareGasForEach, 0, compareGasPair{vm.CREATE2, "create2"}) + }) +} + +func TestProgramKeccakCost(t *testing.T) { + builder := setupGasCostTest(t) + auth := builder.L2Info.GetDefaultTransactOpts("Owner", builder.ctx) + stylusProgram := deployWasm(t, builder.ctx, auth, builder.L2.Client, rustFile("hostio-test")) + evmProgram := deployEvmContract(t, builder.ctx, auth, builder.L2.Client, mocksgen.HostioTestMetaData) + packer, _ := util.NewCallParser(mocksgen.HostioTestABI, "keccak") + + for i := 1; i < 5; i++ { + size := uint64(math.Pow10(i)) + name := fmt.Sprintf("keccak%d", size) + t.Run(name, func(t *testing.T) { + preImage := testhelpers.RandomSlice(size) + preImage[len(preImage)-1] = 0 + data, err := packer(preImage) + Require(t, err) + const maxDiff = 2.5 // stylus keccak charges significantly less gas + compareGasUsage(t, builder, evmProgram, stylusProgram, data, nil, compareGasForEach, maxDiff, compareGasPair{vm.KECCAK256, "native_keccak256"}) + }) + } +} + +func setupGasCostTest(t *testing.T) *NodeBuilder { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + builder := NewNodeBuilder(ctx).DefaultConfig(t, true) + cleanup := builder.Build(t) + t.Cleanup(cleanup) + return builder +} + +// deployEvmContract deploys an Evm contract and return its address. +func deployEvmContract(t *testing.T, ctx context.Context, auth bind.TransactOpts, client *ethclient.Client, metadata *bind.MetaData) common.Address { + t.Helper() + parsed, err := metadata.GetAbi() + Require(t, err) + address, tx, _, err := bind.DeployContract(&auth, *parsed, common.FromHex(metadata.Bin), client) + Require(t, err) + _, err = EnsureTxSucceeded(ctx, client, tx) + Require(t, err) + return address +} + +// measureGasUsage calls an EVM and a Wasm contract passing the same data and the same value. +func measureGasUsage( + t *testing.T, + builder *NodeBuilder, + evmContract common.Address, + stylusContract common.Address, + txData []byte, + txValue *big.Int, +) (map[vm.OpCode][]uint64, map[string][]float64) { + const txGas uint64 = 32_000_000 + txs := []*types.Transaction{ + builder.L2Info.PrepareTxTo("Owner", &evmContract, txGas, txValue, txData), + builder.L2Info.PrepareTxTo("Owner", &stylusContract, txGas, txValue, txData), + } + receipts := builder.L2.SendWaitTestTransactions(t, txs) + + evmGas := receipts[0].GasUsedForL2() + evmGasUsage, err := evmOpcodesGasUsage(builder.ctx, builder.L2.Client.Client(), txs[0]) + Require(t, err) + + stylusGas := receipts[1].GasUsedForL2() + stylusGasUsage, err := stylusHostiosGasUsage(builder.ctx, builder.L2.Client.Client(), txs[1]) + Require(t, err) + + t.Logf("evm total usage: %v - stylus total usage: %v", evmGas, stylusGas) + + return evmGasUsage, stylusGasUsage +} + +type compareGasPair struct { + opcode vm.OpCode + hostio string +} + +type compareGasMode int + +const ( + compareGasForEach compareGasMode = iota + compareGasSum +) + +// compareGasUsage calls measureGasUsage and then it ensures the given opcodes and hostios cost +// roughly the same amount of gas. +func compareGasUsage( + t *testing.T, + builder *NodeBuilder, + evmContract common.Address, + stylusContract common.Address, + txData []byte, + txValue *big.Int, + mode compareGasMode, + maxAllowedDifference float64, + pairs ...compareGasPair, +) { + if evmContract == stylusContract { + Fatal(t, "evm and stylus contract are the same") + } + evmGasUsage, stylusGasUsage := measureGasUsage(t, builder, evmContract, stylusContract, txData, txValue) + for i := range pairs { + opcode := pairs[i].opcode + hostio := pairs[i].hostio + switch mode { + case compareGasForEach: + if len(evmGasUsage[opcode]) != len(stylusGasUsage[hostio]) { + Fatal(t, "mismatch between hostios and opcodes", evmGasUsage, stylusGasUsage) + } + for i := range evmGasUsage[opcode] { + opcodeGas := evmGasUsage[opcode][i] + hostioGas := stylusGasUsage[hostio][i] + t.Logf("evm %v usage: %v - stylus %v usage: %v", opcode, opcodeGas, hostio, hostioGas) + checkPercentDiff(t, float64(opcodeGas), hostioGas, maxAllowedDifference) + } + case compareGasSum: + evmSum := float64(0) + stylusSum := float64(0) + for i := range evmGasUsage[opcode] { + evmSum += float64(evmGasUsage[opcode][i]) + stylusSum += stylusGasUsage[hostio][i] + } + t.Logf("evm %v usage: %v - stylus %v usage: %v", opcode, evmSum, hostio, stylusSum) + checkPercentDiff(t, evmSum, stylusSum, maxAllowedDifference) + } + } +} + +func evmOpcodesGasUsage(ctx context.Context, rpcClient rpc.ClientInterface, tx *types.Transaction) ( + map[vm.OpCode][]uint64, error) { + + var result logger.ExecutionResult + err := rpcClient.CallContext(ctx, &result, "debug_traceTransaction", tx.Hash(), nil) + if err != nil { + return nil, fmt.Errorf("failed to trace evm call: %w", err) + } + + gasUsage := map[vm.OpCode][]uint64{} + for i := range result.StructLogs { + op := vm.StringToOp(result.StructLogs[i].Op) + gasUsed := uint64(0) + if op == vm.CALL || op == vm.STATICCALL || op == vm.DELEGATECALL || op == vm.CREATE || op == vm.CREATE2 { + // For the CALL* opcodes, the GasCost in the tracer represents the gas sent + // to the callee contract, which is 63/64 of the remaining gas. This happens + // because the tracer is evaluated before the call is executed, so the EVM + // doesn't know how much gas will being used. + // + // In the case of the Stylus tracer, the trace is emitted after the + // execution, so the EndInk field is set to the ink after the call returned. + // Hence, it also includes the ink spent by the callee contract. + // + // To make a precise comparison between the EVM and Stylus, we modify the + // EVM measurement to include the gas spent by the callee contract. To do + // so, we go through the opcodes after CALL until we find the first opcode + // in the caller's depth. Then, we subtract the gas before the call by the + // gas after the call returned. + var gasAfterCall uint64 + for j := i + 1; j < len(result.StructLogs); j++ { + if result.StructLogs[j].Depth == result.StructLogs[i].Depth { + // back to the original call + gasAfterCall = result.StructLogs[j].Gas + result.StructLogs[j].GasCost + break + } + } + if gasAfterCall == 0 { + return nil, fmt.Errorf("malformed log: didn't get back to call original depth") + } + if i == 0 { + return nil, fmt.Errorf("malformed log: call is first opcode") + } + gasUsed = result.StructLogs[i-1].Gas - gasAfterCall + } else { + gasUsed = result.StructLogs[i].GasCost + } + gasUsage[op] = append(gasUsage[op], gasUsed) + } + return gasUsage, nil +} + +func stylusHostiosGasUsage(ctx context.Context, rpcClient rpc.ClientInterface, tx *types.Transaction) ( + map[string][]float64, error) { + + traceOpts := struct { + Tracer string `json:"tracer"` + }{ + Tracer: "stylusTracer", + } + var result []gethexec.HostioTraceInfo + err := rpcClient.CallContext(ctx, &result, "debug_traceTransaction", tx.Hash(), traceOpts) + if err != nil { + return nil, fmt.Errorf("failed to trace stylus call: %w", err) + } + + const InkPerGas = 10000 + gasUsage := map[string][]float64{} + for _, hostioLog := range result { + gasCost := float64(hostioLog.StartInk-hostioLog.EndInk) / InkPerGas + gasUsage[hostioLog.Name] = append(gasUsage[hostioLog.Name], gasCost) + } + return gasUsage, nil +} + +// checkPercentDiff checks whether the two values are close enough. +func checkPercentDiff(t *testing.T, a, b float64, maxAllowedDifference float64) { + t.Helper() + if maxAllowedDifference == 0 { + maxAllowedDifference = 0.25 + } + percentageDifference := (max(a, b) / min(a, b)) - 1 + if percentageDifference > maxAllowedDifference { + Fatal(t, fmt.Sprintf("gas usages are too different; got %v, max allowed is %v", percentageDifference, maxAllowedDifference)) + } +} diff --git a/system_tests/program_test.go b/system_tests/program_test.go index 1cbbf268f1..cf8cd72559 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -417,10 +417,15 @@ func storageTest(t *testing.T, jit bool) { key := testhelpers.RandomHash() value := testhelpers.RandomHash() tx := l2info.PrepareTxTo("Owner", &programAddress, l2info.TransferGas, nil, argsForStorageWrite(key, value)) - ensure(tx, l2client.SendTransaction(ctx, tx)) + receipt := ensure(tx, l2client.SendTransaction(ctx, tx)) + assertStorageAt(t, ctx, l2client, programAddress, key, value) validateBlocks(t, 2, jit, builder) + + // Captures a block_input_.json file for the block that included the + // storage write transaction. + recordBlock(t, receipt.BlockNumber.Uint64(), builder) } func TestProgramTransientStorage(t *testing.T) { diff --git a/system_tests/validation_mock_test.go b/system_tests/validation_mock_test.go index 2739c7545e..912b48ea6a 100644 --- a/system_tests/validation_mock_test.go +++ b/system_tests/validation_mock_test.go @@ -96,10 +96,6 @@ func (s *mockSpawner) LatestWasmModuleRoot() containers.PromiseInterface[common. return containers.NewReadyPromise[common.Hash](mockWasmModuleRoots[0], nil) } -func (s *mockSpawner) WriteToFile(input *validator.ValidationInput, expOut validator.GoGlobalState, moduleRoot common.Hash) containers.PromiseInterface[struct{}] { - return containers.NewReadyPromise[struct{}](struct{}{}, nil) -} - type mockValRun struct { containers.Promise[validator.GoGlobalState] root common.Hash diff --git a/system_tests/wrap_transaction_test.go b/system_tests/wrap_transaction_test.go index bd561ad5e5..36052fb2db 100644 --- a/system_tests/wrap_transaction_test.go +++ b/system_tests/wrap_transaction_test.go @@ -15,6 +15,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" "github.com/offchainlabs/nitro/arbutil" @@ -22,7 +23,7 @@ import ( "github.com/offchainlabs/nitro/util/headerreader" ) -func GetPendingBlockNumber(ctx context.Context, client arbutil.L1Interface) (*big.Int, error) { +func GetPendingBlockNumber(ctx context.Context, client *ethclient.Client) (*big.Int, error) { // Attempt to get the block number from ArbSys, if it exists arbSys, err := precompilesgen.NewArbSys(common.BigToAddress(big.NewInt(100)), client) if err != nil { @@ -37,7 +38,7 @@ func GetPendingBlockNumber(ctx context.Context, client arbutil.L1Interface) (*bi } // Will wait until txhash is in the blockchain and return its receipt -func WaitForTx(ctxinput context.Context, client arbutil.L1Interface, txhash common.Hash, timeout time.Duration) (*types.Receipt, error) { +func WaitForTx(ctxinput context.Context, client *ethclient.Client, txhash common.Hash, timeout time.Duration) (*types.Receipt, error) { ctx, cancel := context.WithTimeout(ctxinput, timeout) defer cancel() @@ -75,11 +76,11 @@ func WaitForTx(ctxinput context.Context, client arbutil.L1Interface, txhash comm } } -func EnsureTxSucceeded(ctx context.Context, client arbutil.L1Interface, tx *types.Transaction) (*types.Receipt, error) { +func EnsureTxSucceeded(ctx context.Context, client *ethclient.Client, tx *types.Transaction) (*types.Receipt, error) { return EnsureTxSucceededWithTimeout(ctx, client, tx, time.Second*5) } -func EnsureTxSucceededWithTimeout(ctx context.Context, client arbutil.L1Interface, tx *types.Transaction, timeout time.Duration) (*types.Receipt, error) { +func EnsureTxSucceededWithTimeout(ctx context.Context, client *ethclient.Client, tx *types.Transaction, timeout time.Duration) (*types.Receipt, error) { receipt, err := WaitForTx(ctx, client, tx.Hash(), timeout) if err != nil { return nil, fmt.Errorf("waitFoxTx (tx=%s) got: %w", tx.Hash().Hex(), err) @@ -103,12 +104,12 @@ func EnsureTxSucceededWithTimeout(ctx context.Context, client arbutil.L1Interfac return receipt, arbutil.DetailTxError(ctx, client, tx, receipt) } -func EnsureTxFailed(t *testing.T, ctx context.Context, client arbutil.L1Interface, tx *types.Transaction) *types.Receipt { +func EnsureTxFailed(t *testing.T, ctx context.Context, client *ethclient.Client, tx *types.Transaction) *types.Receipt { t.Helper() return EnsureTxFailedWithTimeout(t, ctx, client, tx, time.Second*5) } -func EnsureTxFailedWithTimeout(t *testing.T, ctx context.Context, client arbutil.L1Interface, tx *types.Transaction, timeout time.Duration) *types.Receipt { +func EnsureTxFailedWithTimeout(t *testing.T, ctx context.Context, client *ethclient.Client, tx *types.Transaction, timeout time.Duration) *types.Receipt { t.Helper() receipt, err := WaitForTx(ctx, client, tx.Hash(), timeout) Require(t, err) diff --git a/util/headerreader/blob_client.go b/util/headerreader/blob_client.go index 160323cf60..4831994bba 100644 --- a/util/headerreader/blob_client.go +++ b/util/headerreader/blob_client.go @@ -18,8 +18,8 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto/kzg4844" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" - "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/util/blobs" "github.com/offchainlabs/nitro/util/jsonapi" "github.com/offchainlabs/nitro/util/pretty" @@ -28,7 +28,7 @@ import ( ) type BlobClient struct { - ec arbutil.L1Interface + ec *ethclient.Client beaconUrl *url.URL secondaryBeaconUrl *url.URL httpClient *http.Client @@ -63,7 +63,7 @@ func BlobClientAddOptions(prefix string, f *pflag.FlagSet) { f.String(prefix+".authorization", DefaultBlobClientConfig.Authorization, "Value to send with the HTTP Authorization: header for Beacon REST requests, must include both scheme and scheme parameters") } -func NewBlobClient(config BlobClientConfig, ec arbutil.L1Interface) (*BlobClient, error) { +func NewBlobClient(config BlobClientConfig, ec *ethclient.Client) (*BlobClient, error) { beaconUrl, err := url.Parse(config.BeaconUrl) if err != nil { return nil, fmt.Errorf("failed to parse beacon chain URL: %w", err) diff --git a/util/headerreader/header_reader.go b/util/headerreader/header_reader.go index c8041dc871..98f778dee8 100644 --- a/util/headerreader/header_reader.go +++ b/util/headerreader/header_reader.go @@ -16,6 +16,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" "github.com/offchainlabs/nitro/arbutil" @@ -33,7 +34,7 @@ type ArbSysInterface interface { type HeaderReader struct { stopwaiter.StopWaiter config ConfigFetcher - client arbutil.L1Interface + client *ethclient.Client isParentChainArbitrum bool arbSys ArbSysInterface @@ -120,7 +121,7 @@ var TestConfig = Config{ }, } -func New(ctx context.Context, client arbutil.L1Interface, config ConfigFetcher, arbSysPrecompile ArbSysInterface) (*HeaderReader, error) { +func New(ctx context.Context, client *ethclient.Client, config ConfigFetcher, arbSysPrecompile ArbSysInterface) (*HeaderReader, error) { isParentChainArbitrum := false var arbSys ArbSysInterface if arbSysPrecompile != nil { @@ -522,7 +523,7 @@ func (s *HeaderReader) LatestFinalizedBlockNr(ctx context.Context) (uint64, erro return header.Number.Uint64(), nil } -func (s *HeaderReader) Client() arbutil.L1Interface { +func (s *HeaderReader) Client() *ethclient.Client { return s.client } diff --git a/validator/client/validation_client.go b/validator/client/validation_client.go index 3b18ad1851..934362f00a 100644 --- a/validator/client/validation_client.go +++ b/validator/client/validation_client.go @@ -188,19 +188,6 @@ func (c *ExecutionClient) LatestWasmModuleRoot() containers.PromiseInterface[com }) } -func (c *ExecutionClient) WriteToFile(input *validator.ValidationInput, expOut validator.GoGlobalState, moduleRoot common.Hash) containers.PromiseInterface[struct{}] { - jsonInput := server_api.ValidationInputToJson(input) - if err := jsonInput.WriteToFile(); err != nil { - return stopwaiter.LaunchPromiseThread[struct{}](c, func(ctx context.Context) (struct{}, error) { - return struct{}{}, err - }) - } - return stopwaiter.LaunchPromiseThread[struct{}](c, func(ctx context.Context) (struct{}, error) { - err := c.client.CallContext(ctx, nil, server_api.Namespace+"_writeToFile", jsonInput, expOut, moduleRoot) - return struct{}{}, err - }) -} - func (r *ExecutionClientRun) SendKeepAlive(ctx context.Context) time.Duration { err := r.client.client.CallContext(ctx, nil, server_api.Namespace+"_execKeepAlive", r.id) if err != nil { diff --git a/validator/inputs/writer.go b/validator/inputs/writer.go new file mode 100644 index 0000000000..a45e584f52 --- /dev/null +++ b/validator/inputs/writer.go @@ -0,0 +1,141 @@ +package inputs + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "github.com/offchainlabs/nitro/validator/server_api" +) + +// Writer is a configurable writer of InputJSON files. +// +// The default Writer will write to a path like: +// +// $HOME/.arbuitrum/validation-inputs//block_inputs_.json +// +// The path can be nested under a slug directory so callers can provide a +// recognizable name to differentiate various contexts in which the InputJSON +// is being written. If the Writer is configured by calling SetSlug, then the +// path will be like: +// +// $HOME/.arbuitrum/validation-inputs///block_inputs_.json +// +// The inclusion of a timestamp directory is on by default to avoid conflicts which +// would result in files being overwritten. However, the Writer can be configured +// to not use a timestamp directory. If the Writer is configured by calling +// SetUseTimestampDir(false), then the path will be like: +// +// $HOME/.arbuitrum/validation-inputs//block_inputs_.json +// +// Finally, to give complete control to the clients, the base directory can be +// set directly with SetBaseDir. In which case, the path will be like: +// +// /block_inputs_.json +// or +// //block_inputs_.json +// or +// ///block_inputs_.json +type Writer struct { + clock Clock + baseDir string + slug string + useTimestampDir bool +} + +// WriterOption is a function that configures a Writer. +type WriterOption func(*Writer) + +// Clock is an interface for getting the current time. +type Clock interface { + Now() time.Time +} + +type realClock struct{} + +func (realClock) Now() time.Time { + return time.Now() +} + +// NewWriter creates a new Writer with default settings. +func NewWriter(options ...WriterOption) (*Writer, error) { + homeDir, err := os.UserHomeDir() + if err != nil { + return nil, err + } + baseDir := filepath.Join(homeDir, ".arbitrum", "validation-inputs") + w := &Writer{ + clock: realClock{}, + baseDir: baseDir, + slug: "", + useTimestampDir: true, + } + for _, o := range options { + o(w) + } + return w, nil +} + +// withTestClock configures the Writer to use the given clock. +// +// This is only intended for testing. +func withTestClock(clock Clock) WriterOption { + return func(w *Writer) { + w.clock = clock + } +} + +// WithSlug configures the Writer to use the given slug as a directory name. +func WithSlug(slug string) WriterOption { + return func(w *Writer) { + w.slug = slug + } +} + +// WithoutSlug clears the slug configuration. +// +// This is equivalent to the WithSlug("") option but is more readable. +func WithoutSlug() WriterOption { + return WithSlug("") +} + +// WithBaseDir configures the Writer to use the given base directory. +func WithBaseDir(baseDir string) WriterOption { + return func(w *Writer) { + w.baseDir = baseDir + } +} + +// WithTimestampDirEnabled controls the addition of a timestamp directory. +func WithTimestampDirEnabled(useTimestampDir bool) WriterOption { + return func(w *Writer) { + w.useTimestampDir = useTimestampDir + } +} + +// Write writes the given InputJSON to a file in JSON format. +func (w *Writer) Write(json *server_api.InputJSON) error { + dir := w.baseDir + if w.slug != "" { + dir = filepath.Join(dir, w.slug) + } + if w.useTimestampDir { + t := w.clock.Now() + tStr := t.Format("20060102_150405") + dir = filepath.Join(dir, tStr) + } + if err := os.MkdirAll(dir, 0700); err != nil { + return err + } + contents, err := json.Marshal() + if err != nil { + return err + } + if err = os.WriteFile( + filepath.Join(dir, fmt.Sprintf("block_inputs_%d.json", json.Id)), + contents, 0600); err != nil { + return err + } + return nil +} diff --git a/validator/inputs/writer_test.go b/validator/inputs/writer_test.go new file mode 100644 index 0000000000..59cb63dae7 --- /dev/null +++ b/validator/inputs/writer_test.go @@ -0,0 +1,92 @@ +package inputs + +import ( + "os" + "testing" + "time" + + "github.com/offchainlabs/nitro/validator/server_api" +) + +func TestDefaultBaseDir(t *testing.T) { + // Simply testing that the default baseDir is set relative to the user's home directory. + // This way, the other tests can all override the baseDir to a temporary directory. + w, err := NewWriter() + if err != nil { + t.Fatal(err) + } + homeDir, err := os.UserHomeDir() + if err != nil { + t.Fatal(err) + } + if w.baseDir != homeDir+"/.arbitrum/validation-inputs" { + t.Errorf("unexpected baseDir: %v", w.baseDir) + } +} + +type fakeClock struct { + now time.Time +} + +func (c fakeClock) Now() time.Time { + return c.now +} + +func TestWriting(t *testing.T) { + dir := t.TempDir() + w, err := NewWriter( + withTestClock(fakeClock{now: time.Date(2021, 1, 2, 3, 4, 5, 0, time.UTC)}), + WithBaseDir(dir), + ) + if err != nil { + t.Fatal(err) + } + err = w.Write(&server_api.InputJSON{Id: 24601}) + if err != nil { + t.Fatal(err) + } + // The file should exist. + if _, err := os.Stat(dir + "/20210102_030405/block_inputs_24601.json"); err != nil { + t.Error(err) + } +} + +func TestWritingWithSlug(t *testing.T) { + dir := t.TempDir() + w, err := NewWriter( + withTestClock(fakeClock{now: time.Date(2021, 1, 2, 3, 4, 5, 0, time.UTC)}), + WithBaseDir(dir), + WithSlug("foo"), + ) + if err != nil { + t.Fatal(err) + } + err = w.Write(&server_api.InputJSON{Id: 24601}) + if err != nil { + t.Fatal(err) + } + // The file should exist. + if _, err := os.Stat(dir + "/foo/20210102_030405/block_inputs_24601.json"); err != nil { + t.Error(err) + } +} + +func TestWritingWithoutTimestampDir(t *testing.T) { + dir := t.TempDir() + w, err := NewWriter( + withTestClock(fakeClock{now: time.Date(2021, 1, 2, 3, 4, 5, 0, time.UTC)}), + WithBaseDir(dir), + WithTimestampDirEnabled(false), + ) + if err != nil { + t.Fatal(err) + } + err = w.Write(&server_api.InputJSON{Id: 24601}) + if err != nil { + t.Fatal(err) + } + // The file should exist. + if _, err := os.Stat(dir + "/block_inputs_24601.json"); err != nil { + t.Error(err) + } +} diff --git a/validator/interface.go b/validator/interface.go index af08629137..9fb831ca0d 100644 --- a/validator/interface.go +++ b/validator/interface.go @@ -27,7 +27,6 @@ type ExecutionSpawner interface { ValidationSpawner CreateExecutionRun(wasmModuleRoot common.Hash, input *ValidationInput) containers.PromiseInterface[ExecutionRun] LatestWasmModuleRoot() containers.PromiseInterface[common.Hash] - WriteToFile(input *ValidationInput, expOut GoGlobalState, moduleRoot common.Hash) containers.PromiseInterface[struct{}] } type ExecutionRun interface { diff --git a/validator/server_api/json.go b/validator/server_api/json.go index 6fe936e17d..8dfbc8446a 100644 --- a/validator/server_api/json.go +++ b/validator/server_api/json.go @@ -8,7 +8,6 @@ import ( "encoding/json" "errors" "fmt" - "os" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" @@ -68,15 +67,9 @@ type InputJSON struct { DebugChain bool } -func (i *InputJSON) WriteToFile() error { - contents, err := json.MarshalIndent(i, "", " ") - if err != nil { - return err - } - if err = os.WriteFile(fmt.Sprintf("block_inputs_%d.json", i.Id), contents, 0600); err != nil { - return err - } - return nil +// Marshal returns the JSON encoding of the InputJSON. +func (i *InputJSON) Marshal() ([]byte, error) { + return json.MarshalIndent(i, "", " ") } type BatchInfoJson struct { diff --git a/validator/server_arb/validator_spawner.go b/validator/server_arb/validator_spawner.go index 6f0d0cee1d..07971e2ba5 100644 --- a/validator/server_arb/validator_spawner.go +++ b/validator/server_arb/validator_spawner.go @@ -2,11 +2,8 @@ package server_arb import ( "context" - "encoding/binary" "errors" "fmt" - "os" - "path/filepath" "runtime" "sync/atomic" "time" @@ -98,7 +95,7 @@ func (s *ArbitratorSpawner) Name() string { return "arbitrator" } -func (v *ArbitratorSpawner) loadEntryToMachine(ctx context.Context, entry *validator.ValidationInput, mach *ArbitratorMachine) error { +func (v *ArbitratorSpawner) loadEntryToMachine(_ context.Context, entry *validator.ValidationInput, mach *ArbitratorMachine) error { resolver := func(ty arbutil.PreimageType, hash common.Hash) ([]byte, error) { // Check if it's a known preimage if preimage, ok := entry.Preimages[ty][hash]; ok { @@ -192,6 +189,7 @@ func (v *ArbitratorSpawner) execute( } func (v *ArbitratorSpawner) Launch(entry *validator.ValidationInput, moduleRoot common.Hash) validator.ValidationRun { + println("LAUCHING ARBITRATOR VALIDATION") v.count.Add(1) promise := stopwaiter.LaunchPromiseThread[validator.GoGlobalState](v, func(ctx context.Context) (validator.GoGlobalState, error) { defer v.count.Add(-1) @@ -208,139 +206,6 @@ func (v *ArbitratorSpawner) Room() int { return avail } -var launchTime = time.Now().Format("2006_01_02__15_04") - -//nolint:gosec -func (v *ArbitratorSpawner) writeToFile(ctx context.Context, input *validator.ValidationInput, expOut validator.GoGlobalState, moduleRoot common.Hash) error { - outDirPath := filepath.Join(v.locator.RootPath(), v.config().OutputPath, launchTime, fmt.Sprintf("block_%d", input.Id)) - err := os.MkdirAll(outDirPath, 0755) - if err != nil { - return err - } - if ctx.Err() != nil { - return ctx.Err() - } - - rootPathAssign := "" - if executable, err := os.Executable(); err == nil { - rootPathAssign = "ROOTPATH=\"" + filepath.Dir(executable) + "\"\n" - } - cmdFile, err := os.OpenFile(filepath.Join(outDirPath, "run-prover.sh"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755) - if err != nil { - return err - } - defer cmdFile.Close() - _, err = cmdFile.WriteString("#!/bin/bash\n" + - fmt.Sprintf("# expected output: batch %d, postion %d, hash %s\n", expOut.Batch, expOut.PosInBatch, expOut.BlockHash) + - "MACHPATH=\"" + v.locator.GetMachinePath(moduleRoot) + "\"\n" + - rootPathAssign + - "if (( $# > 1 )); then\n" + - " if [[ $1 == \"-m\" ]]; then\n" + - " MACHPATH=$2\n" + - " shift\n" + - " shift\n" + - " fi\n" + - "fi\n" + - "${ROOTPATH}/bin/prover ${MACHPATH}/replay.wasm") - if err != nil { - return err - } - if ctx.Err() != nil { - return ctx.Err() - } - - libraries := []string{"soft-float.wasm", "wasi_stub.wasm", "go_stub.wasm", "host_io.wasm", "brotli.wasm"} - for _, module := range libraries { - _, err = cmdFile.WriteString(" -l " + "${MACHPATH}/" + module) - if err != nil { - return err - } - } - _, err = cmdFile.WriteString(fmt.Sprintf(" --inbox-position %d --position-within-message %d --last-block-hash %s", input.StartState.Batch, input.StartState.PosInBatch, input.StartState.BlockHash)) - if err != nil { - return err - } - - for _, msg := range input.BatchInfo { - if ctx.Err() != nil { - return ctx.Err() - } - sequencerFileName := fmt.Sprintf("sequencer_%d.bin", msg.Number) - err = os.WriteFile(filepath.Join(outDirPath, sequencerFileName), msg.Data, 0644) - if err != nil { - return err - } - _, err = cmdFile.WriteString(" --inbox " + sequencerFileName) - if err != nil { - return err - } - } - - preimageFile, err := os.Create(filepath.Join(outDirPath, "preimages.bin")) - if err != nil { - return err - } - defer preimageFile.Close() - for ty, preimages := range input.Preimages { - _, err = preimageFile.Write([]byte{byte(ty)}) - if err != nil { - return err - } - for _, data := range preimages { - if ctx.Err() != nil { - return ctx.Err() - } - lenbytes := make([]byte, 8) - binary.LittleEndian.PutUint64(lenbytes, uint64(len(data))) - _, err := preimageFile.Write(lenbytes) - if err != nil { - return err - } - _, err = preimageFile.Write(data) - if err != nil { - return err - } - } - } - - _, err = cmdFile.WriteString(" --preimages preimages.bin") - if err != nil { - return err - } - - if input.HasDelayedMsg { - if ctx.Err() != nil { - return ctx.Err() - } - _, err = cmdFile.WriteString(fmt.Sprintf(" --delayed-inbox-position %d", input.DelayedMsgNr)) - if err != nil { - return err - } - filename := fmt.Sprintf("delayed_%d.bin", input.DelayedMsgNr) - err = os.WriteFile(filepath.Join(outDirPath, filename), input.DelayedMsg, 0644) - if err != nil { - return err - } - _, err = cmdFile.WriteString(fmt.Sprintf(" --delayed-inbox %s", filename)) - if err != nil { - return err - } - } - - _, err = cmdFile.WriteString(" \"$@\"\n") - if err != nil { - return err - } - return nil -} - -func (v *ArbitratorSpawner) WriteToFile(input *validator.ValidationInput, expOut validator.GoGlobalState, moduleRoot common.Hash) containers.PromiseInterface[struct{}] { - return stopwaiter.LaunchPromiseThread[struct{}](v, func(ctx context.Context) (struct{}, error) { - err := v.writeToFile(ctx, input, expOut, moduleRoot) - return struct{}{}, err - }) -} - func (v *ArbitratorSpawner) CreateExecutionRun(wasmModuleRoot common.Hash, input *validator.ValidationInput) containers.PromiseInterface[validator.ExecutionRun] { getMachine := func(ctx context.Context) (MachineInterface, error) { initialFrozenMachine, err := v.machineLoader.GetZeroStepMachine(ctx, wasmModuleRoot) diff --git a/validator/server_jit/jit_machine.go b/validator/server_jit/jit_machine.go index 2bea75fbe9..0748101277 100644 --- a/validator/server_jit/jit_machine.go +++ b/validator/server_jit/jit_machine.go @@ -33,7 +33,7 @@ type JitMachine struct { maxExecutionTime time.Duration } -func createJitMachine(jitBinary string, binaryPath string, cranelift bool, wasmMemoryUsageLimit int, maxExecutionTime time.Duration, moduleRoot common.Hash, fatalErrChan chan error) (*JitMachine, error) { +func createJitMachine(jitBinary string, binaryPath string, cranelift bool, wasmMemoryUsageLimit int, maxExecutionTime time.Duration, _ common.Hash, fatalErrChan chan error) (*JitMachine, error) { invocation := []string{"--binary", binaryPath, "--forks"} if cranelift { invocation = append(invocation, "--cranelift") diff --git a/validator/valnode/validation_api.go b/validator/valnode/validation_api.go index a10d931dfc..ef3e1b2c49 100644 --- a/validator/valnode/validation_api.go +++ b/validator/valnode/validation_api.go @@ -118,15 +118,6 @@ func (a *ExecServerAPI) Start(ctx_in context.Context) { a.CallIteratively(a.removeOldRuns) } -func (a *ExecServerAPI) WriteToFile(ctx context.Context, jsonInput *server_api.InputJSON, expOut validator.GoGlobalState, moduleRoot common.Hash) error { - input, err := server_api.ValidationInputFromJson(jsonInput) - if err != nil { - return err - } - _, err = a.execSpawner.WriteToFile(input, expOut, moduleRoot).Await(ctx) - return err -} - var errRunNotFound error = errors.New("run not found") func (a *ExecServerAPI) getRun(id uint64) (validator.ExecutionRun, error) { From 67e09f35682ce40332c9983f4f7cd3b4a016652a Mon Sep 17 00:00:00 2001 From: Maciej Kulawik <10907694+magicxyyz@users.noreply.github.com> Date: Tue, 8 Oct 2024 14:56:22 +0200 Subject: [PATCH 341/489] system_tests: use wasmCacheTag when calling WrapDatabaseWithWasm Co-authored-by: Diego Ximenes Mendes --- system_tests/common_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system_tests/common_test.go b/system_tests/common_test.go index b5e71e3385..807d39d082 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -1420,7 +1420,7 @@ func Create2ndNodeWithConfig( Require(t, err) wasmData, err := chainStack.OpenDatabaseWithExtraOptions("wasm", 0, 0, "wasm/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("wasm")) Require(t, err) - chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmData, 1, execConfig.StylusTarget.WasmTargets()) + chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmData, wasmCacheTag, execConfig.StylusTarget.WasmTargets()) arbDb, err := chainStack.OpenDatabaseWithExtraOptions("arbitrumdata", 0, 0, "arbitrumdata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("arbitrumdata")) Require(t, err) From b178d70275e2bc0fe1d4701fb8715cddfc03c8e1 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Tue, 8 Oct 2024 15:06:44 +0200 Subject: [PATCH 342/489] fix names of stylus cache metrics --- arbos/programs/native.go | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/arbos/programs/native.go b/arbos/programs/native.go index 5baacea381..f1fde5c556 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -47,16 +47,16 @@ type rustBytes = C.RustBytes type rustSlice = C.RustSlice var ( - stylusLRUCacheSizeBytesGauge = metrics.NewRegisteredGauge("arb/arbos/stylus/cache/lru/size_bytes", nil) - stylusLRUCacheSizeCountGauge = metrics.NewRegisteredGauge("arb/arbos/stylus/cache/lru/count", nil) - stylusLRUCacheSizeHitsCounter = metrics.NewRegisteredCounter("arb/arbos/stylus/cache/lru/hits", nil) - stylusLRUCacheSizeMissesCounter = metrics.NewRegisteredCounter("arb/arbos/stylus/cache/lru/misses", nil) - stylusLRUCacheSizeDoesNotFitCounter = metrics.NewRegisteredCounter("arb/arbos/stylus/cache/lru/does_not_fit", nil) - - stylusLongTermCacheSizeBytesGauge = metrics.NewRegisteredGauge("arb/arbos/stylus/cache/long_term/size_bytes", nil) - stylusLongTermCacheSizeCountGauge = metrics.NewRegisteredGauge("arb/arbos/stylus/cache/long_term/count", nil) - stylusLongTermCacheSizeHitsCounter = metrics.NewRegisteredCounter("arb/arbos/stylus/cache/long_term/hits", nil) - stylusLongTermCacheSizeMissesCounter = metrics.NewRegisteredCounter("arb/arbos/stylus/cache/long_term/misses", nil) + stylusLRUCacheSizeBytesGauge = metrics.NewRegisteredGauge("arb/arbos/stylus/cache/lru/size_bytes", nil) + stylusLRUCacheCountGauge = metrics.NewRegisteredGauge("arb/arbos/stylus/cache/lru/count", nil) + stylusLRUCacheHitsCounter = metrics.NewRegisteredCounter("arb/arbos/stylus/cache/lru/hits", nil) + stylusLRUCacheMissesCounter = metrics.NewRegisteredCounter("arb/arbos/stylus/cache/lru/misses", nil) + stylusLRUCacheDoesNotFitCounter = metrics.NewRegisteredCounter("arb/arbos/stylus/cache/lru/does_not_fit", nil) + + stylusLongTermCacheSizeBytesGauge = metrics.NewRegisteredGauge("arb/arbos/stylus/cache/long_term/size_bytes", nil) + stylusLongTermCacheCountGauge = metrics.NewRegisteredGauge("arb/arbos/stylus/cache/long_term/count", nil) + stylusLongTermCacheHitsCounter = metrics.NewRegisteredCounter("arb/arbos/stylus/cache/long_term/hits", nil) + stylusLongTermCacheMissesCounter = metrics.NewRegisteredCounter("arb/arbos/stylus/cache/long_term/misses", nil) ) func activateProgram( @@ -342,15 +342,15 @@ func UpdateWasmCacheMetrics() { metrics := C.stylus_get_cache_metrics() stylusLRUCacheSizeBytesGauge.Update(int64(metrics.lru.size_bytes)) - stylusLRUCacheSizeCountGauge.Update(int64(metrics.lru.count)) - stylusLRUCacheSizeHitsCounter.Inc(int64(metrics.lru.hits)) - stylusLRUCacheSizeMissesCounter.Inc(int64(metrics.lru.misses)) - stylusLRUCacheSizeDoesNotFitCounter.Inc(int64(metrics.lru.does_not_fit)) + stylusLRUCacheCountGauge.Update(int64(metrics.lru.count)) + stylusLRUCacheHitsCounter.Inc(int64(metrics.lru.hits)) + stylusLRUCacheMissesCounter.Inc(int64(metrics.lru.misses)) + stylusLRUCacheDoesNotFitCounter.Inc(int64(metrics.lru.does_not_fit)) stylusLongTermCacheSizeBytesGauge.Update(int64(metrics.long_term.size_bytes)) - stylusLongTermCacheSizeCountGauge.Update(int64(metrics.long_term.count)) - stylusLongTermCacheSizeHitsCounter.Inc(int64(metrics.long_term.hits)) - stylusLongTermCacheSizeMissesCounter.Inc(int64(metrics.long_term.misses)) + stylusLongTermCacheCountGauge.Update(int64(metrics.long_term.count)) + stylusLongTermCacheHitsCounter.Inc(int64(metrics.long_term.hits)) + stylusLongTermCacheMissesCounter.Inc(int64(metrics.long_term.misses)) } // Used for testing From c6619a5d3335433a2294c05f627f38d1b897d9c7 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Tue, 8 Oct 2024 15:12:19 +0200 Subject: [PATCH 343/489] fix build2ndNode --- system_tests/common_test.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 807d39d082..1cde8fd7bc 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -86,12 +86,13 @@ import ( type info = *BlockchainTestInfo type SecondNodeParams struct { - nodeConfig *arbnode.Config - execConfig *gethexec.Config - stackConfig *node.Config - dasConfig *das.DataAvailabilityConfig - initData *statetransfer.ArbosInitializationInfo - addresses *chaininfo.RollupAddresses + nodeConfig *arbnode.Config + execConfig *gethexec.Config + stackConfig *node.Config + dasConfig *das.DataAvailabilityConfig + initData *statetransfer.ArbosInitializationInfo + addresses *chaininfo.RollupAddresses + wasmCacheTag uint32 } type TestClient struct { @@ -717,7 +718,7 @@ func build2ndNode( testClient := NewTestClient(ctx) testClient.Client, testClient.ConsensusNode = - Create2ndNodeWithConfig(t, ctx, firstNodeTestClient.ConsensusNode, parentChainTestClient.Stack, parentChainInfo, params.initData, params.nodeConfig, params.execConfig, params.stackConfig, valnodeConfig, params.addresses, initMessage) + Create2ndNodeWithConfig(t, ctx, firstNodeTestClient.ConsensusNode, parentChainTestClient.Stack, parentChainInfo, params.initData, params.nodeConfig, params.execConfig, params.stackConfig, valnodeConfig, params.addresses, initMessage, params.wasmCacheTag) testClient.ExecNode = getExecNode(t, testClient.ConsensusNode) testClient.cleanup = func() { testClient.ConsensusNode.StopAndWait() } return testClient, func() { testClient.cleanup() } @@ -1399,6 +1400,7 @@ func Create2ndNodeWithConfig( valnodeConfig *valnode.Config, addresses *chaininfo.RollupAddresses, initMessage *arbostypes.ParsedInitMessage, + wasmCacheTag uint32, ) (*ethclient.Client, *arbnode.Node) { if nodeConfig == nil { nodeConfig = arbnode.ConfigDefaultL1NonSequencerTest() From 495ade6413f6984391dd7b099f11e1309f467160 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Tue, 8 Oct 2024 15:20:05 +0200 Subject: [PATCH 344/489] use fixed arbos tag in stylus_clear_long_term_cache as it's only for testing --- arbitrator/stylus/src/lib.rs | 6 +++--- arbos/programs/native.go | 4 ++-- system_tests/program_test.go | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/arbitrator/stylus/src/lib.rs b/arbitrator/stylus/src/lib.rs index feac828989..a6be21f7bf 100644 --- a/arbitrator/stylus/src/lib.rs +++ b/arbitrator/stylus/src/lib.rs @@ -377,11 +377,11 @@ pub extern "C" fn stylus_clear_lru_cache() { InitCache::clear_lru_cache() } -/// Clears long term cache. +/// Clears long term cache (for arbos_tag = 1) /// Only used for testing purposes. #[no_mangle] -pub extern "C" fn stylus_clear_long_term_cache(arbos_tag: u32) { - InitCache::clear_long_term(arbos_tag); +pub extern "C" fn stylus_clear_long_term_cache() { + InitCache::clear_long_term(1); } /// Gets entry size in bytes. diff --git a/arbos/programs/native.go b/arbos/programs/native.go index f1fde5c556..e5c2632667 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -393,8 +393,8 @@ func ClearWasmLruCache() { } // Used for testing -func ClearWasmLongTermCache(arbos_tag uint32) { - C.stylus_clear_long_term_cache(u32(arbos_tag)) +func ClearWasmLongTermCache() { + C.stylus_clear_long_term_cache() } // Used for testing diff --git a/system_tests/program_test.go b/system_tests/program_test.go index aab207e0f6..fac459118b 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -2183,7 +2183,7 @@ func TestWasmLongTermCache(t *testing.T) { ownerAuth.Value = common.Big0 - programs.ClearWasmLongTermCache(1) + programs.ClearWasmLongTermCache() checkLongTermCacheMetrics(t, programs.WasmLongTermCacheMetrics{ Count: 0, SizeBytes: 0, @@ -2312,7 +2312,7 @@ func TestRepopulateWasmLongTermCacheFromLru(t *testing.T) { ownerAuth.Value = common.Big0 - programs.ClearWasmLongTermCache(1) + programs.ClearWasmLongTermCache() programs.ClearWasmLruCache() // only 2 out of 3 programs should fit lru programs.SetWasmLruCacheCapacity( @@ -2339,7 +2339,7 @@ func TestRepopulateWasmLongTermCacheFromLru(t *testing.T) { }) // clear long term cache to emulate restart - programs.ClearWasmLongTermCache(1) + programs.ClearWasmLongTermCache() programs.ClearWasmLruCache() checkLruCacheMetrics(t, programs.WasmLruCacheMetrics{ From dca2484643f48dad1bf59706c2eace8fea9c075c Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Tue, 8 Oct 2024 15:36:31 +0200 Subject: [PATCH 345/489] program_test: add entry sizes sanity check --- system_tests/program_test.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/system_tests/program_test.go b/system_tests/program_test.go index fac459118b..fa07e3b5f1 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -2074,11 +2074,14 @@ func TestWasmLruCache(t *testing.T) { keccakProgramAddress, keccakEntrySize := deployWasmAndGetEntrySizeEstimateBytes(t, builder, auth, "keccak") mathProgramAddress, mathEntrySize := deployWasmAndGetEntrySizeEstimateBytes(t, builder, auth, "math") t.Log( - "lruEntrySizeEstimateBytes, ", + "entrySizeEstimateBytes, ", "fallible:", fallibleEntrySize, "keccak:", keccakEntrySize, "math:", mathEntrySize, ) + if fallibleEntrySize == keccakEntrySize || fallibleEntrySize == mathEntrySize || keccakEntrySize == mathEntrySize { + Fatal(t, "at least two programs have the same entry size") + } programs.ClearWasmLruCache() checkLruCacheMetrics(t, programs.WasmLruCacheMetrics{ @@ -2175,11 +2178,14 @@ func TestWasmLongTermCache(t *testing.T) { keccakProgramAddress, keccakEntrySize := deployWasmAndGetEntrySizeEstimateBytes(t, builder, ownerAuth, "keccak") mathProgramAddress, mathEntrySize := deployWasmAndGetEntrySizeEstimateBytes(t, builder, ownerAuth, "math") t.Log( - "lruEntrySizeEstimateBytes, ", + "entrySizeEstimateBytes, ", "fallible:", fallibleEntrySize, "keccak:", keccakEntrySize, "math:", mathEntrySize, ) + if fallibleEntrySize == keccakEntrySize || fallibleEntrySize == mathEntrySize || keccakEntrySize == mathEntrySize { + Fatal(t, "at least two programs have the same entry size") + } ownerAuth.Value = common.Big0 @@ -2309,6 +2315,9 @@ func TestRepopulateWasmLongTermCacheFromLru(t *testing.T) { fallibleProgramAddress, fallibleEntrySize := deployWasmAndGetEntrySizeEstimateBytes(t, builder, ownerAuth, "fallible") keccakProgramAddress, keccakEntrySize := deployWasmAndGetEntrySizeEstimateBytes(t, builder, ownerAuth, "keccak") mathProgramAddress, mathEntrySize := deployWasmAndGetEntrySizeEstimateBytes(t, builder, ownerAuth, "math") + if fallibleEntrySize == keccakEntrySize || fallibleEntrySize == mathEntrySize || keccakEntrySize == mathEntrySize { + Fatal(t, "at least two programs have the same entry size") + } ownerAuth.Value = common.Big0 From 737ffb1fe659a9d4e618d46a22d24de466338a0c Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Tue, 8 Oct 2024 15:54:47 +0200 Subject: [PATCH 346/489] program_test: add comment --- system_tests/program_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/system_tests/program_test.go b/system_tests/program_test.go index fa07e3b5f1..4755096b26 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -2366,6 +2366,7 @@ func TestRepopulateWasmLongTermCacheFromLru(t *testing.T) { Require(t, err) // restore nonce in L2Info builder.L2Info.GetInfoWithPrivKey("Owner").Nonce.Store(nonce) + // fallibleProgram should be added only to lru cache as the api call should be processed with wasm cache tag = 0 checkLruCacheMetrics(t, programs.WasmLruCacheMetrics{ Count: 1, SizeBytes: fallibleEntrySize, From 44e6fba5d1cce7b0bf76f3551e8bf95e795e0804 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Tue, 8 Oct 2024 20:19:47 +0530 Subject: [PATCH 347/489] fix race issue --- arbnode/maintenance.go | 2 +- arbnode/seq_coordinator.go | 61 ++++++++++++++++++++------------- arbnode/seq_coordinator_test.go | 18 +++++----- system_tests/forwarder_test.go | 2 +- 4 files changed, 48 insertions(+), 35 deletions(-) diff --git a/arbnode/maintenance.go b/arbnode/maintenance.go index 53d038a0f9..7397229c2e 100644 --- a/arbnode/maintenance.go +++ b/arbnode/maintenance.go @@ -101,7 +101,7 @@ func NewMaintenanceRunner(config MaintenanceConfigFetcher, seqCoordinator *SeqCo if seqCoordinator != nil { c := func() *redislock.SimpleCfg { return &cfg.Lock } r := func() bool { return true } // always ready to lock - rl, err := redislock.NewSimple(seqCoordinator.Client, c, r) + rl, err := redislock.NewSimple(seqCoordinator.RedisCoordinator().Client, c, r) if err != nil { return nil, fmt.Errorf("creating new simple redis lock: %w", err) } diff --git a/arbnode/seq_coordinator.go b/arbnode/seq_coordinator.go index f2bebbf472..4fbc31a108 100644 --- a/arbnode/seq_coordinator.go +++ b/arbnode/seq_coordinator.go @@ -37,7 +37,8 @@ var ( type SeqCoordinator struct { stopwaiter.StopWaiter - redisutil.RedisCoordinator + redisCoordinatorMutex sync.RWMutex + redisCoordinator redisutil.RedisCoordinator prevRedisCoordinator *redisutil.RedisCoordinator prevRedisMessageCount arbutil.MessageIndex @@ -159,7 +160,7 @@ func NewSeqCoordinator( return nil, err } coordinator := &SeqCoordinator{ - RedisCoordinator: *redisCoordinator, + redisCoordinator: *redisCoordinator, sync: sync, streamer: streamer, sequencer: sequencer, @@ -180,6 +181,19 @@ func (c *SeqCoordinator) SetDelayedSequencer(delayedSequencer *DelayedSequencer) c.delayedSequencer = delayedSequencer } +func (c *SeqCoordinator) RedisCoordinator() *redisutil.RedisCoordinator { + c.redisCoordinatorMutex.RLock() + defer c.redisCoordinatorMutex.RUnlock() + return &c.redisCoordinator +} + +func (c *SeqCoordinator) setRedisCoordinator(redisCoordinator *redisutil.RedisCoordinator) { + c.redisCoordinatorMutex.Lock() + defer c.redisCoordinatorMutex.Unlock() + c.prevRedisCoordinator = &c.redisCoordinator + c.redisCoordinator = *redisCoordinator +} + func StandaloneSeqCoordinatorInvalidateMsgIndex(ctx context.Context, redisClient redis.UniversalClient, keyConfig string, msgIndex arbutil.MessageIndex) error { signerConfig := signature.EmptySimpleHmacConfig if keyConfig == "" { @@ -282,7 +296,7 @@ func (c *SeqCoordinator) acquireLockoutAndWriteMessage(ctx context.Context, msgC defer c.wantsLockoutMutex.Unlock() setWantsLockout := c.avoidLockout <= 0 lockoutUntil := time.Now().Add(c.config.LockoutDuration) - err = c.Client.Watch(ctx, func(tx *redis.Tx) error { + err = c.RedisCoordinator().Client.Watch(ctx, func(tx *redis.Tx) error { current, err := tx.Get(ctx, redisutil.CHOSENSEQ_KEY).Result() var wasEmpty bool if errors.Is(err, redis.Nil) { @@ -351,7 +365,7 @@ func (c *SeqCoordinator) acquireLockoutAndWriteMessage(ctx context.Context, msgC } func (c *SeqCoordinator) getRemoteFinalizedMsgCount(ctx context.Context) (arbutil.MessageIndex, error) { - resStr, err := c.Client.Get(ctx, redisutil.FINALIZED_MSG_COUNT_KEY).Result() + resStr, err := c.RedisCoordinator().Client.Get(ctx, redisutil.FINALIZED_MSG_COUNT_KEY).Result() if err != nil { return 0, err } @@ -370,7 +384,7 @@ func (c *SeqCoordinator) getRemoteMsgCountImpl(ctx context.Context, r redis.Cmda } func (c *SeqCoordinator) GetRemoteMsgCount() (arbutil.MessageIndex, error) { - return c.getRemoteMsgCountImpl(c.GetContext(), c.Client) + return c.getRemoteMsgCountImpl(c.GetContext(), c.RedisCoordinator().Client) } func (c *SeqCoordinator) wantsLockoutUpdate(ctx context.Context, client redis.UniversalClient) error { @@ -404,7 +418,7 @@ func (c *SeqCoordinator) wantsLockoutUpdateWithMutex(ctx context.Context, client func (c *SeqCoordinator) chosenOneRelease(ctx context.Context) error { atomicTimeWrite(&c.lockoutUntil, time.Time{}) isActiveSequencer.Update(0) - releaseErr := c.Client.Watch(ctx, func(tx *redis.Tx) error { + releaseErr := c.RedisCoordinator().Client.Watch(ctx, func(tx *redis.Tx) error { current, err := tx.Get(ctx, redisutil.CHOSENSEQ_KEY).Result() if errors.Is(err, redis.Nil) { return nil @@ -427,7 +441,7 @@ func (c *SeqCoordinator) chosenOneRelease(ctx context.Context) error { return nil } // got error - was it still released? - current, readErr := c.Client.Get(ctx, redisutil.CHOSENSEQ_KEY).Result() + current, readErr := c.RedisCoordinator().Client.Get(ctx, redisutil.CHOSENSEQ_KEY).Result() if errors.Is(readErr, redis.Nil) { return nil } @@ -444,10 +458,10 @@ func (c *SeqCoordinator) wantsLockoutRelease(ctx context.Context) error { return nil } myWantsLockoutKey := redisutil.WantsLockoutKeyFor(c.config.Url()) - releaseErr := c.Client.Del(ctx, myWantsLockoutKey).Err() + releaseErr := c.RedisCoordinator().Client.Del(ctx, myWantsLockoutKey).Err() if releaseErr != nil { // got error - was it still deleted? - readErr := c.Client.Get(ctx, myWantsLockoutKey).Err() + readErr := c.RedisCoordinator().Client.Get(ctx, myWantsLockoutKey).Err() if !errors.Is(readErr, redis.Nil) { return releaseErr } @@ -531,7 +545,7 @@ func (c *SeqCoordinator) deleteFinalizedMsgsFromRedis(ctx context.Context, final // In non-init cases it doesn't matter how we delete as we always try to delete from prevFinalized to finalized batchDeleteCount := 1000 for i := len(keys); i > 0; i -= batchDeleteCount { - if err := c.Client.Del(ctx, keys[max(0, i-batchDeleteCount):i]...).Err(); err != nil { + if err := c.RedisCoordinator().Client.Del(ctx, keys[max(0, i-batchDeleteCount):i]...).Err(); err != nil { return fmt.Errorf("error deleting finalized messages and their signatures from redis: %w", err) } } @@ -540,7 +554,7 @@ func (c *SeqCoordinator) deleteFinalizedMsgsFromRedis(ctx context.Context, final if err != nil { return err } - if err = c.Client.Set(ctx, redisutil.FINALIZED_MSG_COUNT_KEY, finalizedBytes, c.config.SeqNumDuration).Err(); err != nil { + if err = c.RedisCoordinator().Client.Set(ctx, redisutil.FINALIZED_MSG_COUNT_KEY, finalizedBytes, c.config.SeqNumDuration).Err(); err != nil { return fmt.Errorf("couldn't set %s key to current finalizedMsgCount in redis: %w", redisutil.FINALIZED_MSG_COUNT_KEY, err) } return nil @@ -549,7 +563,7 @@ func (c *SeqCoordinator) deleteFinalizedMsgsFromRedis(ctx context.Context, final if errors.Is(err, redis.Nil) { var keys []string for msg := finalized - 1; msg > 0; msg-- { - exists, err := c.Client.Exists(ctx, redisutil.MessageKeyFor(msg), redisutil.MessageSigKeyFor(msg)).Result() + exists, err := c.RedisCoordinator().Client.Exists(ctx, redisutil.MessageKeyFor(msg), redisutil.MessageSigKeyFor(msg)).Result() if err != nil { // If there is an error deleting finalized messages during init, we retry later either from this sequencer or from another return err @@ -564,7 +578,7 @@ func (c *SeqCoordinator) deleteFinalizedMsgsFromRedis(ctx context.Context, final } else if err != nil { return fmt.Errorf("error getting finalizedMsgCount value from redis: %w", err) } - remoteMsgCount, err := c.getRemoteMsgCountImpl(ctx, c.Client) + remoteMsgCount, err := c.getRemoteMsgCountImpl(ctx, c.RedisCoordinator().Client) if err != nil { return fmt.Errorf("cannot get remote message count: %w", err) } @@ -580,7 +594,7 @@ func (c *SeqCoordinator) deleteFinalizedMsgsFromRedis(ctx context.Context, final } func (c *SeqCoordinator) update(ctx context.Context) time.Duration { - chosenSeq, err := c.RecommendSequencerWantingLockout(ctx) + chosenSeq, err := c.RedisCoordinator().RecommendSequencerWantingLockout(ctx) if err != nil { log.Warn("coordinator failed finding sequencer wanting lockout", "err", err) return c.retryAfterRedisError() @@ -632,7 +646,7 @@ func (c *SeqCoordinator) update(ctx context.Context) time.Duration { return c.retryAfterRedisError() } readUntil := min(localMsgCount+c.config.MsgPerPoll, remoteMsgCount) - client := c.Client + client := c.RedisCoordinator().Client // If we have a previous redis coordinator, // we can read from it until the local message count catches up to the prev coordinator's message count if c.prevRedisMessageCount > localMsgCount { @@ -744,7 +758,7 @@ func (c *SeqCoordinator) update(ctx context.Context) time.Duration { // this could be just new messages we didn't get yet - even then, we should retry soon log.Info("sequencer failed to become chosen", "err", err, "msgcount", localMsgCount) // make sure we're marked as wanting the lockout - if err := c.wantsLockoutUpdate(ctx, c.Client); err != nil { + if err := c.wantsLockoutUpdate(ctx, c.RedisCoordinator().Client); err != nil { log.Warn("failed to update wants lockout key", "err", err) } c.prevChosenSequencer = "" @@ -772,7 +786,7 @@ func (c *SeqCoordinator) update(ctx context.Context) time.Duration { // update wanting the lockout var wantsLockoutErr error if synced && !c.AvoidingLockout() { - wantsLockoutErr = c.wantsLockoutUpdate(ctx, c.Client) + wantsLockoutErr = c.wantsLockoutUpdate(ctx, c.RedisCoordinator().Client) } else { wantsLockoutErr = c.wantsLockoutRelease(ctx) } @@ -871,7 +885,7 @@ func (c *SeqCoordinator) trySwitchingRedis(ctx context.Context, newRedisCoordina if err != nil { return err } - current, err := c.Client.Get(ctx, redisutil.CHOSENSEQ_KEY).Result() + current, err := c.RedisCoordinator().Client.Get(ctx, redisutil.CHOSENSEQ_KEY).Result() var wasEmpty bool if errors.Is(err, redis.Nil) { wasEmpty = true @@ -883,12 +897,11 @@ func (c *SeqCoordinator) trySwitchingRedis(ctx context.Context, newRedisCoordina } // If the chosen key is set to switch, we need to switch to the new redis coordinator. if !wasEmpty && (current == redisutil.SWITCHED_REDIS) { - err = c.wantsLockoutUpdate(ctx, c.Client) + err = c.wantsLockoutUpdate(ctx, c.RedisCoordinator().Client) if err != nil { return err } - c.prevRedisCoordinator = &c.RedisCoordinator - c.RedisCoordinator = *newRedisCoordinator + c.setRedisCoordinator(newRedisCoordinator) } return nil } @@ -942,7 +955,7 @@ func (c *SeqCoordinator) StopAndWait() { time.Sleep(c.retryAfterRedisError()) } } - _ = c.Client.Close() + _ = c.RedisCoordinator().Client.Close() } func (c *SeqCoordinator) CurrentlyChosen() bool { @@ -984,7 +997,7 @@ func (c *SeqCoordinator) TryToHandoffChosenOne(ctx context.Context) bool { return !c.CurrentlyChosen() }) if success { - wantsLockout, err := c.RecommendSequencerWantingLockout(ctx) + wantsLockout, err := c.RedisCoordinator().RecommendSequencerWantingLockout(ctx) if err == nil { log.Info("released chosen one status; a new sequencer hopefully wants to acquire it", "delay", c.config.SafeShutdownDelay, "wantsLockout", wantsLockout) } else { @@ -1006,7 +1019,7 @@ func (c *SeqCoordinator) SeekLockout(ctx context.Context) { log.Info("seeking lockout", "myUrl", c.config.Url()) if c.sequencer.Synced() { // Even if this errors we still internally marked ourselves as wanting the lockout - err := c.wantsLockoutUpdateWithMutex(ctx, c.Client) + err := c.wantsLockoutUpdateWithMutex(ctx, c.RedisCoordinator().Client) if err != nil { log.Warn("failed to set wants lockout key in redis after seeking lockout again", "err", err) } diff --git a/arbnode/seq_coordinator_test.go b/arbnode/seq_coordinator_test.go index 6498543f3a..3f35011c20 100644 --- a/arbnode/seq_coordinator_test.go +++ b/arbnode/seq_coordinator_test.go @@ -125,7 +125,7 @@ func TestRedisSeqCoordinatorAtomic(t *testing.T) { redisCoordinator, err := redisutil.NewRedisCoordinator(config.RedisUrl) Require(t, err) coordinator := &SeqCoordinator{ - RedisCoordinator: *redisCoordinator, + redisCoordinator: *redisCoordinator, config: config, signer: nullSigner, } @@ -181,7 +181,7 @@ func TestSeqCoordinatorDeletesFinalizedMessages(t *testing.T) { redisCoordinator, err := redisutil.NewRedisCoordinator(config.RedisUrl) Require(t, err) coordinator := &SeqCoordinator{ - RedisCoordinator: *redisCoordinator, + redisCoordinator: *redisCoordinator, config: config, signer: nullSigner, } @@ -191,18 +191,18 @@ func TestSeqCoordinatorDeletesFinalizedMessages(t *testing.T) { msgBytes, err := coordinator.msgCountToSignedBytes(0) Require(t, err) for i := arbutil.MessageIndex(1); i <= 10; i++ { - err = coordinator.Client.Set(ctx, redisutil.MessageKeyFor(i), msgBytes, time.Hour).Err() + err = coordinator.RedisCoordinator().Client.Set(ctx, redisutil.MessageKeyFor(i), msgBytes, time.Hour).Err() Require(t, err) - err = coordinator.Client.Set(ctx, redisutil.MessageSigKeyFor(i), msgBytes, time.Hour).Err() + err = coordinator.RedisCoordinator().Client.Set(ctx, redisutil.MessageSigKeyFor(i), msgBytes, time.Hour).Err() Require(t, err) keys = append(keys, redisutil.MessageKeyFor(i), redisutil.MessageSigKeyFor(i)) } // Set msgCount key msgCountBytes, err := coordinator.msgCountToSignedBytes(11) Require(t, err) - err = coordinator.Client.Set(ctx, redisutil.MSG_COUNT_KEY, msgCountBytes, time.Hour).Err() + err = coordinator.RedisCoordinator().Client.Set(ctx, redisutil.MSG_COUNT_KEY, msgCountBytes, time.Hour).Err() Require(t, err) - exists, err := coordinator.Client.Exists(ctx, keys...).Result() + exists, err := coordinator.RedisCoordinator().Client.Exists(ctx, keys...).Result() Require(t, err) if exists != 20 { t.Fatal("couldn't find all messages and signatures in redis") @@ -213,7 +213,7 @@ func TestSeqCoordinatorDeletesFinalizedMessages(t *testing.T) { Require(t, err) // Check if messages and signatures were deleted successfully - exists, err = coordinator.Client.Exists(ctx, keys[:8]...).Result() + exists, err = coordinator.RedisCoordinator().Client.Exists(ctx, keys[:8]...).Result() Require(t, err) if exists != 0 { t.Fatal("finalized messages and signatures in range 1 to 4 were not deleted") @@ -229,7 +229,7 @@ func TestSeqCoordinatorDeletesFinalizedMessages(t *testing.T) { // Try deleting finalized messages when theres already a finalizedMsgCount err = coordinator.deleteFinalizedMsgsFromRedis(ctx, 7) Require(t, err) - exists, err = coordinator.Client.Exists(ctx, keys[8:12]...).Result() + exists, err = coordinator.RedisCoordinator().Client.Exists(ctx, keys[8:12]...).Result() Require(t, err) if exists != 0 { t.Fatal("finalized messages and signatures in range 5 to 6 were not deleted") @@ -241,7 +241,7 @@ func TestSeqCoordinatorDeletesFinalizedMessages(t *testing.T) { } // Check that non-finalized messages are still available in redis - exists, err = coordinator.Client.Exists(ctx, keys[12:]...).Result() + exists, err = coordinator.RedisCoordinator().Client.Exists(ctx, keys[12:]...).Result() Require(t, err) if exists != 8 { t.Fatal("non-finalized messages and signatures in range 7 to 10 are not fully available") diff --git a/system_tests/forwarder_test.go b/system_tests/forwarder_test.go index 57381ca84e..6a1d1c68d8 100644 --- a/system_tests/forwarder_test.go +++ b/system_tests/forwarder_test.go @@ -170,7 +170,7 @@ func waitForSequencerLockout(ctx context.Context, node *arbnode.Node, duration t case <-time.After(duration): return fmt.Errorf("no sequencer was chosen") default: - if c, err := node.SeqCoordinator.CurrentChosenSequencer(ctx); err == nil && c != "" { + if c, err := node.SeqCoordinator.RedisCoordinator().CurrentChosenSequencer(ctx); err == nil && c != "" { return nil } time.Sleep(100 * time.Millisecond) From d48f24aa49b6a376a68f54d8fcaef1789db8845c Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Tue, 8 Oct 2024 18:18:31 +0200 Subject: [PATCH 348/489] reacreatestate_rpc_test: don't prefix subtest with testcase name --- system_tests/recreatestate_rpc_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/system_tests/recreatestate_rpc_test.go b/system_tests/recreatestate_rpc_test.go index 2f6650c63e..22329a1be5 100644 --- a/system_tests/recreatestate_rpc_test.go +++ b/system_tests/recreatestate_rpc_test.go @@ -454,7 +454,7 @@ func TestSkippingSavingStateAndRecreatingAfterRestart(t *testing.T) { cacheConfig.BlockAge = 0 // use only Caching.BlockCount to keep only last N blocks in dirties cache, no matter how new they are runTestCase := func(t *testing.T, cacheConfig gethexec.CachingConfig, txes int) { - t.Run(fmt.Sprintf("TestSkippingSavingStateAndRecreatingAfterRestart-skip-blocks-%d-skip-gas-%d-txes-%d", cacheConfig.MaxNumberOfBlocksToSkipStateSaving, cacheConfig.MaxAmountOfGasToSkipStateSaving, txes), func(t *testing.T) { + t.Run(fmt.Sprintf("skip-blocks-%d-skip-gas-%d-txes-%d", cacheConfig.MaxNumberOfBlocksToSkipStateSaving, cacheConfig.MaxAmountOfGasToSkipStateSaving, txes), func(t *testing.T) { testSkippingSavingStateAndRecreatingAfterRestart(t, &cacheConfig, txes) }) } @@ -553,7 +553,7 @@ func TestGettingState(t *testing.T) { execConfig.Caching.BlockAge = 0 // use only Caching.BlockCount to keep only last N blocks in dirties cache, no matter how new they are execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 - t.Run("TestGettingStateForRPCFullNode", func(t *testing.T) { + t.Run("full-node", func(t *testing.T) { testGettingState(t, execConfig) }) @@ -567,7 +567,7 @@ func TestGettingState(t *testing.T) { execConfig.Caching.BlockAge = 0 // use only Caching.BlockCount to keep only last N blocks in dirties cache, no matter how new they are execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 - t.Run("TestGettingStateForRPCSparseArchiveNode", func(t *testing.T) { + t.Run("archive-node", func(t *testing.T) { testGettingState(t, execConfig) }) } From 115afdc4d5c9ebadb985a581e6e19a73b635a59b Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Wed, 9 Oct 2024 17:10:48 +0530 Subject: [PATCH 349/489] Jit prover should accept InputJSON format and execute a full block --- arbitrator/jit/src/machine.rs | 120 ++++++++++++++++++---------------- arbitrator/jit/src/main.rs | 5 ++ arbitrator/jit/src/prepare.rs | 73 +++++++++++++++++++++ system_tests/common_test.go | 6 +- system_tests/program_test.go | 8 ++- 5 files changed, 150 insertions(+), 62 deletions(-) create mode 100644 arbitrator/jit/src/prepare.rs diff --git a/arbitrator/jit/src/machine.rs b/arbitrator/jit/src/machine.rs index 02523f740a..0d74c74ef6 100644 --- a/arbitrator/jit/src/machine.rs +++ b/arbitrator/jit/src/machine.rs @@ -2,8 +2,8 @@ // For license information, see https://github.com/nitro/blob/master/LICENSE use crate::{ - arbcompress, caller_env::GoRuntimeState, program, socket, stylus_backend::CothreadHandler, - wasip1_stub, wavmio, Opts, + arbcompress, caller_env::GoRuntimeState, prepare::prepare_env, program, socket, + stylus_backend::CothreadHandler, wasip1_stub, wavmio, Opts, }; use arbutil::{Bytes32, Color, PreimageType}; use eyre::{bail, ErrReport, Result, WrapErr}; @@ -215,72 +215,76 @@ pub struct WasmEnv { impl WasmEnv { pub fn cli(opts: &Opts) -> Result { - let mut env = WasmEnv::default(); - env.process.forks = opts.forks; - env.process.debug = opts.debug; + if let Some(json_inputs) = opts.json_inputs.clone() { + prepare_env(json_inputs, opts.debug) + } else { + let mut env = WasmEnv::default(); + env.process.forks = opts.forks; + env.process.debug = opts.debug; - let mut inbox_position = opts.inbox_position; - let mut delayed_position = opts.delayed_inbox_position; + let mut inbox_position = opts.inbox_position; + let mut delayed_position = opts.delayed_inbox_position; - for path in &opts.inbox { - let mut msg = vec![]; - File::open(path)?.read_to_end(&mut msg)?; - env.sequencer_messages.insert(inbox_position, msg); - inbox_position += 1; - } - for path in &opts.delayed_inbox { - let mut msg = vec![]; - File::open(path)?.read_to_end(&mut msg)?; - env.delayed_messages.insert(delayed_position, msg); - delayed_position += 1; - } + for path in &opts.inbox { + let mut msg = vec![]; + File::open(path)?.read_to_end(&mut msg)?; + env.sequencer_messages.insert(inbox_position, msg); + inbox_position += 1; + } + for path in &opts.delayed_inbox { + let mut msg = vec![]; + File::open(path)?.read_to_end(&mut msg)?; + env.delayed_messages.insert(delayed_position, msg); + delayed_position += 1; + } - if let Some(path) = &opts.preimages { - let mut file = BufReader::new(File::open(path)?); - let mut preimages = Vec::new(); - let filename = path.to_string_lossy(); - loop { - let mut size_buf = [0u8; 8]; - match file.read_exact(&mut size_buf) { - Ok(()) => {} - Err(err) if err.kind() == ErrorKind::UnexpectedEof => break, - Err(err) => bail!("Failed to parse {filename}: {}", err), + if let Some(path) = &opts.preimages { + let mut file = BufReader::new(File::open(path)?); + let mut preimages = Vec::new(); + let filename = path.to_string_lossy(); + loop { + let mut size_buf = [0u8; 8]; + match file.read_exact(&mut size_buf) { + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::UnexpectedEof => break, + Err(err) => bail!("Failed to parse {filename}: {}", err), + } + let size = u64::from_le_bytes(size_buf) as usize; + let mut buf = vec![0u8; size]; + file.read_exact(&mut buf)?; + preimages.push(buf); + } + let keccak_preimages = env.preimages.entry(PreimageType::Keccak256).or_default(); + for preimage in preimages { + let mut hasher = Keccak256::new(); + hasher.update(&preimage); + let hash = hasher.finalize().into(); + keccak_preimages.insert(hash, preimage); } - let size = u64::from_le_bytes(size_buf) as usize; - let mut buf = vec![0u8; size]; - file.read_exact(&mut buf)?; - preimages.push(buf); - } - let keccak_preimages = env.preimages.entry(PreimageType::Keccak256).or_default(); - for preimage in preimages { - let mut hasher = Keccak256::new(); - hasher.update(&preimage); - let hash = hasher.finalize().into(); - keccak_preimages.insert(hash, preimage); } - } - fn parse_hex(arg: &Option, name: &str) -> Result { - match arg { - Some(arg) => { - let mut arg = arg.as_str(); - if arg.starts_with("0x") { - arg = &arg[2..]; + fn parse_hex(arg: &Option, name: &str) -> Result { + match arg { + Some(arg) => { + let mut arg = arg.as_str(); + if arg.starts_with("0x") { + arg = &arg[2..]; + } + let mut bytes32 = [0u8; 32]; + hex::decode_to_slice(arg, &mut bytes32) + .wrap_err_with(|| format!("failed to parse {} contents", name))?; + Ok(bytes32.into()) } - let mut bytes32 = [0u8; 32]; - hex::decode_to_slice(arg, &mut bytes32) - .wrap_err_with(|| format!("failed to parse {} contents", name))?; - Ok(bytes32.into()) + None => Ok(Bytes32::default()), } - None => Ok(Bytes32::default()), } - } - let last_block_hash = parse_hex(&opts.last_block_hash, "--last-block-hash")?; - let last_send_root = parse_hex(&opts.last_send_root, "--last-send-root")?; - env.small_globals = [opts.inbox_position, opts.position_within_message]; - env.large_globals = [last_block_hash, last_send_root]; - Ok(env) + let last_block_hash = parse_hex(&opts.last_block_hash, "--last-block-hash")?; + let last_send_root = parse_hex(&opts.last_send_root, "--last-send-root")?; + env.small_globals = [opts.inbox_position, opts.position_within_message]; + env.large_globals = [last_block_hash, last_send_root]; + Ok(env) + } } pub fn send_results(&mut self, error: Option, memory_used: Pages) { diff --git a/arbitrator/jit/src/main.rs b/arbitrator/jit/src/main.rs index e432dc215c..6e44500215 100644 --- a/arbitrator/jit/src/main.rs +++ b/arbitrator/jit/src/main.rs @@ -10,6 +10,7 @@ use structopt::StructOpt; mod arbcompress; mod caller_env; mod machine; +mod prepare; mod program; mod socket; mod stylus_backend; @@ -46,6 +47,10 @@ pub struct Opts { debug: bool, #[structopt(long)] require_success: bool, + // JSON inputs supercede any of the command-line inputs which could + // be specified in the JSON file. + #[structopt(long)] + json_inputs: Option, } fn main() -> Result<()> { diff --git a/arbitrator/jit/src/prepare.rs b/arbitrator/jit/src/prepare.rs new file mode 100644 index 0000000000..7cf46143cc --- /dev/null +++ b/arbitrator/jit/src/prepare.rs @@ -0,0 +1,73 @@ +// Copyright 2022-2024, Offchain Labs, Inc. +// For license information, see https://github.com/nitro/blob/master/LICENSE + +use crate::WasmEnv; +use arbutil::{Bytes32, PreimageType}; +use eyre::Ok; +use prover::parse_input::FileData; +use std::env; +use std::fs::File; +use std::io::BufReader; +use std::path::PathBuf; + +// local_target matches rawdb.LocalTarget() on the go side. +// While generating json_inputs file, one should make sure user_wasms map +// has entry for the system's arch that jit validation is being run on +pub fn local_target() -> String { + if env::consts::OS == "linux" { + match env::consts::ARCH { + "arm64" => "arm64".to_string(), + "amd64" => "amd64".to_string(), + _ => "host".to_string(), + } + } else { + "host".to_string() + } +} + +pub fn prepare_env(json_inputs: PathBuf, debug: bool) -> eyre::Result { + let file = File::open(json_inputs)?; + let reader = BufReader::new(file); + + let data = FileData::from_reader(reader)?; + + let mut env = WasmEnv::default(); + env.process.forks = false; // Should be set to false when using json_inputs + env.process.debug = debug; + + let block_hash: [u8; 32] = data.start_state.block_hash.try_into().unwrap(); + let block_hash: Bytes32 = block_hash.into(); + let send_root: [u8; 32] = data.start_state.send_root.try_into().unwrap(); + let send_root: Bytes32 = send_root.into(); + let bytes32_vals: [Bytes32; 2] = [block_hash, send_root]; + let u64_vals: [u64; 2] = [data.start_state.batch, data.start_state.pos_in_batch]; + env.small_globals = u64_vals; + env.large_globals = bytes32_vals; + + for batch_info in data.batch_info.iter() { + env.sequencer_messages + .insert(batch_info.number, batch_info.data_b64.clone()); + } + + if data.delayed_msg_nr != 0 && data.delayed_msg_b64.len() != 0 { + env.delayed_messages + .insert(data.delayed_msg_nr, data.delayed_msg_b64.clone()); + } + + for (ty, inner_map) in data.preimages_b64 { + let preimage_ty = PreimageType::try_from(ty as u8)?; + let map = env.preimages.entry(preimage_ty).or_default(); + for (hash, preimage) in inner_map { + map.insert(hash, preimage); + } + } + + if let Some(user_wasms) = data.user_wasms.get(&local_target()) { + for (module_hash, module_asm) in user_wasms.iter() { + env.module_asms + .insert(*module_hash, module_asm.as_vec().into()); + } + } + + Ok(env) +} diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 93c38b5eae..5667c4bcd7 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -1699,8 +1699,8 @@ func logParser[T any](t *testing.T, source string, name string) func(*types.Log) // recordBlock writes a json file with all of the data needed to validate a block. // -// This can be used as an input to the arbitrator prover to validate a block. -func recordBlock(t *testing.T, block uint64, builder *NodeBuilder) { +// This can be used as an input to the arbitrator's prover (target=rawdb.TargetWavm) and jit (target=rawdb.LocalTarget()) binaries to validate a block. +func recordBlock(t *testing.T, block uint64, builder *NodeBuilder, target ethdb.WasmTarget) { t.Helper() ctx := builder.ctx inboxPos := arbutil.MessageIndex(block) @@ -1716,7 +1716,7 @@ func recordBlock(t *testing.T, block uint64, builder *NodeBuilder) { } validationInputsWriter, err := inputs.NewWriter(inputs.WithSlug(t.Name())) Require(t, err) - inputJson, err := builder.L2.ConsensusNode.StatelessBlockValidator.ValidationInputsAt(ctx, inboxPos, rawdb.TargetWavm) + inputJson, err := builder.L2.ConsensusNode.StatelessBlockValidator.ValidationInputsAt(ctx, inboxPos, target) if err != nil { Fatal(t, "failed to get validation inputs", block, err) } diff --git a/system_tests/program_test.go b/system_tests/program_test.go index cf8cd72559..4e7fc8dfd8 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -425,7 +425,7 @@ func storageTest(t *testing.T, jit bool) { // Captures a block_input_.json file for the block that included the // storage write transaction. - recordBlock(t, receipt.BlockNumber.Uint64(), builder) + recordBlock(t, receipt.BlockNumber.Uint64(), builder, rawdb.TargetWavm) } func TestProgramTransientStorage(t *testing.T) { @@ -492,6 +492,12 @@ func transientStorageTest(t *testing.T, jit bool) { } validateBlocks(t, 7, jit, builder) + + // Captures a block_input_.json file for the block that included the storage + // related transaction. Has userwasms for WasmTarget recognized by jit binary + receipt, err := EnsureTxSucceeded(ctx, l2client, tx) + Require(t, err) + recordBlock(t, receipt.BlockNumber.Uint64(), builder, rawdb.LocalTarget()) } func TestProgramMath(t *testing.T) { From 6396a775ef12a9f0ff517b0f39a4f13ee02f0a0c Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Wed, 9 Oct 2024 14:03:06 +0200 Subject: [PATCH 350/489] count long term cache misses only when cache tag is 1 --- arbitrator/stylus/src/cache.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index 6192a30eff..5df4c62f7a 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -177,7 +177,10 @@ impl InitCache { cache.long_term_counters.hits += 1; return Some(data); } - cache.long_term_counters.misses += 1; + if long_term_tag == Self::ARBOS_TAG { + // only count misses only when we can expect to find the item in long term cache + cache.long_term_counters.misses += 1; + } // See if the item is in the LRU cache, promoting if so if let Some(item) = cache.lru.peek(&key).cloned() { From 3a6e8681b630cacaace4c8b9af98bf2d2a4cb459 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Wed, 9 Oct 2024 19:40:51 +0530 Subject: [PATCH 351/489] fix clippy error --- arbitrator/jit/src/prepare.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arbitrator/jit/src/prepare.rs b/arbitrator/jit/src/prepare.rs index 7cf46143cc..6c1b0b84de 100644 --- a/arbitrator/jit/src/prepare.rs +++ b/arbitrator/jit/src/prepare.rs @@ -49,7 +49,7 @@ pub fn prepare_env(json_inputs: PathBuf, debug: bool) -> eyre::Result { .insert(batch_info.number, batch_info.data_b64.clone()); } - if data.delayed_msg_nr != 0 && data.delayed_msg_b64.len() != 0 { + if data.delayed_msg_nr != 0 && !data.delayed_msg_b64.is_empty() { env.delayed_messages .insert(data.delayed_msg_nr, data.delayed_msg_b64.clone()); } From be58669225e8ecf604182a2d65c1d115cf12686d Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Thu, 10 Oct 2024 11:56:28 +0530 Subject: [PATCH 352/489] change implimentation and add CI step to execute arbitrator prover using block input json --- .github/workflows/ci.yml | 12 ++ arbitrator/prover/src/lib.rs | 36 ------ arbitrator/prover/src/parse_input.rs | 27 +--- system_tests/common_test.go | 12 +- system_tests/program_test.go | 5 +- .../validationinputjson_rustfiledata_test.go | 120 ------------------ validator/server_arb/machine.go | 14 -- 7 files changed, 26 insertions(+), 200 deletions(-) delete mode 100644 system_tests/validationinputjson_rustfiledata_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a944f08f40..0b5592dafc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -169,6 +169,18 @@ jobs: run: | echo "Running redis tests" >> full.log TEST_REDIS=redis://localhost:6379/0 gotestsum --format short-verbose -- -p 1 -run TestRedis ./arbnode/... ./system_tests/... -coverprofile=coverage-redis.txt -covermode=atomic -coverpkg=./... + + - name: create block input json file + if: matrix.test-mode == 'defaults' + run: | + echo "BLOCK_INPUT_JSON_PATH=$(pwd)/target/block_input.json" >> "$GITHUB_ENV" + ${{ github.workspace }}/.github/workflows/gotestsum.sh --run TestProgramStorage$ --count 1 + + - name: run arbitrator prover on block input json + if: matrix.test-mode == 'defaults' + run: | + make build-prover-bin + target/bin/prover target/machines/latest/machine.wavm.br -b --json-inputs=$BLOCK_INPUT_JSON_PATH - name: run challenge tests if: matrix.test-mode == 'challenge' diff --git a/arbitrator/prover/src/lib.rs b/arbitrator/prover/src/lib.rs index 993ed1b365..08473c2598 100644 --- a/arbitrator/prover/src/lib.rs +++ b/arbitrator/prover/src/lib.rs @@ -33,12 +33,9 @@ use machine::{ PreimageResolver, }; use once_cell::sync::OnceCell; -use parse_input::FileData; use static_assertions::const_assert_eq; use std::{ ffi::CStr, - fs::File, - io::BufReader, num::NonZeroUsize, os::raw::{c_char, c_int}, path::Path, @@ -87,39 +84,6 @@ pub unsafe extern "C" fn arbitrator_load_machine( } } -#[no_mangle] -pub unsafe extern "C" fn arbitrator_deserialize_and_serialize_file_data( - read_path: *const c_char, - write_path: *const c_char, -) -> c_int { - let read_path = cstr_to_string(read_path); - let write_path = cstr_to_string(write_path); - - let file = File::open(read_path); - let reader = match file { - Ok(file) => BufReader::new(file), - Err(err) => { - eprintln!("Failed to open read_path of FileData: {}", err); - return 1; - } - }; - let data = match FileData::from_reader(reader) { - Ok(data) => data, - Err(err) => { - eprintln!("Failed to deserialize FileData: {}", err); - return 2; - } - }; - - match data.write_to_file(&write_path) { - Ok(()) => 0, - Err(err) => { - eprintln!("Failed to serialize FileData: {}", err); - 3 - } - } -} - unsafe fn arbitrator_load_machine_impl( binary_path: *const c_char, library_paths: *const *const c_char, diff --git a/arbitrator/prover/src/parse_input.rs b/arbitrator/prover/src/parse_input.rs index edfaddf26f..fa7adb4c41 100644 --- a/arbitrator/prover/src/parse_input.rs +++ b/arbitrator/prover/src/parse_input.rs @@ -1,14 +1,12 @@ use arbutil::Bytes32; use serde::Deserialize; -use serde::Serialize; use serde_json; use serde_with::base64::Base64; use serde_with::As; use serde_with::DisplayFromStr; use std::{ collections::HashMap, - fs::File, - io::{self, BufRead, BufWriter}, + io::{self, BufRead}, }; /// prefixed_hex deserializes hex strings which are prefixed with `0x` @@ -18,7 +16,7 @@ use std::{ /// It is an error to use this deserializer on a string that does not /// begin with `0x`. mod prefixed_hex { - use serde::{self, Deserialize, Deserializer, Serializer}; + use serde::{self, Deserialize, Deserializer}; pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> where @@ -31,14 +29,6 @@ mod prefixed_hex { Err(serde::de::Error::custom("missing 0x prefix")) } } - - pub fn serialize(bytes: &Vec, serializer: S) -> Result - where - S: Serializer, - { - let hex_string = format!("0x{}", hex::encode(bytes)); - serializer.serialize_str(&hex_string) - } } #[derive(Debug)] @@ -71,7 +61,7 @@ impl From> for UserWasm { } } -#[derive(Debug, Clone, Deserialize, Serialize)] +#[derive(Debug, Clone, Deserialize)] #[serde(rename_all = "PascalCase")] pub struct BatchInfo { pub number: u64, @@ -79,7 +69,7 @@ pub struct BatchInfo { pub data_b64: Vec, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, Deserialize)] #[serde(rename_all = "PascalCase")] pub struct StartState { #[serde(with = "prefixed_hex")] @@ -98,7 +88,7 @@ pub struct StartState { /// /// Note: It is important to change this file whenever the go JSON /// serialization changes. -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, Deserialize)] #[serde(rename_all = "PascalCase")] pub struct FileData { pub id: u64, @@ -119,11 +109,4 @@ impl FileData { let data = serde_json::from_reader(&mut reader)?; Ok(data) } - - pub fn write_to_file(&self, file_path: &str) -> io::Result<()> { - let file = File::create(file_path)?; - let writer = BufWriter::new(file); - serde_json::to_writer_pretty(writer, &self)?; - Ok(()) - } } diff --git a/system_tests/common_test.go b/system_tests/common_test.go index dbb2b86f13..95e8883e6b 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -36,7 +36,6 @@ import ( "github.com/offchainlabs/nitro/util/headerreader" "github.com/offchainlabs/nitro/util/redisutil" "github.com/offchainlabs/nitro/util/signature" - "github.com/offchainlabs/nitro/validator/inputs" "github.com/offchainlabs/nitro/validator/server_api" "github.com/offchainlabs/nitro/validator/server_common" "github.com/offchainlabs/nitro/validator/valnode" @@ -1719,7 +1718,7 @@ func logParser[T any](t *testing.T, source string, name string) func(*types.Log) // recordBlock writes a json file with all of the data needed to validate a block. // // This can be used as an input to the arbitrator prover to validate a block. -func recordBlock(t *testing.T, block uint64, builder *NodeBuilder) { +func recordBlock(t *testing.T, block uint64, builder *NodeBuilder, blockInputJSONPath string) { t.Helper() ctx := builder.ctx inboxPos := arbutil.MessageIndex(block) @@ -1733,15 +1732,14 @@ func recordBlock(t *testing.T, block uint64, builder *NodeBuilder) { break } } - validationInputsWriter, err := inputs.NewWriter(inputs.WithSlug(t.Name())) - Require(t, err) inputJson, err := builder.L2.ConsensusNode.StatelessBlockValidator.ValidationInputsAt(ctx, inboxPos, rawdb.TargetWavm) if err != nil { Fatal(t, "failed to get validation inputs", block, err) } - if err := validationInputsWriter.Write(&inputJson); err != nil { - Fatal(t, "failed to write validation inputs", block, err) - } + contents, err := json.Marshal(inputJson) + Require(t, err) + err = os.WriteFile(blockInputJSONPath, contents, 0600) + Require(t, err) } func populateMachineDir(t *testing.T, cr *github.ConsensusRelease) string { diff --git a/system_tests/program_test.go b/system_tests/program_test.go index 4755096b26..d2c3336af7 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -425,7 +425,10 @@ func storageTest(t *testing.T, jit bool) { // Captures a block_input_.json file for the block that included the // storage write transaction. - recordBlock(t, receipt.BlockNumber.Uint64(), builder) + blockInputJSONPath := os.Getenv("BLOCK_INPUT_JSON_PATH") + if blockInputJSONPath != "" { + recordBlock(t, receipt.BlockNumber.Uint64(), builder, blockInputJSONPath) + } } func TestProgramTransientStorage(t *testing.T) { diff --git a/system_tests/validationinputjson_rustfiledata_test.go b/system_tests/validationinputjson_rustfiledata_test.go deleted file mode 100644 index a27a68b12c..0000000000 --- a/system_tests/validationinputjson_rustfiledata_test.go +++ /dev/null @@ -1,120 +0,0 @@ -package arbtest - -import ( - "encoding/base64" - "encoding/json" - "fmt" - "os" - "path/filepath" - "reflect" - "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/ethdb" - - "github.com/offchainlabs/nitro/arbutil" - "github.com/offchainlabs/nitro/validator" - "github.com/offchainlabs/nitro/validator/server_api" - "github.com/offchainlabs/nitro/validator/server_arb" -) - -func getWriteDataFromRustSide(t *testing.T, inputJSON *server_api.InputJSON) []byte { - t.Helper() - dir := t.TempDir() - - readData, err := inputJSON.Marshal() - Require(t, err) - readPath := filepath.Join(dir, fmt.Sprintf("block_inputs_%d_read.json", inputJSON.Id)) - Require(t, os.WriteFile(readPath, readData, 0600)) - - writePath := filepath.Join(dir, fmt.Sprintf("block_inputs_%d_write.json", inputJSON.Id)) - Require(t, server_arb.DeserializeAndSerializeFileData(readPath, writePath)) - writeData, err := os.ReadFile(writePath) - Require(t, err) - - return writeData -} - -func TestGoInputJSONRustFileDataRoundtripWithoutUserWasms(t *testing.T) { - preimages := make(map[arbutil.PreimageType]map[common.Hash][]byte) - preimages[arbutil.Keccak256PreimageType] = make(map[common.Hash][]byte) - preimages[arbutil.Keccak256PreimageType][common.MaxHash] = []byte{1} - - // Don't include DebugChain as it isn't used on rust side - sampleValidationInput := &validator.ValidationInput{ - Id: 1, - HasDelayedMsg: true, - DelayedMsgNr: 2, - Preimages: preimages, - BatchInfo: []validator.BatchInfo{{Number: 3}}, - DelayedMsg: []byte{4}, - StartState: validator.GoGlobalState{ - BlockHash: common.MaxHash, - SendRoot: common.MaxHash, - Batch: 5, - PosInBatch: 6, - }, - } - sampleValidationInputJSON := server_api.ValidationInputToJson(sampleValidationInput) - writeData := getWriteDataFromRustSide(t, sampleValidationInputJSON) - - var resWithoutUserWasms server_api.InputJSON - Require(t, json.Unmarshal(writeData, &resWithoutUserWasms)) - if !reflect.DeepEqual(*sampleValidationInputJSON, resWithoutUserWasms) { - t.Fatal("ValidationInputJSON without UserWasms, mismatch on rust and go side") - } - -} - -type inputJSONWithUserWasmsOnly struct { - UserWasms map[ethdb.WasmTarget]map[common.Hash][]byte -} - -// UnmarshalJSON is a custom function defined to encapsulate how UserWasms are handled on the rust side. -// When ValidationInputToJson is called on ValidationInput, it compresses the wasm data byte array and -// then encodes this to a base64 string, this when deserialized on the rust side through FileData- the -// compressed data is first uncompressed and also the module hash (Bytes32) is read without the 0x prefix, -// so we define a custom UnmarshalJSON to extract UserWasms map from the data written by rust side. -func (u *inputJSONWithUserWasmsOnly) UnmarshalJSON(data []byte) error { - type rawUserWasms struct { - UserWasms map[ethdb.WasmTarget]map[string]string - } - var rawUWs rawUserWasms - if err := json.Unmarshal(data, &rawUWs); err != nil { - return err - } - tmp := make(map[ethdb.WasmTarget]map[common.Hash][]byte) - for wasmTarget, innerMap := range rawUWs.UserWasms { - tmp[wasmTarget] = make(map[common.Hash][]byte) - for hashKey, value := range innerMap { - valBytes, err := base64.StdEncoding.DecodeString(value) - if err != nil { - return err - } - tmp[wasmTarget][common.HexToHash("0x"+hashKey)] = valBytes - } - } - u.UserWasms = tmp - return nil -} - -func TestGoInputJSONRustFileDataRoundtripWithUserWasms(t *testing.T) { - userWasms := make(map[ethdb.WasmTarget]map[common.Hash][]byte) - userWasms["arch1"] = make(map[common.Hash][]byte) - userWasms["arch1"][common.MaxHash] = []byte{2} - - // Don't include DebugChain as it isn't used on rust side - sampleValidationInput := &validator.ValidationInput{ - Id: 1, - UserWasms: userWasms, - BatchInfo: []validator.BatchInfo{{Number: 1}}, // This needs to be set for FileData to successfully deserialize, else it errors for invalid type null - } - sampleValidationInputJSON := server_api.ValidationInputToJson(sampleValidationInput) - writeData := getWriteDataFromRustSide(t, sampleValidationInputJSON) - - var resUserWasmsOnly inputJSONWithUserWasmsOnly - Require(t, json.Unmarshal(writeData, &resUserWasmsOnly)) - if !reflect.DeepEqual(userWasms, resUserWasmsOnly.UserWasms) { - t.Fatal("ValidationInputJSON with UserWasms only, mismatch on rust and go side") - } -} diff --git a/validator/server_arb/machine.go b/validator/server_arb/machine.go index d11f015916..1e73e6b212 100644 --- a/validator/server_arb/machine.go +++ b/validator/server_arb/machine.go @@ -312,20 +312,6 @@ func (m *ArbitratorMachine) DeserializeAndReplaceState(path string) error { } } -func DeserializeAndSerializeFileData(readPath, writePath string) error { - cReadPath := C.CString(readPath) - cWritePath := C.CString(writePath) - status := C.arbitrator_deserialize_and_serialize_file_data(cReadPath, cWritePath) - C.free(unsafe.Pointer(cReadPath)) - C.free(unsafe.Pointer(cWritePath)) - - if status != 0 { - return fmt.Errorf("failed to call arbitrator_deserialize_and_serialize_file_data. Error code: %d", status) - } else { - return nil - } -} - func (m *ArbitratorMachine) AddSequencerInboxMessage(index uint64, data []byte) error { defer runtime.KeepAlive(m) m.mutex.Lock() From f0d14fb2f681d4f10c4844bf83141cec1380d98d Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Thu, 10 Oct 2024 12:33:35 +0530 Subject: [PATCH 353/489] fix ci step and comment other steps in test(defaults) to speedup verification. will be undone after debug fix env variable setting impl fix gotestsum invocation fix gotestsum invocation fix gotestsum invocation --- .github/workflows/ci.yml | 45 ++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b5592dafc..bd69e8c905 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -140,21 +140,21 @@ jobs: echo "GOGC=80" >> "$GITHUB_ENV" echo "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" >> "$GITHUB_ENV" - - name: run tests without race detection and path state scheme - if: matrix.test-mode == 'defaults' - env: - TEST_STATE_SCHEME: path - run: | - echo "Running tests with Path Scheme" >> full.log - ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags cionly --timeout 20m --cover - - - name: run tests without race detection and hash state scheme - if: matrix.test-mode == 'defaults' - env: - TEST_STATE_SCHEME: hash - run: | - echo "Running tests with Hash Scheme" >> full.log - ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags cionly --timeout 20m + # - name: run tests without race detection and path state scheme + # if: matrix.test-mode == 'defaults' + # env: + # TEST_STATE_SCHEME: path + # run: | + # echo "Running tests with Path Scheme" >> full.log + # ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags cionly --timeout 20m --cover + + # - name: run tests without race detection and hash state scheme + # if: matrix.test-mode == 'defaults' + # env: + # TEST_STATE_SCHEME: hash + # run: | + # echo "Running tests with Hash Scheme" >> full.log + # ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags cionly --timeout 20m - name: run tests with race detection and hash state scheme if: matrix.test-mode == 'race' @@ -164,23 +164,22 @@ jobs: echo "Running tests with Hash Scheme" >> full.log ${{ github.workspace }}/.github/workflows/gotestsum.sh --race --timeout 30m - - name: run redis tests - if: matrix.test-mode == 'defaults' - run: | - echo "Running redis tests" >> full.log - TEST_REDIS=redis://localhost:6379/0 gotestsum --format short-verbose -- -p 1 -run TestRedis ./arbnode/... ./system_tests/... -coverprofile=coverage-redis.txt -covermode=atomic -coverpkg=./... + # - name: run redis tests + # if: matrix.test-mode == 'defaults' + # run: | + # echo "Running redis tests" >> full.log + # TEST_REDIS=redis://localhost:6379/0 gotestsum --format short-verbose -- -p 1 -run TestRedis ./arbnode/... ./system_tests/... -coverprofile=coverage-redis.txt -covermode=atomic -coverpkg=./... - name: create block input json file if: matrix.test-mode == 'defaults' run: | - echo "BLOCK_INPUT_JSON_PATH=$(pwd)/target/block_input.json" >> "$GITHUB_ENV" - ${{ github.workspace }}/.github/workflows/gotestsum.sh --run TestProgramStorage$ --count 1 + BLOCK_INPUT_JSON_PATH="${{ github.workspace }}/target/block_input.json" gotestsum --format short-verbose -- -run TestProgramStorage$ --count 1 - name: run arbitrator prover on block input json if: matrix.test-mode == 'defaults' run: | make build-prover-bin - target/bin/prover target/machines/latest/machine.wavm.br -b --json-inputs=$BLOCK_INPUT_JSON_PATH + target/bin/prover target/machines/latest/machine.wavm.br -b --json-inputs="${{ github.workspace }}/target/block_input.json" - name: run challenge tests if: matrix.test-mode == 'challenge' From d50926ed69c02028ddce01bcb32732d0324dd7a7 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Thu, 10 Oct 2024 13:44:18 +0530 Subject: [PATCH 354/489] fix gotestsum invocation --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd69e8c905..c194a4936b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -173,7 +173,7 @@ jobs: - name: create block input json file if: matrix.test-mode == 'defaults' run: | - BLOCK_INPUT_JSON_PATH="${{ github.workspace }}/target/block_input.json" gotestsum --format short-verbose -- -run TestProgramStorage$ --count 1 + BLOCK_INPUT_JSON_PATH="${{ github.workspace }}/target/block_input.json" gotestsum --format short-verbose -- -run TestProgramStorage$ ./system_tests/... --count 1 - name: run arbitrator prover on block input json if: matrix.test-mode == 'defaults' From a1cb06dfd95c4043234eef0c1eba0fd932af4459 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Thu, 10 Oct 2024 13:55:47 +0530 Subject: [PATCH 355/489] uncomment CI steps --- .github/workflows/ci.yml | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c194a4936b..dcb3b9f422 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -140,21 +140,21 @@ jobs: echo "GOGC=80" >> "$GITHUB_ENV" echo "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" >> "$GITHUB_ENV" - # - name: run tests without race detection and path state scheme - # if: matrix.test-mode == 'defaults' - # env: - # TEST_STATE_SCHEME: path - # run: | - # echo "Running tests with Path Scheme" >> full.log - # ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags cionly --timeout 20m --cover - - # - name: run tests without race detection and hash state scheme - # if: matrix.test-mode == 'defaults' - # env: - # TEST_STATE_SCHEME: hash - # run: | - # echo "Running tests with Hash Scheme" >> full.log - # ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags cionly --timeout 20m + - name: run tests without race detection and path state scheme + if: matrix.test-mode == 'defaults' + env: + TEST_STATE_SCHEME: path + run: | + echo "Running tests with Path Scheme" >> full.log + ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags cionly --timeout 20m --cover + + - name: run tests without race detection and hash state scheme + if: matrix.test-mode == 'defaults' + env: + TEST_STATE_SCHEME: hash + run: | + echo "Running tests with Hash Scheme" >> full.log + ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags cionly --timeout 20m - name: run tests with race detection and hash state scheme if: matrix.test-mode == 'race' @@ -164,11 +164,11 @@ jobs: echo "Running tests with Hash Scheme" >> full.log ${{ github.workspace }}/.github/workflows/gotestsum.sh --race --timeout 30m - # - name: run redis tests - # if: matrix.test-mode == 'defaults' - # run: | - # echo "Running redis tests" >> full.log - # TEST_REDIS=redis://localhost:6379/0 gotestsum --format short-verbose -- -p 1 -run TestRedis ./arbnode/... ./system_tests/... -coverprofile=coverage-redis.txt -covermode=atomic -coverpkg=./... + - name: run redis tests + if: matrix.test-mode == 'defaults' + run: | + echo "Running redis tests" >> full.log + TEST_REDIS=redis://localhost:6379/0 gotestsum --format short-verbose -- -p 1 -run TestRedis ./arbnode/... ./system_tests/... -coverprofile=coverage-redis.txt -covermode=atomic -coverpkg=./... - name: create block input json file if: matrix.test-mode == 'defaults' From 70c5b3f40c4851b682208cdb6a645a84eca25160 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Thu, 10 Oct 2024 14:56:05 +0530 Subject: [PATCH 356/489] add ci step to run jit binary on the block input json file --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dcb3b9f422..6722b38ed6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,6 +180,12 @@ jobs: run: | make build-prover-bin target/bin/prover target/machines/latest/machine.wavm.br -b --json-inputs="${{ github.workspace }}/target/block_input.json" + + - name: run jit prover on block input json + if: matrix.test-mode == 'defaults' + run: | + make build-jit + target/bin/jit --binary target/machines/latest/replay.wasm --cranelift --json-inputs="${{ github.workspace }}/target/block_input.json" - name: run challenge tests if: matrix.test-mode == 'challenge' From 62d76d08d653e846aff3b5275c207d9aa2223db3 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Thu, 10 Oct 2024 16:02:06 +0530 Subject: [PATCH 357/489] check if theres a local target mismatch on go and rust side --- .github/workflows/ci.yml | 52 +++++++++++++++++------------------ arbitrator/jit/src/prepare.rs | 2 +- system_tests/program_test.go | 1 + 3 files changed, 28 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6722b38ed6..d50fe5da59 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -140,21 +140,21 @@ jobs: echo "GOGC=80" >> "$GITHUB_ENV" echo "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" >> "$GITHUB_ENV" - - name: run tests without race detection and path state scheme - if: matrix.test-mode == 'defaults' - env: - TEST_STATE_SCHEME: path - run: | - echo "Running tests with Path Scheme" >> full.log - ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags cionly --timeout 20m --cover - - - name: run tests without race detection and hash state scheme - if: matrix.test-mode == 'defaults' - env: - TEST_STATE_SCHEME: hash - run: | - echo "Running tests with Hash Scheme" >> full.log - ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags cionly --timeout 20m + # - name: run tests without race detection and path state scheme + # if: matrix.test-mode == 'defaults' + # env: + # TEST_STATE_SCHEME: path + # run: | + # echo "Running tests with Path Scheme" >> full.log + # ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags cionly --timeout 20m --cover + + # - name: run tests without race detection and hash state scheme + # if: matrix.test-mode == 'defaults' + # env: + # TEST_STATE_SCHEME: hash + # run: | + # echo "Running tests with Hash Scheme" >> full.log + # ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags cionly --timeout 20m - name: run tests with race detection and hash state scheme if: matrix.test-mode == 'race' @@ -164,22 +164,22 @@ jobs: echo "Running tests with Hash Scheme" >> full.log ${{ github.workspace }}/.github/workflows/gotestsum.sh --race --timeout 30m - - name: run redis tests - if: matrix.test-mode == 'defaults' - run: | - echo "Running redis tests" >> full.log - TEST_REDIS=redis://localhost:6379/0 gotestsum --format short-verbose -- -p 1 -run TestRedis ./arbnode/... ./system_tests/... -coverprofile=coverage-redis.txt -covermode=atomic -coverpkg=./... + # - name: run redis tests + # if: matrix.test-mode == 'defaults' + # run: | + # echo "Running redis tests" >> full.log + # TEST_REDIS=redis://localhost:6379/0 gotestsum --format short-verbose -- -p 1 -run TestRedis ./arbnode/... ./system_tests/... -coverprofile=coverage-redis.txt -covermode=atomic -coverpkg=./... - name: create block input json file if: matrix.test-mode == 'defaults' run: | - BLOCK_INPUT_JSON_PATH="${{ github.workspace }}/target/block_input.json" gotestsum --format short-verbose -- -run TestProgramStorage$ ./system_tests/... --count 1 + BLOCK_INPUT_JSON_PATH="${{ github.workspace }}/target/block_input.json" gotestsum --format standard-verbose -- -run TestProgramStorage$ ./system_tests/... --count 1 - - name: run arbitrator prover on block input json - if: matrix.test-mode == 'defaults' - run: | - make build-prover-bin - target/bin/prover target/machines/latest/machine.wavm.br -b --json-inputs="${{ github.workspace }}/target/block_input.json" + # - name: run arbitrator prover on block input json + # if: matrix.test-mode == 'defaults' + # run: | + # make build-prover-bin + # target/bin/prover target/machines/latest/machine.wavm.br -b --json-inputs="${{ github.workspace }}/target/block_input.json" - name: run jit prover on block input json if: matrix.test-mode == 'defaults' diff --git a/arbitrator/jit/src/prepare.rs b/arbitrator/jit/src/prepare.rs index 6c1b0b84de..b766936f3e 100644 --- a/arbitrator/jit/src/prepare.rs +++ b/arbitrator/jit/src/prepare.rs @@ -68,6 +68,6 @@ pub fn prepare_env(json_inputs: PathBuf, debug: bool) -> eyre::Result { .insert(*module_hash, module_asm.as_vec().into()); } } - + eprintln!("localTarget {}", local_target()); Ok(env) } diff --git a/system_tests/program_test.go b/system_tests/program_test.go index 92aeddaf53..1f195630a6 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -429,6 +429,7 @@ func storageTest(t *testing.T, jit bool) { if blockInputJSONPath != "" { recordBlock(t, receipt.BlockNumber.Uint64(), builder, []ethdb.WasmTarget{rawdb.TargetWavm, rawdb.LocalTarget()}, blockInputJSONPath) } + log.Info("printing debug info", "localTarget", rawdb.LocalTarget()) } func TestProgramTransientStorage(t *testing.T) { From 93fca3d126de1e64fefaf1a1ce1223b5b0fbf12c Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Thu, 10 Oct 2024 16:17:02 +0530 Subject: [PATCH 358/489] add more debug info --- arbitrator/jit/src/prepare.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/arbitrator/jit/src/prepare.rs b/arbitrator/jit/src/prepare.rs index b766936f3e..68cdcd3479 100644 --- a/arbitrator/jit/src/prepare.rs +++ b/arbitrator/jit/src/prepare.rs @@ -68,6 +68,13 @@ pub fn prepare_env(json_inputs: PathBuf, debug: bool) -> eyre::Result { .insert(*module_hash, module_asm.as_vec().into()); } } + + eprintln!( + "env Info. OS- {}, Arch- {}", + env::consts::OS, + env::consts::ARCH + ); eprintln!("localTarget {}", local_target()); + Ok(env) } From 5e355bc623e30b1ccc787bead4c6161ccf301639 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Thu, 10 Oct 2024 16:33:10 +0530 Subject: [PATCH 359/489] check if jit proving succeeds in CI --- .github/workflows/ci.yml | 7 +++++-- arbitrator/jit/src/prepare.rs | 11 ++--------- system_tests/program_test.go | 1 - 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d50fe5da59..40215fa163 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -173,7 +173,7 @@ jobs: - name: create block input json file if: matrix.test-mode == 'defaults' run: | - BLOCK_INPUT_JSON_PATH="${{ github.workspace }}/target/block_input.json" gotestsum --format standard-verbose -- -run TestProgramStorage$ ./system_tests/... --count 1 + BLOCK_INPUT_JSON_PATH="${{ github.workspace }}/target/block_input.json" gotestsum --format short-verbose -- -run TestProgramStorage$ ./system_tests/... --count 1 # - name: run arbitrator prover on block input json # if: matrix.test-mode == 'defaults' @@ -185,7 +185,10 @@ jobs: if: matrix.test-mode == 'defaults' run: | make build-jit - target/bin/jit --binary target/machines/latest/replay.wasm --cranelift --json-inputs="${{ github.workspace }}/target/block_input.json" + if [ -n "$(target/bin/jit --binary target/machines/latest/replay.wasm --cranelift --json-inputs='${{ github.workspace }}/target/block_input.json')" ]; then + echo "Error: Command produced output." + exit 1 + fi - name: run challenge tests if: matrix.test-mode == 'challenge' diff --git a/arbitrator/jit/src/prepare.rs b/arbitrator/jit/src/prepare.rs index 68cdcd3479..e7a7ba0f4d 100644 --- a/arbitrator/jit/src/prepare.rs +++ b/arbitrator/jit/src/prepare.rs @@ -16,8 +16,8 @@ use std::path::PathBuf; pub fn local_target() -> String { if env::consts::OS == "linux" { match env::consts::ARCH { - "arm64" => "arm64".to_string(), - "amd64" => "amd64".to_string(), + "aarch64" => "arm64".to_string(), + "x86_64" => "amd64".to_string(), _ => "host".to_string(), } } else { @@ -69,12 +69,5 @@ pub fn prepare_env(json_inputs: PathBuf, debug: bool) -> eyre::Result { } } - eprintln!( - "env Info. OS- {}, Arch- {}", - env::consts::OS, - env::consts::ARCH - ); - eprintln!("localTarget {}", local_target()); - Ok(env) } diff --git a/system_tests/program_test.go b/system_tests/program_test.go index 1f195630a6..92aeddaf53 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -429,7 +429,6 @@ func storageTest(t *testing.T, jit bool) { if blockInputJSONPath != "" { recordBlock(t, receipt.BlockNumber.Uint64(), builder, []ethdb.WasmTarget{rawdb.TargetWavm, rawdb.LocalTarget()}, blockInputJSONPath) } - log.Info("printing debug info", "localTarget", rawdb.LocalTarget()) } func TestProgramTransientStorage(t *testing.T) { From 7259ae122d694f08b005d8efc0a461f8314d729e Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Thu, 10 Oct 2024 16:51:02 +0530 Subject: [PATCH 360/489] remove debug statements and comments --- .github/workflows/ci.yml | 50 ++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 40215fa163..f0f44744f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -140,21 +140,21 @@ jobs: echo "GOGC=80" >> "$GITHUB_ENV" echo "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" >> "$GITHUB_ENV" - # - name: run tests without race detection and path state scheme - # if: matrix.test-mode == 'defaults' - # env: - # TEST_STATE_SCHEME: path - # run: | - # echo "Running tests with Path Scheme" >> full.log - # ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags cionly --timeout 20m --cover - - # - name: run tests without race detection and hash state scheme - # if: matrix.test-mode == 'defaults' - # env: - # TEST_STATE_SCHEME: hash - # run: | - # echo "Running tests with Hash Scheme" >> full.log - # ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags cionly --timeout 20m + - name: run tests without race detection and path state scheme + if: matrix.test-mode == 'defaults' + env: + TEST_STATE_SCHEME: path + run: | + echo "Running tests with Path Scheme" >> full.log + ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags cionly --timeout 20m --cover + + - name: run tests without race detection and hash state scheme + if: matrix.test-mode == 'defaults' + env: + TEST_STATE_SCHEME: hash + run: | + echo "Running tests with Hash Scheme" >> full.log + ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags cionly --timeout 20m - name: run tests with race detection and hash state scheme if: matrix.test-mode == 'race' @@ -164,22 +164,22 @@ jobs: echo "Running tests with Hash Scheme" >> full.log ${{ github.workspace }}/.github/workflows/gotestsum.sh --race --timeout 30m - # - name: run redis tests - # if: matrix.test-mode == 'defaults' - # run: | - # echo "Running redis tests" >> full.log - # TEST_REDIS=redis://localhost:6379/0 gotestsum --format short-verbose -- -p 1 -run TestRedis ./arbnode/... ./system_tests/... -coverprofile=coverage-redis.txt -covermode=atomic -coverpkg=./... + - name: run redis tests + if: matrix.test-mode == 'defaults' + run: | + echo "Running redis tests" >> full.log + TEST_REDIS=redis://localhost:6379/0 gotestsum --format short-verbose -- -p 1 -run TestRedis ./arbnode/... ./system_tests/... -coverprofile=coverage-redis.txt -covermode=atomic -coverpkg=./... - name: create block input json file if: matrix.test-mode == 'defaults' run: | BLOCK_INPUT_JSON_PATH="${{ github.workspace }}/target/block_input.json" gotestsum --format short-verbose -- -run TestProgramStorage$ ./system_tests/... --count 1 - # - name: run arbitrator prover on block input json - # if: matrix.test-mode == 'defaults' - # run: | - # make build-prover-bin - # target/bin/prover target/machines/latest/machine.wavm.br -b --json-inputs="${{ github.workspace }}/target/block_input.json" + - name: run arbitrator prover on block input json + if: matrix.test-mode == 'defaults' + run: | + make build-prover-bin + target/bin/prover target/machines/latest/machine.wavm.br -b --json-inputs="${{ github.workspace }}/target/block_input.json" - name: run jit prover on block input json if: matrix.test-mode == 'defaults' From af974b8048399fd70b14c502df54ddc7191a81ac Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 10 Sep 2024 11:39:46 -0300 Subject: [PATCH 361/489] AllowPostingBatchWithOnlyBatchPostingReport config --- arbnode/batch_poster.go | 41 ++++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/arbnode/batch_poster.go b/arbnode/batch_poster.go index 44b360e76e..b022566846 100644 --- a/arbnode/batch_poster.go +++ b/arbnode/batch_poster.go @@ -154,24 +154,25 @@ type BatchPosterConfig struct { // Batch post polling interval. PollInterval time.Duration `koanf:"poll-interval" reload:"hot"` // Batch posting error delay. - ErrorDelay time.Duration `koanf:"error-delay" reload:"hot"` - CompressionLevel int `koanf:"compression-level" reload:"hot"` - DASRetentionPeriod time.Duration `koanf:"das-retention-period" reload:"hot"` - GasRefunderAddress string `koanf:"gas-refunder-address" reload:"hot"` - DataPoster dataposter.DataPosterConfig `koanf:"data-poster" reload:"hot"` - RedisUrl string `koanf:"redis-url"` - RedisLock redislock.SimpleCfg `koanf:"redis-lock" reload:"hot"` - ExtraBatchGas uint64 `koanf:"extra-batch-gas" reload:"hot"` - Post4844Blobs bool `koanf:"post-4844-blobs" reload:"hot"` - IgnoreBlobPrice bool `koanf:"ignore-blob-price" reload:"hot"` - ParentChainWallet genericconf.WalletConfig `koanf:"parent-chain-wallet"` - L1BlockBound string `koanf:"l1-block-bound" reload:"hot"` - L1BlockBoundBypass time.Duration `koanf:"l1-block-bound-bypass" reload:"hot"` - UseAccessLists bool `koanf:"use-access-lists" reload:"hot"` - GasEstimateBaseFeeMultipleBips arbmath.UBips `koanf:"gas-estimate-base-fee-multiple-bips"` - Dangerous BatchPosterDangerousConfig `koanf:"dangerous"` - ReorgResistanceMargin time.Duration `koanf:"reorg-resistance-margin" reload:"hot"` - CheckBatchCorrectness bool `koanf:"check-batch-correctness"` + ErrorDelay time.Duration `koanf:"error-delay" reload:"hot"` + CompressionLevel int `koanf:"compression-level" reload:"hot"` + DASRetentionPeriod time.Duration `koanf:"das-retention-period" reload:"hot"` + GasRefunderAddress string `koanf:"gas-refunder-address" reload:"hot"` + DataPoster dataposter.DataPosterConfig `koanf:"data-poster" reload:"hot"` + RedisUrl string `koanf:"redis-url"` + RedisLock redislock.SimpleCfg `koanf:"redis-lock" reload:"hot"` + ExtraBatchGas uint64 `koanf:"extra-batch-gas" reload:"hot"` + Post4844Blobs bool `koanf:"post-4844-blobs" reload:"hot"` + IgnoreBlobPrice bool `koanf:"ignore-blob-price" reload:"hot"` + ParentChainWallet genericconf.WalletConfig `koanf:"parent-chain-wallet"` + L1BlockBound string `koanf:"l1-block-bound" reload:"hot"` + L1BlockBoundBypass time.Duration `koanf:"l1-block-bound-bypass" reload:"hot"` + UseAccessLists bool `koanf:"use-access-lists" reload:"hot"` + GasEstimateBaseFeeMultipleBips arbmath.UBips `koanf:"gas-estimate-base-fee-multiple-bips"` + Dangerous BatchPosterDangerousConfig `koanf:"dangerous"` + ReorgResistanceMargin time.Duration `koanf:"reorg-resistance-margin" reload:"hot"` + CheckBatchCorrectness bool `koanf:"check-batch-correctness"` + AllowPostingBatchWithOnlyBatchPostingReport bool `koanf:"allow-posting-batch-with-only-batch-posting-report"` gasRefunder common.Address l1BlockBound l1BlockBound @@ -225,6 +226,7 @@ func BatchPosterConfigAddOptions(prefix string, f *pflag.FlagSet) { f.Uint64(prefix+".gas-estimate-base-fee-multiple-bips", uint64(DefaultBatchPosterConfig.GasEstimateBaseFeeMultipleBips), "for gas estimation, use this multiple of the basefee (measured in basis points) as the max fee per gas") f.Duration(prefix+".reorg-resistance-margin", DefaultBatchPosterConfig.ReorgResistanceMargin, "do not post batch if its within this duration from layer 1 minimum bounds. Requires l1-block-bound option not be set to \"ignore\"") f.Bool(prefix+".check-batch-correctness", DefaultBatchPosterConfig.CheckBatchCorrectness, "setting this to true will run the batch against an inbox multiplexer and verifies that it produces the correct set of messages") + f.Bool(prefix+".allow-posting-batch-with-only-batch-posting-report", DefaultBatchPosterConfig.AllowPostingBatchWithOnlyBatchPostingReport, "setting this to true will enable batch poster to post batches that only have a batch posting report message") redislock.AddConfigOptions(prefix+".redis-lock", f) dataposter.DataPosterConfigAddOptions(prefix+".data-poster", f, dataposter.DefaultDataPosterConfig) genericconf.WalletConfigAddOptions(prefix+".parent-chain-wallet", f, DefaultBatchPosterConfig.ParentChainWallet.Pathname) @@ -256,6 +258,7 @@ var DefaultBatchPosterConfig = BatchPosterConfig{ GasEstimateBaseFeeMultipleBips: arbmath.OneInUBips * 3 / 2, ReorgResistanceMargin: 10 * time.Minute, CheckBatchCorrectness: true, + AllowPostingBatchWithOnlyBatchPostingReport: false, } var DefaultBatchPosterL1WalletConfig = genericconf.WalletConfig{ @@ -1309,7 +1312,7 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) b.building.muxBackend.delayedInbox = append(b.building.muxBackend.delayedInbox, msg) } } - if msg.Message.Header.Kind != arbostypes.L1MessageType_BatchPostingReport { + if config.AllowPostingBatchWithOnlyBatchPostingReport || (msg.Message.Header.Kind != arbostypes.L1MessageType_BatchPostingReport) { b.building.haveUsefulMessage = true } b.building.msgCount++ From 52f6a6c59ef971471c7d4f66d43f62d66d7d7207 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 10 Oct 2024 11:36:31 -0300 Subject: [PATCH 362/489] PeriodToAllowPostingBatchWithOnlyBatchPostingReport config --- arbnode/batch_poster.go | 44 ++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/arbnode/batch_poster.go b/arbnode/batch_poster.go index b022566846..63175d48ac 100644 --- a/arbnode/batch_poster.go +++ b/arbnode/batch_poster.go @@ -154,25 +154,25 @@ type BatchPosterConfig struct { // Batch post polling interval. PollInterval time.Duration `koanf:"poll-interval" reload:"hot"` // Batch posting error delay. - ErrorDelay time.Duration `koanf:"error-delay" reload:"hot"` - CompressionLevel int `koanf:"compression-level" reload:"hot"` - DASRetentionPeriod time.Duration `koanf:"das-retention-period" reload:"hot"` - GasRefunderAddress string `koanf:"gas-refunder-address" reload:"hot"` - DataPoster dataposter.DataPosterConfig `koanf:"data-poster" reload:"hot"` - RedisUrl string `koanf:"redis-url"` - RedisLock redislock.SimpleCfg `koanf:"redis-lock" reload:"hot"` - ExtraBatchGas uint64 `koanf:"extra-batch-gas" reload:"hot"` - Post4844Blobs bool `koanf:"post-4844-blobs" reload:"hot"` - IgnoreBlobPrice bool `koanf:"ignore-blob-price" reload:"hot"` - ParentChainWallet genericconf.WalletConfig `koanf:"parent-chain-wallet"` - L1BlockBound string `koanf:"l1-block-bound" reload:"hot"` - L1BlockBoundBypass time.Duration `koanf:"l1-block-bound-bypass" reload:"hot"` - UseAccessLists bool `koanf:"use-access-lists" reload:"hot"` - GasEstimateBaseFeeMultipleBips arbmath.UBips `koanf:"gas-estimate-base-fee-multiple-bips"` - Dangerous BatchPosterDangerousConfig `koanf:"dangerous"` - ReorgResistanceMargin time.Duration `koanf:"reorg-resistance-margin" reload:"hot"` - CheckBatchCorrectness bool `koanf:"check-batch-correctness"` - AllowPostingBatchWithOnlyBatchPostingReport bool `koanf:"allow-posting-batch-with-only-batch-posting-report"` + ErrorDelay time.Duration `koanf:"error-delay" reload:"hot"` + CompressionLevel int `koanf:"compression-level" reload:"hot"` + DASRetentionPeriod time.Duration `koanf:"das-retention-period" reload:"hot"` + GasRefunderAddress string `koanf:"gas-refunder-address" reload:"hot"` + DataPoster dataposter.DataPosterConfig `koanf:"data-poster" reload:"hot"` + RedisUrl string `koanf:"redis-url"` + RedisLock redislock.SimpleCfg `koanf:"redis-lock" reload:"hot"` + ExtraBatchGas uint64 `koanf:"extra-batch-gas" reload:"hot"` + Post4844Blobs bool `koanf:"post-4844-blobs" reload:"hot"` + IgnoreBlobPrice bool `koanf:"ignore-blob-price" reload:"hot"` + ParentChainWallet genericconf.WalletConfig `koanf:"parent-chain-wallet"` + L1BlockBound string `koanf:"l1-block-bound" reload:"hot"` + L1BlockBoundBypass time.Duration `koanf:"l1-block-bound-bypass" reload:"hot"` + UseAccessLists bool `koanf:"use-access-lists" reload:"hot"` + GasEstimateBaseFeeMultipleBips arbmath.UBips `koanf:"gas-estimate-base-fee-multiple-bips"` + Dangerous BatchPosterDangerousConfig `koanf:"dangerous"` + ReorgResistanceMargin time.Duration `koanf:"reorg-resistance-margin" reload:"hot"` + CheckBatchCorrectness bool `koanf:"check-batch-correctness"` + PeriodToAllowPostingBatchWithOnlyBatchPostingReport time.Duration `koanf:"period-to-allow-posting-batch-with-only-batch-posting-report"` gasRefunder common.Address l1BlockBound l1BlockBound @@ -226,7 +226,7 @@ func BatchPosterConfigAddOptions(prefix string, f *pflag.FlagSet) { f.Uint64(prefix+".gas-estimate-base-fee-multiple-bips", uint64(DefaultBatchPosterConfig.GasEstimateBaseFeeMultipleBips), "for gas estimation, use this multiple of the basefee (measured in basis points) as the max fee per gas") f.Duration(prefix+".reorg-resistance-margin", DefaultBatchPosterConfig.ReorgResistanceMargin, "do not post batch if its within this duration from layer 1 minimum bounds. Requires l1-block-bound option not be set to \"ignore\"") f.Bool(prefix+".check-batch-correctness", DefaultBatchPosterConfig.CheckBatchCorrectness, "setting this to true will run the batch against an inbox multiplexer and verifies that it produces the correct set of messages") - f.Bool(prefix+".allow-posting-batch-with-only-batch-posting-report", DefaultBatchPosterConfig.AllowPostingBatchWithOnlyBatchPostingReport, "setting this to true will enable batch poster to post batches that only have a batch posting report message") + f.Duration(prefix+".period-to-allow-posting-batch-with-only-batch-posting-report", DefaultBatchPosterConfig.PeriodToAllowPostingBatchWithOnlyBatchPostingReport, "batch poster will only be able to post a batch with only batch posting report messages if this time period building a batch has passed") redislock.AddConfigOptions(prefix+".redis-lock", f) dataposter.DataPosterConfigAddOptions(prefix+".data-poster", f, dataposter.DefaultDataPosterConfig) genericconf.WalletConfigAddOptions(prefix+".parent-chain-wallet", f, DefaultBatchPosterConfig.ParentChainWallet.Pathname) @@ -258,7 +258,7 @@ var DefaultBatchPosterConfig = BatchPosterConfig{ GasEstimateBaseFeeMultipleBips: arbmath.OneInUBips * 3 / 2, ReorgResistanceMargin: 10 * time.Minute, CheckBatchCorrectness: true, - AllowPostingBatchWithOnlyBatchPostingReport: false, + PeriodToAllowPostingBatchWithOnlyBatchPostingReport: 3 * 24 * time.Hour, } var DefaultBatchPosterL1WalletConfig = genericconf.WalletConfig{ @@ -1312,7 +1312,7 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) b.building.muxBackend.delayedInbox = append(b.building.muxBackend.delayedInbox, msg) } } - if config.AllowPostingBatchWithOnlyBatchPostingReport || (msg.Message.Header.Kind != arbostypes.L1MessageType_BatchPostingReport) { + if (msg.Message.Header.Kind != arbostypes.L1MessageType_BatchPostingReport) || (time.Since(firstMsgTime) >= config.PeriodToAllowPostingBatchWithOnlyBatchPostingReport) { b.building.haveUsefulMessage = true } b.building.msgCount++ From 057fcb3e40619fda5c00cc45c7f86579703e8276 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Thu, 10 Oct 2024 20:34:24 +0530 Subject: [PATCH 363/489] Create a build diagnostic tool --- diagnose.sh | 132 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100755 diagnose.sh diff --git a/diagnose.sh b/diagnose.sh new file mode 100755 index 0000000000..8cfde2694f --- /dev/null +++ b/diagnose.sh @@ -0,0 +1,132 @@ +#!/bin/bash + +# Color codes +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Documentation link for installation instructions +INSTALLATION_DOCS_URL="Refer to https://docs.arbitrum.io/run-arbitrum-node/nitro/build-nitro-locally for installation." + +# Function to check if a command exists +command_exists() { + command -v "$1" >/dev/null 2>&1 +} + +# Detect operating system +OS=$(uname -s) +echo -e "${BLUE}Detected OS: $OS${NC}" +echo -e "${BLUE}Checking prerequisites for building Nitro locally...${NC}" + +# Step 1: Check Docker Installation +if command_exists docker; then + echo -e "${GREEN}Docker is installed.${NC}" +else + echo -e "${RED}Docker is not installed. $INSTALLATION_DOCS_URL${NC}" +fi + +# Step 2: Check if Docker service is running +if [[ "$OS" == "Linux" ]] && ! sudo service docker status >/dev/null; then + echo -e "${YELLOW}Docker service is not running on Linux. Start it with: sudo service docker start${NC}" +elif [[ "$OS" == "Darwin" ]] && ! docker info >/dev/null 2>&1; then + echo -e "${YELLOW}Docker service is not running on macOS. Ensure Docker Desktop is started.${NC}" +else + echo -e "${GREEN}Docker service is running.${NC}" +fi + +# Step 3: Check if the correct Nitro branch is checked out and submodules are updated +EXPECTED_BRANCH="v3.2.1" +CURRENT_BRANCH=$(git branch --show-current) +if [ "$CURRENT_BRANCH" != "$EXPECTED_BRANCH" ]; then + echo -e "${YELLOW}Switch to the correct branch using: git fetch origin && git checkout $EXPECTED_BRANCH${NC}" +else + echo -e "${GREEN}The Nitro repository is on the correct branch: $EXPECTED_BRANCH.${NC}" +fi + +# Check if submodules are properly initialized and updated +if git submodule status | grep -qE '^-|\+'; then + echo -e "${YELLOW}Submodules are not properly initialized or updated. Run: git submodule update --init --recursive --force${NC}" +else + echo -e "${GREEN}All submodules are properly initialized and up to date.${NC}" +fi + +# Step 4: Check if Nitro Docker Image is built +if docker images | grep -q "nitro-node"; then + echo -e "${GREEN}Nitro Docker image is built.${NC}" +else + echo -e "${RED}Nitro Docker image is not built. Build it using: docker build . --tag nitro-node${NC}" +fi + +# Step 5: Check prerequisites for building binaries +echo -e "${BLUE}Checking prerequisites for building Nitro's binaries...${NC}" +if [[ "$OS" == "Linux" ]]; then + prerequisites=(git curl build-essential cmake npm golang clang make gotestsum wasm2wat lld-13 python3 yarn) +else + prerequisites=(git curl make cmake npm go gvm golangci-lint wasm2wat clang gotestsum yarn) +fi + +for pkg in "${prerequisites[@]}"; do + if command_exists "$pkg"; then + [[ "$pkg" == "wasm2wat" ]] && pkg="wabt" + [[ "$pkg" == "clang" ]] && pkg="llvm" + + # Check for specific symbolic links related to wasm-ld on Linux and macOS + if [[ "$pkg" == "llvm" ]]; then + if [[ "$OS" == "Linux" ]]; then + if [ ! -L /usr/local/bin/wasm-ld ]; then + echo -e "${YELLOW}Creating symbolic link for wasm-ld on Linux.${NC}" + sudo ln -s /usr/bin/wasm-ld-13 /usr/local/bin/wasm-ld + else + echo -e "${GREEN}Symbolic link for wasm-ld on Linux is already present.${NC}" + fi + elif [[ "$OS" == "Darwin" ]]; then + if [ ! -L /usr/local/bin/wasm-ld ]; then + echo -e "${YELLOW}Creating symbolic link for wasm-ld on macOS.${NC}" + sudo mkdir -p /usr/local/bin + sudo ln -s /opt/homebrew/opt/llvm/bin/wasm-ld /usr/local/bin/wasm-ld + else + echo -e "${GREEN}Symbolic link for wasm-ld on macOS is already present.${NC}" + fi + fi + fi + + echo -e "${GREEN}$pkg is installed.${NC}" + else + [[ "$pkg" == "wasm2wat" ]] && pkg="wabt" + [[ "$pkg" == "clang" ]] && pkg="llvm" + echo -e "${RED}$pkg is not installed. Please install $pkg. $INSTALLATION_DOCS_URL${NC}" + fi +done + +# Step 6: Check Node.js version +if command_exists node && node -v | grep -q "v18"; then + echo -e "${GREEN}Node.js version 18 is installed.${NC}" +else + echo -e "${RED}Node.js version 18 not installed. $INSTALLATION_DOCS_URL${NC}" +fi + +# Step 7: Check Rust version +if command_exists rustc && rustc --version | grep -q "1.80.1"; then + echo -e "${GREEN}Rust version 1.80.1 is installed.${NC}" +else + echo -e "${RED}Rust version 1.80.1 not installed. $INSTALLATION_DOCS_URL${NC}" +fi + +# Step 8: Check Go version +if command_exists go && go version | grep -q "go1.23"; then + echo -e "${GREEN}Go version 1.23 is installed.${NC}" +else + echo -e "${RED}Go version 1.23 not installed. $INSTALLATION_DOCS_URL${NC}" +fi + +# Step 9: Check Foundry installation +if command_exists foundryup; then + echo -e "${GREEN}Foundry is installed.${NC}" +else + echo -e "${RED}Foundry is not installed. $INSTALLATION_DOCS_URL${NC}" +fi + +echo -e "${BLUE}Verification complete.${NC}" +echo -e "${YELLOW}Refer to https://docs.arbitrum.io/run-arbitrum-node/nitro/build-nitro-locally if the build fails for any other reason.${NC}" From 61348e4e570ffeced53714c1ae6141c4f7f46acf Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Thu, 10 Oct 2024 20:46:19 +0530 Subject: [PATCH 364/489] minor changes --- diagnose.sh => prerequisite_nitro_local_build.sh | 4 ++++ 1 file changed, 4 insertions(+) rename diagnose.sh => prerequisite_nitro_local_build.sh (93%) diff --git a/diagnose.sh b/prerequisite_nitro_local_build.sh similarity index 93% rename from diagnose.sh rename to prerequisite_nitro_local_build.sh index 8cfde2694f..915ca0ae42 100755 --- a/diagnose.sh +++ b/prerequisite_nitro_local_build.sh @@ -1,4 +1,5 @@ #!/bin/bash +# This script checks the prerequisites for building Arbitrum Nitro locally. # Color codes RED='\033[0;31m' @@ -69,6 +70,9 @@ fi for pkg in "${prerequisites[@]}"; do if command_exists "$pkg"; then + # There is no way to check for wabt / llvm directly, since they install multiple tools + # So instead, we check for wasm2wat and clang, which are part of wabt and llvm respectively + # and if they are installed, we assume wabt / llvm is installed else we ask the user to install wabt / llvm [[ "$pkg" == "wasm2wat" ]] && pkg="wabt" [[ "$pkg" == "clang" ]] && pkg="llvm" From bb5e8b7456a491901e68fa9e8e8213ec240648b9 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Thu, 10 Oct 2024 21:24:46 +0530 Subject: [PATCH 365/489] address PR comments --- arbnode/api.go | 2 +- staker/stateless_block_validator.go | 2 +- system_tests/common_test.go | 4 ++-- system_tests/program_test.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/arbnode/api.go b/arbnode/api.go index 8afb78770b..2dabd41bff 100644 --- a/arbnode/api.go +++ b/arbnode/api.go @@ -59,5 +59,5 @@ func (a *BlockValidatorDebugAPI) ValidateMessageNumber( func (a *BlockValidatorDebugAPI) ValidationInputsAt(ctx context.Context, msgNum hexutil.Uint64, target ethdb.WasmTarget, ) (server_api.InputJSON, error) { - return a.val.ValidationInputsAt(ctx, arbutil.MessageIndex(msgNum), []ethdb.WasmTarget{target}) + return a.val.ValidationInputsAt(ctx, arbutil.MessageIndex(msgNum), target) } diff --git a/staker/stateless_block_validator.go b/staker/stateless_block_validator.go index 786cf3aa94..d9c9c5446b 100644 --- a/staker/stateless_block_validator.go +++ b/staker/stateless_block_validator.go @@ -511,7 +511,7 @@ func (v *StatelessBlockValidator) ValidateResult( return true, &entry.End, nil } -func (v *StatelessBlockValidator) ValidationInputsAt(ctx context.Context, pos arbutil.MessageIndex, targets []ethdb.WasmTarget) (server_api.InputJSON, error) { +func (v *StatelessBlockValidator) ValidationInputsAt(ctx context.Context, pos arbutil.MessageIndex, targets ...ethdb.WasmTarget) (server_api.InputJSON, error) { entry, err := v.CreateReadyValidationEntry(ctx, pos) if err != nil { return server_api.InputJSON{}, err diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 287882655f..6bc3fe4af8 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -1718,7 +1718,7 @@ func logParser[T any](t *testing.T, source string, name string) func(*types.Log) // recordBlock writes a json file with all of the data needed to validate a block. // // This can be used as an input to the arbitrator prover to validate a block. -func recordBlock(t *testing.T, block uint64, builder *NodeBuilder, targets []ethdb.WasmTarget, blockInputJSONPath string) { +func recordBlock(t *testing.T, block uint64, builder *NodeBuilder, blockInputJSONPath string, targets ...ethdb.WasmTarget) { t.Helper() ctx := builder.ctx inboxPos := arbutil.MessageIndex(block) @@ -1732,7 +1732,7 @@ func recordBlock(t *testing.T, block uint64, builder *NodeBuilder, targets []eth break } } - inputJson, err := builder.L2.ConsensusNode.StatelessBlockValidator.ValidationInputsAt(ctx, inboxPos, targets) + inputJson, err := builder.L2.ConsensusNode.StatelessBlockValidator.ValidationInputsAt(ctx, inboxPos, targets...) if err != nil { Fatal(t, "failed to get validation inputs", block, err) } diff --git a/system_tests/program_test.go b/system_tests/program_test.go index 92aeddaf53..9d5b173fc3 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -427,7 +427,7 @@ func storageTest(t *testing.T, jit bool) { // storage write transaction. Include wasm targets necessary for arbitrator prover and jit binaries blockInputJSONPath := os.Getenv("BLOCK_INPUT_JSON_PATH") if blockInputJSONPath != "" { - recordBlock(t, receipt.BlockNumber.Uint64(), builder, []ethdb.WasmTarget{rawdb.TargetWavm, rawdb.LocalTarget()}, blockInputJSONPath) + recordBlock(t, receipt.BlockNumber.Uint64(), builder, blockInputJSONPath, rawdb.TargetWavm, rawdb.LocalTarget()) } } From f9e60d61728182cff0819dfb3e5009e5754db54b Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 10 Oct 2024 16:30:54 -0300 Subject: [PATCH 366/489] Fix memory leak when getting stylus cache metrics --- arbitrator/stylus/src/cache.rs | 36 ++++++++++++---------------------- arbitrator/stylus/src/lib.rs | 5 +++-- arbos/programs/native.go | 6 ++++-- 3 files changed, 19 insertions(+), 28 deletions(-) diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs index 5df4c62f7a..9b788a45db 100644 --- a/arbitrator/stylus/src/cache.rs +++ b/arbitrator/stylus/src/cache.rs @@ -275,28 +275,21 @@ impl InitCache { cache.long_term_size_bytes = 0; } - pub fn get_metrics() -> CacheMetrics { + pub fn get_metrics(output: &mut CacheMetrics) { let mut cache = cache!(); let lru_count = cache.lru.len(); - let lru_metrics = LruCacheMetrics { - // adds 1 to each entry to account that we subtracted 1 in the weight calculation - size_bytes: (cache.lru.weight() + lru_count).try_into().unwrap(), - - count: lru_count.try_into().unwrap(), - - hits: cache.lru_counters.hits, - misses: cache.lru_counters.misses, - does_not_fit: cache.lru_counters.does_not_fit, - }; - - let long_term_metrics = LongTermCacheMetrics { - size_bytes: cache.long_term_size_bytes.try_into().unwrap(), - count: cache.long_term.len().try_into().unwrap(), - - hits: cache.long_term_counters.hits, - misses: cache.long_term_counters.misses, - }; + // adds 1 to each entry to account that we subtracted 1 in the weight calculation + output.lru.size_bytes = (cache.lru.weight() + lru_count).try_into().unwrap(); + output.lru.count = lru_count.try_into().unwrap(); + output.lru.hits = cache.lru_counters.hits; + output.lru.misses = cache.lru_counters.misses; + output.lru.does_not_fit = cache.lru_counters.does_not_fit; + + output.long_term.size_bytes = cache.long_term_size_bytes.try_into().unwrap(); + output.long_term.count = cache.long_term.len().try_into().unwrap(); + output.long_term.hits = cache.long_term_counters.hits; + output.long_term.misses = cache.long_term_counters.misses; // Empty counters. // go side, which is the only consumer of this function besides tests, @@ -307,11 +300,6 @@ impl InitCache { does_not_fit: 0, }; cache.long_term_counters = LongTermCounters { hits: 0, misses: 0 }; - - CacheMetrics { - lru: lru_metrics, - long_term: long_term_metrics, - } } // only used for testing diff --git a/arbitrator/stylus/src/lib.rs b/arbitrator/stylus/src/lib.rs index a6be21f7bf..a2a4cd5059 100644 --- a/arbitrator/stylus/src/lib.rs +++ b/arbitrator/stylus/src/lib.rs @@ -366,8 +366,9 @@ pub unsafe extern "C" fn stylus_drop_vec(vec: RustBytes) { /// Gets cache metrics. #[no_mangle] -pub extern "C" fn stylus_get_cache_metrics() -> CacheMetrics { - InitCache::get_metrics() +pub unsafe extern "C" fn stylus_get_cache_metrics(output: *mut CacheMetrics) { + let output = &mut *output; + InitCache::get_metrics(output); } /// Clears lru cache. diff --git a/arbos/programs/native.go b/arbos/programs/native.go index e5c2632667..725b302ac0 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -339,7 +339,8 @@ func SetWasmLruCacheCapacity(capacityBytes uint64) { } func UpdateWasmCacheMetrics() { - metrics := C.stylus_get_cache_metrics() + metrics := &C.CacheMetrics{} + C.stylus_get_cache_metrics(metrics) stylusLRUCacheSizeBytesGauge.Update(int64(metrics.lru.size_bytes)) stylusLRUCacheCountGauge.Update(int64(metrics.lru.count)) @@ -373,7 +374,8 @@ type WasmCacheMetrics struct { // Used for testing func GetWasmCacheMetrics() *WasmCacheMetrics { - metrics := C.stylus_get_cache_metrics() + metrics := &C.CacheMetrics{} + C.stylus_get_cache_metrics(metrics) return &WasmCacheMetrics{ Lru: WasmLruCacheMetrics{ From c5625d69cb0ee1868f708acdc21c4977b05e3765 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 10 Oct 2024 16:51:21 -0300 Subject: [PATCH 367/489] Adds Safety comment to stylus_get_cache_metrics --- arbitrator/stylus/src/lib.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arbitrator/stylus/src/lib.rs b/arbitrator/stylus/src/lib.rs index a2a4cd5059..5962817d78 100644 --- a/arbitrator/stylus/src/lib.rs +++ b/arbitrator/stylus/src/lib.rs @@ -365,6 +365,10 @@ pub unsafe extern "C" fn stylus_drop_vec(vec: RustBytes) { } /// Gets cache metrics. +/// +/// # Safety +/// +/// `output` must not be null. #[no_mangle] pub unsafe extern "C" fn stylus_get_cache_metrics(output: *mut CacheMetrics) { let output = &mut *output; From a0be0412986c75f5f3a53bb24d34a9fa2867fb4a Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Fri, 11 Oct 2024 18:43:03 +0530 Subject: [PATCH 368/489] update submodule --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 72e4db5d66..64718d37c1 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 72e4db5d667da17ccb8b10e38e2e99e5b6b4c3d5 +Subproject commit 64718d37c11aee14a85a6cda740e094a6ceaea6c From 16cdbeb4d6d698ade4cf8d3627e0b2baf234d49d Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Fri, 11 Oct 2024 18:54:17 +0530 Subject: [PATCH 369/489] Trigger Build From f83a10fd3d98b25d831c8a94df74003f26b0dd35 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Fri, 11 Oct 2024 19:22:27 +0530 Subject: [PATCH 370/489] use inputs.Writer to generate the json file --- .github/workflows/ci.yml | 6 ++--- system_tests/common_test.go | 16 +++++++++---- system_tests/program_test.go | 9 ++++--- validator/inputs/writer.go | 46 +++++++++++++++++++++++++----------- 4 files changed, 52 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f0f44744f5..c1ca8aa698 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -173,19 +173,19 @@ jobs: - name: create block input json file if: matrix.test-mode == 'defaults' run: | - BLOCK_INPUT_JSON_PATH="${{ github.workspace }}/target/block_input.json" gotestsum --format short-verbose -- -run TestProgramStorage$ ./system_tests/... --count 1 + gotestsum --format short-verbose -- -run TestProgramStorage$ ./system_tests/... --count 1 --validator.inputswriter.basedir=""${{ github.workspace }}/target" - name: run arbitrator prover on block input json if: matrix.test-mode == 'defaults' run: | make build-prover-bin - target/bin/prover target/machines/latest/machine.wavm.br -b --json-inputs="${{ github.workspace }}/target/block_input.json" + target/bin/prover target/machines/latest/machine.wavm.br -b --json-inputs="${{ github.workspace }}/target/block_inputs.json" - name: run jit prover on block input json if: matrix.test-mode == 'defaults' run: | make build-jit - if [ -n "$(target/bin/jit --binary target/machines/latest/replay.wasm --cranelift --json-inputs='${{ github.workspace }}/target/block_input.json')" ]; then + if [ -n "$(target/bin/jit --binary target/machines/latest/replay.wasm --cranelift --json-inputs='${{ github.workspace }}/target/block_inputs.json')" ]; then echo "Error: Command produced output." exit 1 fi diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 6bc3fe4af8..189e83c965 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -36,6 +36,7 @@ import ( "github.com/offchainlabs/nitro/util/headerreader" "github.com/offchainlabs/nitro/util/redisutil" "github.com/offchainlabs/nitro/util/signature" + "github.com/offchainlabs/nitro/validator/inputs" "github.com/offchainlabs/nitro/validator/server_api" "github.com/offchainlabs/nitro/validator/server_common" "github.com/offchainlabs/nitro/validator/valnode" @@ -1718,7 +1719,7 @@ func logParser[T any](t *testing.T, source string, name string) func(*types.Log) // recordBlock writes a json file with all of the data needed to validate a block. // // This can be used as an input to the arbitrator prover to validate a block. -func recordBlock(t *testing.T, block uint64, builder *NodeBuilder, blockInputJSONPath string, targets ...ethdb.WasmTarget) { +func recordBlock(t *testing.T, block uint64, builder *NodeBuilder, baseDirectory string, targets ...ethdb.WasmTarget) { t.Helper() ctx := builder.ctx inboxPos := arbutil.MessageIndex(block) @@ -1732,14 +1733,19 @@ func recordBlock(t *testing.T, block uint64, builder *NodeBuilder, blockInputJSO break } } + validationInputsWriter, err := inputs.NewWriter( + inputs.WithBaseDir(baseDirectory), + inputs.WithTimestampDirEnabled(false), + inputs.WithBlockIdInFileNameEnabled(false), + ) + Require(t, err) inputJson, err := builder.L2.ConsensusNode.StatelessBlockValidator.ValidationInputsAt(ctx, inboxPos, targets...) if err != nil { Fatal(t, "failed to get validation inputs", block, err) } - contents, err := json.Marshal(inputJson) - Require(t, err) - err = os.WriteFile(blockInputJSONPath, contents, 0600) - Require(t, err) + if err := validationInputsWriter.Write(&inputJson); err != nil { + Fatal(t, "failed to write validation inputs", block, err) + } } func populateMachineDir(t *testing.T, cr *github.ConsensusRelease) string { diff --git a/system_tests/program_test.go b/system_tests/program_test.go index 9d5b173fc3..d4c64d7436 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -7,6 +7,7 @@ import ( "bytes" "context" "encoding/binary" + "flag" "fmt" "math" "math/big" @@ -393,6 +394,8 @@ func errorTest(t *testing.T, jit bool) { validateBlocks(t, 7, jit, builder) } +var validatorInputsWriterBaseDir = flag.String("validator.inputswriter.basedir", "", "Base directory for validationInputsWriter") + func TestProgramStorage(t *testing.T) { t.Parallel() storageTest(t, true) @@ -425,9 +428,9 @@ func storageTest(t *testing.T, jit bool) { // Captures a block_input_.json file for the block that included the // storage write transaction. Include wasm targets necessary for arbitrator prover and jit binaries - blockInputJSONPath := os.Getenv("BLOCK_INPUT_JSON_PATH") - if blockInputJSONPath != "" { - recordBlock(t, receipt.BlockNumber.Uint64(), builder, blockInputJSONPath, rawdb.TargetWavm, rawdb.LocalTarget()) + flag.Parse() + if *validatorInputsWriterBaseDir != "" { + recordBlock(t, receipt.BlockNumber.Uint64(), builder, *validatorInputsWriterBaseDir, rawdb.TargetWavm, rawdb.LocalTarget()) } } diff --git a/validator/inputs/writer.go b/validator/inputs/writer.go index a45e584f52..fc7456b8bf 100644 --- a/validator/inputs/writer.go +++ b/validator/inputs/writer.go @@ -17,20 +17,25 @@ import ( // // The path can be nested under a slug directory so callers can provide a // recognizable name to differentiate various contexts in which the InputJSON -// is being written. If the Writer is configured by calling SetSlug, then the +// is being written. If the Writer is configured by calling WithSlug, then the // path will be like: // // $HOME/.arbuitrum/validation-inputs///block_inputs_.json // +// The inclusion of BlockId in the file's name is on by default, however that can be disabled +// by calling WithBlockIdInFileNameEnabled(false). In which case, the path will be like: +// +// $HOME/.arbuitrum/validation-inputs///block_inputs.json +// // The inclusion of a timestamp directory is on by default to avoid conflicts which // would result in files being overwritten. However, the Writer can be configured // to not use a timestamp directory. If the Writer is configured by calling -// SetUseTimestampDir(false), then the path will be like: +// WithTimestampDirEnabled(false), then the path will be like: // // $HOME/.arbuitrum/validation-inputs//block_inputs_.json // // Finally, to give complete control to the clients, the base directory can be -// set directly with SetBaseDir. In which case, the path will be like: +// set directly with WithBaseDir. In which case, the path will be like: // // /block_inputs_.json // or @@ -38,10 +43,11 @@ import ( // or // ///block_inputs_.json type Writer struct { - clock Clock - baseDir string - slug string - useTimestampDir bool + clock Clock + baseDir string + slug string + useTimestampDir bool + useBlockIdInFileName bool } // WriterOption is a function that configures a Writer. @@ -66,10 +72,11 @@ func NewWriter(options ...WriterOption) (*Writer, error) { } baseDir := filepath.Join(homeDir, ".arbitrum", "validation-inputs") w := &Writer{ - clock: realClock{}, - baseDir: baseDir, - slug: "", - useTimestampDir: true, + clock: realClock{}, + baseDir: baseDir, + slug: "", + useTimestampDir: true, + useBlockIdInFileName: true, } for _, o := range options { o(w) @@ -114,6 +121,13 @@ func WithTimestampDirEnabled(useTimestampDir bool) WriterOption { } } +// WithBlockIdInFileNameEnabled controls the inclusion of Block Id in the input json file's name +func WithBlockIdInFileNameEnabled(useBlockIdInFileName bool) WriterOption { + return func(w *Writer) { + w.useBlockIdInFileName = useBlockIdInFileName + } +} + // Write writes the given InputJSON to a file in JSON format. func (w *Writer) Write(json *server_api.InputJSON) error { dir := w.baseDir @@ -132,9 +146,13 @@ func (w *Writer) Write(json *server_api.InputJSON) error { if err != nil { return err } - if err = os.WriteFile( - filepath.Join(dir, fmt.Sprintf("block_inputs_%d.json", json.Id)), - contents, 0600); err != nil { + var fileName string + if w.useBlockIdInFileName { + fileName = filepath.Join(dir, fmt.Sprintf("block_inputs_%d.json", json.Id)) + } else { + fileName = filepath.Join(dir, "block_inputs.json") + } + if err = os.WriteFile(fileName, contents, 0600); err != nil { return err } return nil From 05be76e4686254d9070884a6f221b535ff75e2de Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Fri, 11 Oct 2024 19:31:39 +0530 Subject: [PATCH 371/489] use inputs.Writer to generate the json file --- .github/workflows/ci.yml | 4 ++-- system_tests/common_test.go | 16 +++++++++---- system_tests/program_test.go | 11 +++++---- validator/inputs/writer.go | 46 +++++++++++++++++++++++++----------- 4 files changed, 52 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dcb3b9f422..671615233a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -173,13 +173,13 @@ jobs: - name: create block input json file if: matrix.test-mode == 'defaults' run: | - BLOCK_INPUT_JSON_PATH="${{ github.workspace }}/target/block_input.json" gotestsum --format short-verbose -- -run TestProgramStorage$ ./system_tests/... --count 1 + gotestsum --format short-verbose -- -run TestProgramStorage$ ./system_tests/... --count 1 --validator.inputswriter.basedir=""${{ github.workspace }}/target" - name: run arbitrator prover on block input json if: matrix.test-mode == 'defaults' run: | make build-prover-bin - target/bin/prover target/machines/latest/machine.wavm.br -b --json-inputs="${{ github.workspace }}/target/block_input.json" + target/bin/prover target/machines/latest/machine.wavm.br -b --json-inputs="${{ github.workspace }}/target/block_inputs.json" - name: run challenge tests if: matrix.test-mode == 'challenge' diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 95e8883e6b..f4e2edad42 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -36,6 +36,7 @@ import ( "github.com/offchainlabs/nitro/util/headerreader" "github.com/offchainlabs/nitro/util/redisutil" "github.com/offchainlabs/nitro/util/signature" + "github.com/offchainlabs/nitro/validator/inputs" "github.com/offchainlabs/nitro/validator/server_api" "github.com/offchainlabs/nitro/validator/server_common" "github.com/offchainlabs/nitro/validator/valnode" @@ -1718,7 +1719,7 @@ func logParser[T any](t *testing.T, source string, name string) func(*types.Log) // recordBlock writes a json file with all of the data needed to validate a block. // // This can be used as an input to the arbitrator prover to validate a block. -func recordBlock(t *testing.T, block uint64, builder *NodeBuilder, blockInputJSONPath string) { +func recordBlock(t *testing.T, block uint64, builder *NodeBuilder, baseDirectory string) { t.Helper() ctx := builder.ctx inboxPos := arbutil.MessageIndex(block) @@ -1732,14 +1733,19 @@ func recordBlock(t *testing.T, block uint64, builder *NodeBuilder, blockInputJSO break } } + validationInputsWriter, err := inputs.NewWriter( + inputs.WithBaseDir(baseDirectory), + inputs.WithTimestampDirEnabled(false), + inputs.WithBlockIdInFileNameEnabled(false), + ) + Require(t, err) inputJson, err := builder.L2.ConsensusNode.StatelessBlockValidator.ValidationInputsAt(ctx, inboxPos, rawdb.TargetWavm) if err != nil { Fatal(t, "failed to get validation inputs", block, err) } - contents, err := json.Marshal(inputJson) - Require(t, err) - err = os.WriteFile(blockInputJSONPath, contents, 0600) - Require(t, err) + if err := validationInputsWriter.Write(&inputJson); err != nil { + Fatal(t, "failed to write validation inputs", block, err) + } } func populateMachineDir(t *testing.T, cr *github.ConsensusRelease) string { diff --git a/system_tests/program_test.go b/system_tests/program_test.go index d2c3336af7..d007a40ddd 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -7,6 +7,7 @@ import ( "bytes" "context" "encoding/binary" + "flag" "fmt" "math" "math/big" @@ -393,6 +394,8 @@ func errorTest(t *testing.T, jit bool) { validateBlocks(t, 7, jit, builder) } +var validatorInputsWriterBaseDir = flag.String("validator.inputswriter.basedir", "", "Base directory for validationInputsWriter") + func TestProgramStorage(t *testing.T) { t.Parallel() storageTest(t, true) @@ -424,10 +427,10 @@ func storageTest(t *testing.T, jit bool) { validateBlocks(t, 2, jit, builder) // Captures a block_input_.json file for the block that included the - // storage write transaction. - blockInputJSONPath := os.Getenv("BLOCK_INPUT_JSON_PATH") - if blockInputJSONPath != "" { - recordBlock(t, receipt.BlockNumber.Uint64(), builder, blockInputJSONPath) + // storage write transaction. Include wasm targets necessary for arbitrator prover and jit binaries + flag.Parse() + if *validatorInputsWriterBaseDir != "" { + recordBlock(t, receipt.BlockNumber.Uint64(), builder, *validatorInputsWriterBaseDir) } } diff --git a/validator/inputs/writer.go b/validator/inputs/writer.go index a45e584f52..fc7456b8bf 100644 --- a/validator/inputs/writer.go +++ b/validator/inputs/writer.go @@ -17,20 +17,25 @@ import ( // // The path can be nested under a slug directory so callers can provide a // recognizable name to differentiate various contexts in which the InputJSON -// is being written. If the Writer is configured by calling SetSlug, then the +// is being written. If the Writer is configured by calling WithSlug, then the // path will be like: // // $HOME/.arbuitrum/validation-inputs///block_inputs_.json // +// The inclusion of BlockId in the file's name is on by default, however that can be disabled +// by calling WithBlockIdInFileNameEnabled(false). In which case, the path will be like: +// +// $HOME/.arbuitrum/validation-inputs///block_inputs.json +// // The inclusion of a timestamp directory is on by default to avoid conflicts which // would result in files being overwritten. However, the Writer can be configured // to not use a timestamp directory. If the Writer is configured by calling -// SetUseTimestampDir(false), then the path will be like: +// WithTimestampDirEnabled(false), then the path will be like: // // $HOME/.arbuitrum/validation-inputs//block_inputs_.json // // Finally, to give complete control to the clients, the base directory can be -// set directly with SetBaseDir. In which case, the path will be like: +// set directly with WithBaseDir. In which case, the path will be like: // // /block_inputs_.json // or @@ -38,10 +43,11 @@ import ( // or // ///block_inputs_.json type Writer struct { - clock Clock - baseDir string - slug string - useTimestampDir bool + clock Clock + baseDir string + slug string + useTimestampDir bool + useBlockIdInFileName bool } // WriterOption is a function that configures a Writer. @@ -66,10 +72,11 @@ func NewWriter(options ...WriterOption) (*Writer, error) { } baseDir := filepath.Join(homeDir, ".arbitrum", "validation-inputs") w := &Writer{ - clock: realClock{}, - baseDir: baseDir, - slug: "", - useTimestampDir: true, + clock: realClock{}, + baseDir: baseDir, + slug: "", + useTimestampDir: true, + useBlockIdInFileName: true, } for _, o := range options { o(w) @@ -114,6 +121,13 @@ func WithTimestampDirEnabled(useTimestampDir bool) WriterOption { } } +// WithBlockIdInFileNameEnabled controls the inclusion of Block Id in the input json file's name +func WithBlockIdInFileNameEnabled(useBlockIdInFileName bool) WriterOption { + return func(w *Writer) { + w.useBlockIdInFileName = useBlockIdInFileName + } +} + // Write writes the given InputJSON to a file in JSON format. func (w *Writer) Write(json *server_api.InputJSON) error { dir := w.baseDir @@ -132,9 +146,13 @@ func (w *Writer) Write(json *server_api.InputJSON) error { if err != nil { return err } - if err = os.WriteFile( - filepath.Join(dir, fmt.Sprintf("block_inputs_%d.json", json.Id)), - contents, 0600); err != nil { + var fileName string + if w.useBlockIdInFileName { + fileName = filepath.Join(dir, fmt.Sprintf("block_inputs_%d.json", json.Id)) + } else { + fileName = filepath.Join(dir, "block_inputs.json") + } + if err = os.WriteFile(fileName, contents, 0600); err != nil { return err } return nil From ceb785ef3237e85ce6a036855034d2be2682574a Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Fri, 11 Oct 2024 20:12:46 +0530 Subject: [PATCH 372/489] typo fix --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 671615233a..660f9f4dea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -173,7 +173,7 @@ jobs: - name: create block input json file if: matrix.test-mode == 'defaults' run: | - gotestsum --format short-verbose -- -run TestProgramStorage$ ./system_tests/... --count 1 --validator.inputswriter.basedir=""${{ github.workspace }}/target" + gotestsum --format short-verbose -- -run TestProgramStorage$ ./system_tests/... --count 1 --validator.inputswriter.basedir="${{ github.workspace }}/target" - name: run arbitrator prover on block input json if: matrix.test-mode == 'defaults' From 2dcc837ce179351bf4215c6a70eee9e2f3cbf358 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Fri, 11 Oct 2024 21:44:32 +0530 Subject: [PATCH 373/489] Changes based on PR comments --- .../check-build.sh | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) rename prerequisite_nitro_local_build.sh => util/check-build.sh (86%) diff --git a/prerequisite_nitro_local_build.sh b/util/check-build.sh similarity index 86% rename from prerequisite_nitro_local_build.sh rename to util/check-build.sh index 915ca0ae42..755cfda9ad 100755 --- a/prerequisite_nitro_local_build.sh +++ b/util/check-build.sh @@ -26,29 +26,28 @@ if command_exists docker; then echo -e "${GREEN}Docker is installed.${NC}" else echo -e "${RED}Docker is not installed. $INSTALLATION_DOCS_URL${NC}" + exit 1 fi # Step 2: Check if Docker service is running if [[ "$OS" == "Linux" ]] && ! sudo service docker status >/dev/null; then echo -e "${YELLOW}Docker service is not running on Linux. Start it with: sudo service docker start${NC}" + exit 1 elif [[ "$OS" == "Darwin" ]] && ! docker info >/dev/null 2>&1; then echo -e "${YELLOW}Docker service is not running on macOS. Ensure Docker Desktop is started.${NC}" + exit 1 else echo -e "${GREEN}Docker service is running.${NC}" fi -# Step 3: Check if the correct Nitro branch is checked out and submodules are updated -EXPECTED_BRANCH="v3.2.1" -CURRENT_BRANCH=$(git branch --show-current) -if [ "$CURRENT_BRANCH" != "$EXPECTED_BRANCH" ]; then - echo -e "${YELLOW}Switch to the correct branch using: git fetch origin && git checkout $EXPECTED_BRANCH${NC}" -else - echo -e "${GREEN}The Nitro repository is on the correct branch: $EXPECTED_BRANCH.${NC}" -fi +# Step 3: Check the version tag +VERSION_TAG=$(git tag --points-at HEAD | sed '/-/!s/$/_/' | sort -rV | sed 's/_$//' | head -n 1 | grep ^ || git show -s --pretty=%D | sed 's/, /\n/g' | grep -v '^origin/' | grep -v '^grafted\|HEAD\|master\|main$' || echo "dev") +echo -e "${YELLOW}You are on the version tag: $VERSION_TAG${NC}" # Check if submodules are properly initialized and updated if git submodule status | grep -qE '^-|\+'; then echo -e "${YELLOW}Submodules are not properly initialized or updated. Run: git submodule update --init --recursive --force${NC}" + exit 1 else echo -e "${GREEN}All submodules are properly initialized and up to date.${NC}" fi @@ -58,6 +57,7 @@ if docker images | grep -q "nitro-node"; then echo -e "${GREEN}Nitro Docker image is built.${NC}" else echo -e "${RED}Nitro Docker image is not built. Build it using: docker build . --tag nitro-node${NC}" + exit 1 fi # Step 5: Check prerequisites for building binaries @@ -101,6 +101,7 @@ for pkg in "${prerequisites[@]}"; do [[ "$pkg" == "wasm2wat" ]] && pkg="wabt" [[ "$pkg" == "clang" ]] && pkg="llvm" echo -e "${RED}$pkg is not installed. Please install $pkg. $INSTALLATION_DOCS_URL${NC}" + exit 1 fi done @@ -109,13 +110,23 @@ if command_exists node && node -v | grep -q "v18"; then echo -e "${GREEN}Node.js version 18 is installed.${NC}" else echo -e "${RED}Node.js version 18 not installed. $INSTALLATION_DOCS_URL${NC}" + exit 1 fi -# Step 7: Check Rust version +# Step 7a: Check Rust version if command_exists rustc && rustc --version | grep -q "1.80.1"; then echo -e "${GREEN}Rust version 1.80.1 is installed.${NC}" else echo -e "${RED}Rust version 1.80.1 not installed. $INSTALLATION_DOCS_URL${NC}" + exit 1 +fi + +# Step 7b: Check Rust nightly toolchain +if rustup toolchain list | grep -q "nightly"; then + echo -e "${GREEN}Rust nightly toolchain is installed.${NC}" +else + echo -e "${RED}Rust nightly toolchain is not installed. Install it using: rustup toolchain install nightly${NC}" + exit 1 fi # Step 8: Check Go version @@ -123,6 +134,7 @@ if command_exists go && go version | grep -q "go1.23"; then echo -e "${GREEN}Go version 1.23 is installed.${NC}" else echo -e "${RED}Go version 1.23 not installed. $INSTALLATION_DOCS_URL${NC}" + exit 1 fi # Step 9: Check Foundry installation @@ -130,7 +142,8 @@ if command_exists foundryup; then echo -e "${GREEN}Foundry is installed.${NC}" else echo -e "${RED}Foundry is not installed. $INSTALLATION_DOCS_URL${NC}" + exit 1 fi echo -e "${BLUE}Verification complete.${NC}" -echo -e "${YELLOW}Refer to https://docs.arbitrum.io/run-arbitrum-node/nitro/build-nitro-locally if the build fails for any other reason.${NC}" +exit 0 From c33f09fd00767143fe46f5ce57e828f917480f20 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 14 Oct 2024 08:48:43 -0300 Subject: [PATCH 374/489] Rename PeriodToAllowPostingBatchWithOnlyBatchPostingReport to MaxEmptyBatchDelay --- arbnode/batch_poster.go | 44 ++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/arbnode/batch_poster.go b/arbnode/batch_poster.go index 63175d48ac..6a3340d7af 100644 --- a/arbnode/batch_poster.go +++ b/arbnode/batch_poster.go @@ -154,25 +154,25 @@ type BatchPosterConfig struct { // Batch post polling interval. PollInterval time.Duration `koanf:"poll-interval" reload:"hot"` // Batch posting error delay. - ErrorDelay time.Duration `koanf:"error-delay" reload:"hot"` - CompressionLevel int `koanf:"compression-level" reload:"hot"` - DASRetentionPeriod time.Duration `koanf:"das-retention-period" reload:"hot"` - GasRefunderAddress string `koanf:"gas-refunder-address" reload:"hot"` - DataPoster dataposter.DataPosterConfig `koanf:"data-poster" reload:"hot"` - RedisUrl string `koanf:"redis-url"` - RedisLock redislock.SimpleCfg `koanf:"redis-lock" reload:"hot"` - ExtraBatchGas uint64 `koanf:"extra-batch-gas" reload:"hot"` - Post4844Blobs bool `koanf:"post-4844-blobs" reload:"hot"` - IgnoreBlobPrice bool `koanf:"ignore-blob-price" reload:"hot"` - ParentChainWallet genericconf.WalletConfig `koanf:"parent-chain-wallet"` - L1BlockBound string `koanf:"l1-block-bound" reload:"hot"` - L1BlockBoundBypass time.Duration `koanf:"l1-block-bound-bypass" reload:"hot"` - UseAccessLists bool `koanf:"use-access-lists" reload:"hot"` - GasEstimateBaseFeeMultipleBips arbmath.UBips `koanf:"gas-estimate-base-fee-multiple-bips"` - Dangerous BatchPosterDangerousConfig `koanf:"dangerous"` - ReorgResistanceMargin time.Duration `koanf:"reorg-resistance-margin" reload:"hot"` - CheckBatchCorrectness bool `koanf:"check-batch-correctness"` - PeriodToAllowPostingBatchWithOnlyBatchPostingReport time.Duration `koanf:"period-to-allow-posting-batch-with-only-batch-posting-report"` + ErrorDelay time.Duration `koanf:"error-delay" reload:"hot"` + CompressionLevel int `koanf:"compression-level" reload:"hot"` + DASRetentionPeriod time.Duration `koanf:"das-retention-period" reload:"hot"` + GasRefunderAddress string `koanf:"gas-refunder-address" reload:"hot"` + DataPoster dataposter.DataPosterConfig `koanf:"data-poster" reload:"hot"` + RedisUrl string `koanf:"redis-url"` + RedisLock redislock.SimpleCfg `koanf:"redis-lock" reload:"hot"` + ExtraBatchGas uint64 `koanf:"extra-batch-gas" reload:"hot"` + Post4844Blobs bool `koanf:"post-4844-blobs" reload:"hot"` + IgnoreBlobPrice bool `koanf:"ignore-blob-price" reload:"hot"` + ParentChainWallet genericconf.WalletConfig `koanf:"parent-chain-wallet"` + L1BlockBound string `koanf:"l1-block-bound" reload:"hot"` + L1BlockBoundBypass time.Duration `koanf:"l1-block-bound-bypass" reload:"hot"` + UseAccessLists bool `koanf:"use-access-lists" reload:"hot"` + GasEstimateBaseFeeMultipleBips arbmath.UBips `koanf:"gas-estimate-base-fee-multiple-bips"` + Dangerous BatchPosterDangerousConfig `koanf:"dangerous"` + ReorgResistanceMargin time.Duration `koanf:"reorg-resistance-margin" reload:"hot"` + CheckBatchCorrectness bool `koanf:"check-batch-correctness"` + MaxEmptyBatchDelay time.Duration `koanf:"max-empty-batch-delay"` gasRefunder common.Address l1BlockBound l1BlockBound @@ -226,7 +226,7 @@ func BatchPosterConfigAddOptions(prefix string, f *pflag.FlagSet) { f.Uint64(prefix+".gas-estimate-base-fee-multiple-bips", uint64(DefaultBatchPosterConfig.GasEstimateBaseFeeMultipleBips), "for gas estimation, use this multiple of the basefee (measured in basis points) as the max fee per gas") f.Duration(prefix+".reorg-resistance-margin", DefaultBatchPosterConfig.ReorgResistanceMargin, "do not post batch if its within this duration from layer 1 minimum bounds. Requires l1-block-bound option not be set to \"ignore\"") f.Bool(prefix+".check-batch-correctness", DefaultBatchPosterConfig.CheckBatchCorrectness, "setting this to true will run the batch against an inbox multiplexer and verifies that it produces the correct set of messages") - f.Duration(prefix+".period-to-allow-posting-batch-with-only-batch-posting-report", DefaultBatchPosterConfig.PeriodToAllowPostingBatchWithOnlyBatchPostingReport, "batch poster will only be able to post a batch with only batch posting report messages if this time period building a batch has passed") + f.Duration(prefix+".max-empty-batch-delay", DefaultBatchPosterConfig.MaxEmptyBatchDelay, "maximum empty batch posting delay, batch poster will only be able to post an empty batch if this time period building a batch has passed") redislock.AddConfigOptions(prefix+".redis-lock", f) dataposter.DataPosterConfigAddOptions(prefix+".data-poster", f, dataposter.DefaultDataPosterConfig) genericconf.WalletConfigAddOptions(prefix+".parent-chain-wallet", f, DefaultBatchPosterConfig.ParentChainWallet.Pathname) @@ -258,7 +258,7 @@ var DefaultBatchPosterConfig = BatchPosterConfig{ GasEstimateBaseFeeMultipleBips: arbmath.OneInUBips * 3 / 2, ReorgResistanceMargin: 10 * time.Minute, CheckBatchCorrectness: true, - PeriodToAllowPostingBatchWithOnlyBatchPostingReport: 3 * 24 * time.Hour, + MaxEmptyBatchDelay: 3 * 24 * time.Hour, } var DefaultBatchPosterL1WalletConfig = genericconf.WalletConfig{ @@ -1312,7 +1312,7 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) b.building.muxBackend.delayedInbox = append(b.building.muxBackend.delayedInbox, msg) } } - if (msg.Message.Header.Kind != arbostypes.L1MessageType_BatchPostingReport) || (time.Since(firstMsgTime) >= config.PeriodToAllowPostingBatchWithOnlyBatchPostingReport) { + if (msg.Message.Header.Kind != arbostypes.L1MessageType_BatchPostingReport) || (time.Since(firstMsgTime) >= config.MaxEmptyBatchDelay) { b.building.haveUsefulMessage = true } b.building.msgCount++ From 4a12f9c8620260a7a388aad9ee747bf49693ba4a Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Wed, 18 Sep 2024 10:13:59 -0500 Subject: [PATCH 375/489] Add Gas and Ink types in rust --- arbitrator/arbutil/src/evm/api.rs | 118 ++++++++++++++---- arbitrator/arbutil/src/evm/mod.rs | 49 ++++---- arbitrator/arbutil/src/evm/req.rs | 67 +++++----- arbitrator/arbutil/src/evm/storage.rs | 20 +-- arbitrator/arbutil/src/pricing.rs | 14 ++- arbitrator/jit/src/program.rs | 5 +- arbitrator/jit/src/stylus_backend.rs | 12 +- arbitrator/prover/src/machine.rs | 2 +- arbitrator/prover/src/merkle.rs | 5 +- arbitrator/prover/src/programs/config.rs | 9 +- arbitrator/prover/src/programs/memory.rs | 22 ++-- arbitrator/prover/src/programs/meter.rs | 40 +++--- arbitrator/prover/src/test.rs | 4 +- arbitrator/stylus/src/env.rs | 16 +-- arbitrator/stylus/src/evm_api.rs | 6 +- arbitrator/stylus/src/host.rs | 14 +-- arbitrator/stylus/src/lib.rs | 8 +- arbitrator/stylus/src/native.rs | 4 +- arbitrator/stylus/src/run.rs | 8 +- arbitrator/stylus/src/test/api.rs | 54 ++++---- arbitrator/stylus/src/test/mod.rs | 16 +-- arbitrator/stylus/src/test/native.rs | 22 ++-- arbitrator/stylus/src/test/wavm.rs | 11 +- .../wasm-libraries/user-host-trait/src/lib.rs | 28 ++--- 24 files changed, 319 insertions(+), 235 deletions(-) diff --git a/arbitrator/arbutil/src/evm/api.rs b/arbitrator/arbutil/src/evm/api.rs index 9d4c78c0de..a9f8d927fe 100644 --- a/arbitrator/arbutil/src/evm/api.rs +++ b/arbitrator/arbutil/src/evm/api.rs @@ -73,19 +73,93 @@ impl DataReader for VecReader { } } +macro_rules! derive_math { + ($t:ident) => { + impl std::ops::Add for $t { + type Output = Self; + + fn add(self, rhs: Self) -> Self { + Self(self.0 + rhs.0) + } + } + + impl std::ops::AddAssign for $t { + fn add_assign(&mut self, rhs: Self) { + self.0 += rhs.0; + } + } + + impl std::ops::Sub for $t { + type Output = Self; + + fn sub(self, rhs: Self) -> Self { + Self(self.0 - rhs.0) + } + } + + impl std::ops::SubAssign for $t { + fn sub_assign(&mut self, rhs: Self) { + self.0 -= rhs.0; + } + } + + impl std::ops::Mul for $t { + type Output = Self; + + fn mul(self, rhs: u64) -> Self { + Self(self.0 * rhs) + } + } + + impl std::ops::Mul<$t> for u64 { + type Output = $t; + + fn mul(self, rhs: $t) -> $t { + $t(self * rhs.0) + } + } + + impl $t { + pub const fn saturating_add(self, rhs: Self) -> Self { + Self(self.0.saturating_add(rhs.0)) + } + + pub const fn saturating_sub(self, rhs: Self) -> Self { + Self(self.0.saturating_sub(rhs.0)) + } + + pub fn to_be_bytes(self) -> [u8; 8] { + self.0.to_be_bytes() + } + } + }; +} + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +#[must_use] +pub struct Gas(pub u64); + +derive_math!(Gas); + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +#[must_use] +pub struct Ink(pub u64); + +derive_math!(Ink); + pub trait EvmApi: Send + 'static { /// Reads the 32-byte value in the EVM state trie at offset `key`. /// Returns the value and the access cost in gas. /// Analogous to `vm.SLOAD`. - fn get_bytes32(&mut self, key: Bytes32, evm_api_gas_to_use: u64) -> (Bytes32, u64); + fn get_bytes32(&mut self, key: Bytes32, evm_api_gas_to_use: Gas) -> (Bytes32, Gas); /// Stores the given value at the given key in Stylus VM's cache of the EVM state trie. /// Note that the actual values only get written after calls to `set_trie_slots`. - fn cache_bytes32(&mut self, key: Bytes32, value: Bytes32) -> u64; + fn cache_bytes32(&mut self, key: Bytes32, value: Bytes32) -> Gas; /// Persists any dirty values in the storage cache to the EVM state trie, dropping the cache entirely if requested. /// Analogous to repeated invocations of `vm.SSTORE`. - fn flush_storage_cache(&mut self, clear: bool, gas_left: u64) -> Result; + fn flush_storage_cache(&mut self, clear: bool, gas_left: Gas) -> Result; /// Reads the 32-byte value in the EVM's transient state trie at offset `key`. /// Analogous to `vm.TLOAD`. @@ -102,10 +176,10 @@ pub trait EvmApi: Send + 'static { &mut self, contract: Bytes20, calldata: &[u8], - gas_left: u64, - gas_req: u64, + gas_left: Gas, + gas_req: Gas, value: Bytes32, - ) -> (u32, u64, UserOutcomeKind); + ) -> (u32, Gas, UserOutcomeKind); /// Delegate-calls the contract at the given address. /// Returns the EVM return data's length, the gas cost, and whether the call succeeded. @@ -114,9 +188,9 @@ pub trait EvmApi: Send + 'static { &mut self, contract: Bytes20, calldata: &[u8], - gas_left: u64, - gas_req: u64, - ) -> (u32, u64, UserOutcomeKind); + gas_left: Gas, + gas_req: Gas, + ) -> (u32, Gas, UserOutcomeKind); /// Static-calls the contract at the given address. /// Returns the EVM return data's length, the gas cost, and whether the call succeeded. @@ -125,9 +199,9 @@ pub trait EvmApi: Send + 'static { &mut self, contract: Bytes20, calldata: &[u8], - gas_left: u64, - gas_req: u64, - ) -> (u32, u64, UserOutcomeKind); + gas_left: Gas, + gas_req: Gas, + ) -> (u32, Gas, UserOutcomeKind); /// Deploys a new contract using the init code provided. /// Returns the new contract's address on success, or the error reason on failure. @@ -137,8 +211,8 @@ pub trait EvmApi: Send + 'static { &mut self, code: Vec, endowment: Bytes32, - gas: u64, - ) -> (eyre::Result, u32, u64); + gas: Gas, + ) -> (eyre::Result, u32, Gas); /// Deploys a new contract using the init code provided, with an address determined in part by the `salt`. /// Returns the new contract's address on success, or the error reason on failure. @@ -149,8 +223,8 @@ pub trait EvmApi: Send + 'static { code: Vec, endowment: Bytes32, salt: Bytes32, - gas: u64, - ) -> (eyre::Result, u32, u64); + gas: Gas, + ) -> (eyre::Result, u32, Gas); /// Returns the EVM return data. /// Analogous to `vm.RETURNDATACOPY`. @@ -164,21 +238,21 @@ pub trait EvmApi: Send + 'static { /// Gets the balance of the given account. /// Returns the balance and the access cost in gas. /// Analogous to `vm.BALANCE`. - fn account_balance(&mut self, address: Bytes20) -> (Bytes32, u64); + fn account_balance(&mut self, address: Bytes20) -> (Bytes32, Gas); /// Returns the code and the access cost in gas. /// Analogous to `vm.EXTCODECOPY`. - fn account_code(&mut self, address: Bytes20, gas_left: u64) -> (D, u64); + fn account_code(&mut self, address: Bytes20, gas_left: Gas) -> (D, Gas); /// Gets the hash of the given address's code. /// Returns the hash and the access cost in gas. /// Analogous to `vm.EXTCODEHASH`. - fn account_codehash(&mut self, address: Bytes20) -> (Bytes32, u64); + fn account_codehash(&mut self, address: Bytes20) -> (Bytes32, Gas); /// Determines the cost in gas of allocating additional wasm pages. /// Note: has the side effect of updating Geth's memory usage tracker. /// Not analogous to any EVM opcode. - fn add_pages(&mut self, pages: u16) -> u64; + fn add_pages(&mut self, pages: u16) -> Gas; /// Captures tracing information for hostio invocations during native execution. fn capture_hostio( @@ -186,7 +260,7 @@ pub trait EvmApi: Send + 'static { name: &str, args: &[u8], outs: &[u8], - start_ink: u64, - end_ink: u64, + start_ink: Ink, + end_ink: Ink, ); } diff --git a/arbitrator/arbutil/src/evm/mod.rs b/arbitrator/arbutil/src/evm/mod.rs index 36dadd906a..063194b0c6 100644 --- a/arbitrator/arbutil/src/evm/mod.rs +++ b/arbitrator/arbutil/src/evm/mod.rs @@ -2,6 +2,7 @@ // For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE use crate::{Bytes20, Bytes32}; +use api::Gas; pub mod api; pub mod req; @@ -9,70 +10,70 @@ pub mod storage; pub mod user; // params.SstoreSentryGasEIP2200 -pub const SSTORE_SENTRY_GAS: u64 = 2300; +pub const SSTORE_SENTRY_GAS: Gas = Gas(2300); // params.ColdAccountAccessCostEIP2929 -pub const COLD_ACCOUNT_GAS: u64 = 2600; +pub const COLD_ACCOUNT_GAS: Gas = Gas(2600); // params.ColdSloadCostEIP2929 -pub const COLD_SLOAD_GAS: u64 = 2100; +pub const COLD_SLOAD_GAS: Gas = Gas(2100); // params.WarmStorageReadCostEIP2929 -pub const WARM_SLOAD_GAS: u64 = 100; +pub const WARM_SLOAD_GAS: Gas = Gas(100); // params.WarmStorageReadCostEIP2929 (see enable1153 in jump_table.go) -pub const TLOAD_GAS: u64 = WARM_SLOAD_GAS; -pub const TSTORE_GAS: u64 = WARM_SLOAD_GAS; +pub const TLOAD_GAS: Gas = WARM_SLOAD_GAS; +pub const TSTORE_GAS: Gas = WARM_SLOAD_GAS; // params.LogGas and params.LogDataGas -pub const LOG_TOPIC_GAS: u64 = 375; -pub const LOG_DATA_GAS: u64 = 8; +pub const LOG_TOPIC_GAS: Gas = Gas(375); +pub const LOG_DATA_GAS: Gas = Gas(8); // params.CopyGas -pub const COPY_WORD_GAS: u64 = 3; +pub const COPY_WORD_GAS: Gas = Gas(3); // params.Keccak256Gas -pub const KECCAK_256_GAS: u64 = 30; -pub const KECCAK_WORD_GAS: u64 = 6; +pub const KECCAK_256_GAS: Gas = Gas(30); +pub const KECCAK_WORD_GAS: Gas = Gas(6); // vm.GasQuickStep (see gas.go) -pub const GAS_QUICK_STEP: u64 = 2; +pub const GAS_QUICK_STEP: Gas = Gas(2); // vm.GasQuickStep (see jump_table.go) -pub const ADDRESS_GAS: u64 = GAS_QUICK_STEP; +pub const ADDRESS_GAS: Gas = GAS_QUICK_STEP; // vm.GasQuickStep (see eips.go) -pub const BASEFEE_GAS: u64 = GAS_QUICK_STEP; +pub const BASEFEE_GAS: Gas = GAS_QUICK_STEP; // vm.GasQuickStep (see eips.go) -pub const CHAINID_GAS: u64 = GAS_QUICK_STEP; +pub const CHAINID_GAS: Gas = GAS_QUICK_STEP; // vm.GasQuickStep (see jump_table.go) -pub const COINBASE_GAS: u64 = GAS_QUICK_STEP; +pub const COINBASE_GAS: Gas = GAS_QUICK_STEP; // vm.GasQuickStep (see jump_table.go) -pub const GASLIMIT_GAS: u64 = GAS_QUICK_STEP; +pub const GASLIMIT_GAS: Gas = GAS_QUICK_STEP; // vm.GasQuickStep (see jump_table.go) -pub const NUMBER_GAS: u64 = GAS_QUICK_STEP; +pub const NUMBER_GAS: Gas = GAS_QUICK_STEP; // vm.GasQuickStep (see jump_table.go) -pub const TIMESTAMP_GAS: u64 = GAS_QUICK_STEP; +pub const TIMESTAMP_GAS: Gas = GAS_QUICK_STEP; // vm.GasQuickStep (see jump_table.go) -pub const GASLEFT_GAS: u64 = GAS_QUICK_STEP; +pub const GASLEFT_GAS: Gas = GAS_QUICK_STEP; // vm.GasQuickStep (see jump_table.go) -pub const CALLER_GAS: u64 = GAS_QUICK_STEP; +pub const CALLER_GAS: Gas = GAS_QUICK_STEP; // vm.GasQuickStep (see jump_table.go) -pub const CALLVALUE_GAS: u64 = GAS_QUICK_STEP; +pub const CALLVALUE_GAS: Gas = GAS_QUICK_STEP; // vm.GasQuickStep (see jump_table.go) -pub const GASPRICE_GAS: u64 = GAS_QUICK_STEP; +pub const GASPRICE_GAS: Gas = GAS_QUICK_STEP; // vm.GasQuickStep (see jump_table.go) -pub const ORIGIN_GAS: u64 = GAS_QUICK_STEP; +pub const ORIGIN_GAS: Gas = GAS_QUICK_STEP; pub const ARBOS_VERSION_STYLUS_CHARGING_FIXES: u64 = 32; diff --git a/arbitrator/arbutil/src/evm/req.rs b/arbitrator/arbutil/src/evm/req.rs index 0304f2d378..621f41e951 100644 --- a/arbitrator/arbutil/src/evm/req.rs +++ b/arbitrator/arbutil/src/evm/req.rs @@ -12,8 +12,10 @@ use crate::{ use eyre::{bail, eyre, Result}; use std::collections::hash_map::Entry; +use super::api::{Gas, Ink}; + pub trait RequestHandler: Send + 'static { - fn request(&mut self, req_type: EvmApiMethod, req_data: impl AsRef<[u8]>) -> (Vec, D, u64); + fn request(&mut self, req_type: EvmApiMethod, req_data: impl AsRef<[u8]>) -> (Vec, D, Gas); } pub struct EvmApiRequestor> { @@ -33,7 +35,7 @@ impl> EvmApiRequestor { } } - fn request(&mut self, req_type: EvmApiMethod, req_data: impl AsRef<[u8]>) -> (Vec, D, u64) { + fn request(&mut self, req_type: EvmApiMethod, req_data: impl AsRef<[u8]>) -> (Vec, D, Gas) { self.handler.request(req_type, req_data) } @@ -43,10 +45,10 @@ impl> EvmApiRequestor { call_type: EvmApiMethod, contract: Bytes20, input: &[u8], - gas_left: u64, - gas_req: u64, + gas_left: Gas, + gas_req: Gas, value: Bytes32, - ) -> (u32, u64, UserOutcomeKind) { + ) -> (u32, Gas, UserOutcomeKind) { let mut request = Vec::with_capacity(20 + 32 + 8 + 8 + input.len()); request.extend(contract); request.extend(value); @@ -71,8 +73,8 @@ impl> EvmApiRequestor { code: Vec, endowment: Bytes32, salt: Option, - gas: u64, - ) -> (Result, u32, u64) { + gas: Gas, + ) -> (Result, u32, Gas) { let mut request = Vec::with_capacity(8 + 2 * 32 + code.len()); request.extend(gas.to_be_bytes()); request.extend(endowment); @@ -98,7 +100,7 @@ impl> EvmApiRequestor { } impl> EvmApi for EvmApiRequestor { - fn get_bytes32(&mut self, key: Bytes32, evm_api_gas_to_use: u64) -> (Bytes32, u64) { + fn get_bytes32(&mut self, key: Bytes32, evm_api_gas_to_use: Gas) -> (Bytes32, Gas) { let cache = &mut self.storage_cache; let mut cost = cache.read_gas(); @@ -110,7 +112,7 @@ impl> EvmApi for EvmApiRequestor { (value.value, cost) } - fn cache_bytes32(&mut self, key: Bytes32, value: Bytes32) -> u64 { + fn cache_bytes32(&mut self, key: Bytes32, value: Bytes32) -> Gas { let cost = self.storage_cache.write_gas(); match self.storage_cache.entry(key) { Entry::Occupied(mut key) => key.get_mut().value = value, @@ -119,7 +121,7 @@ impl> EvmApi for EvmApiRequestor { cost } - fn flush_storage_cache(&mut self, clear: bool, gas_left: u64) -> Result { + fn flush_storage_cache(&mut self, clear: bool, gas_left: Gas) -> Result { let mut data = Vec::with_capacity(64 * self.storage_cache.len() + 8); data.extend(gas_left.to_be_bytes()); @@ -134,7 +136,7 @@ impl> EvmApi for EvmApiRequestor { self.storage_cache.clear(); } if data.len() == 8 { - return Ok(0); // no need to make request + return Ok(Gas(0)); // no need to make request } let (res, _, cost) = self.request(EvmApiMethod::SetTrieSlots, data); @@ -174,10 +176,10 @@ impl> EvmApi for EvmApiRequestor { &mut self, contract: Bytes20, input: &[u8], - gas_left: u64, - gas_req: u64, + gas_left: Gas, + gas_req: Gas, value: Bytes32, - ) -> (u32, u64, UserOutcomeKind) { + ) -> (u32, Gas, UserOutcomeKind) { self.call_request( EvmApiMethod::ContractCall, contract, @@ -192,9 +194,9 @@ impl> EvmApi for EvmApiRequestor { &mut self, contract: Bytes20, input: &[u8], - gas_left: u64, - gas_req: u64, - ) -> (u32, u64, UserOutcomeKind) { + gas_left: Gas, + gas_req: Gas, + ) -> (u32, Gas, UserOutcomeKind) { self.call_request( EvmApiMethod::DelegateCall, contract, @@ -209,9 +211,9 @@ impl> EvmApi for EvmApiRequestor { &mut self, contract: Bytes20, input: &[u8], - gas_left: u64, - gas_req: u64, - ) -> (u32, u64, UserOutcomeKind) { + gas_left: Gas, + gas_req: Gas, + ) -> (u32, Gas, UserOutcomeKind) { self.call_request( EvmApiMethod::StaticCall, contract, @@ -226,8 +228,8 @@ impl> EvmApi for EvmApiRequestor { &mut self, code: Vec, endowment: Bytes32, - gas: u64, - ) -> (Result, u32, u64) { + gas: Gas, + ) -> (Result, u32, Gas) { self.create_request(EvmApiMethod::Create1, code, endowment, None, gas) } @@ -236,8 +238,8 @@ impl> EvmApi for EvmApiRequestor { code: Vec, endowment: Bytes32, salt: Bytes32, - gas: u64, - ) -> (Result, u32, u64) { + gas: Gas, + ) -> (Result, u32, Gas) { self.create_request(EvmApiMethod::Create2, code, endowment, Some(salt), gas) } @@ -258,15 +260,15 @@ impl> EvmApi for EvmApiRequestor { Ok(()) } - fn account_balance(&mut self, address: Bytes20) -> (Bytes32, u64) { + fn account_balance(&mut self, address: Bytes20) -> (Bytes32, Gas) { let (res, _, cost) = self.request(EvmApiMethod::AccountBalance, address); (res.try_into().unwrap(), cost) } - fn account_code(&mut self, address: Bytes20, gas_left: u64) -> (D, u64) { + fn account_code(&mut self, address: Bytes20, gas_left: Gas) -> (D, Gas) { if let Some((stored_address, data)) = self.last_code.as_ref() { if address == *stored_address { - return (data.clone(), 0); + return (data.clone(), Gas(0)); } } let mut req = Vec::with_capacity(20 + 8); @@ -278,12 +280,12 @@ impl> EvmApi for EvmApiRequestor { (data, cost) } - fn account_codehash(&mut self, address: Bytes20) -> (Bytes32, u64) { + fn account_codehash(&mut self, address: Bytes20) -> (Bytes32, Gas) { let (res, _, cost) = self.request(EvmApiMethod::AccountCodeHash, address); (res.try_into().unwrap(), cost) } - fn add_pages(&mut self, pages: u16) -> u64 { + fn add_pages(&mut self, pages: u16) -> Gas { self.request(EvmApiMethod::AddPages, pages.to_be_bytes()).2 } @@ -292,8 +294,8 @@ impl> EvmApi for EvmApiRequestor { name: &str, args: &[u8], outs: &[u8], - start_ink: u64, - end_ink: u64, + start_ink: Ink, + end_ink: Ink, ) { let mut request = Vec::with_capacity(2 * 8 + 3 * 2 + name.len() + args.len() + outs.len()); request.extend(start_ink.to_be_bytes()); @@ -305,6 +307,7 @@ impl> EvmApi for EvmApiRequestor { request.extend(name.as_bytes()); request.extend(args); request.extend(outs); - self.request(EvmApiMethod::CaptureHostIO, request); + // ignore response (including gas) as we're just tracing + _ = self.request(EvmApiMethod::CaptureHostIO, request); } } diff --git a/arbitrator/arbutil/src/evm/storage.rs b/arbitrator/arbutil/src/evm/storage.rs index 32b60dd21b..5f688364d7 100644 --- a/arbitrator/arbutil/src/evm/storage.rs +++ b/arbitrator/arbutil/src/evm/storage.rs @@ -5,6 +5,8 @@ use crate::Bytes32; use fnv::FnvHashMap as HashMap; use std::ops::{Deref, DerefMut}; +use super::api::Gas; + /// Represents the EVM word at a given key. #[derive(Debug)] pub struct StorageWord { @@ -37,23 +39,23 @@ pub struct StorageCache { } impl StorageCache { - pub const REQUIRED_ACCESS_GAS: u64 = 10; + pub const REQUIRED_ACCESS_GAS: Gas = Gas(10); - pub fn read_gas(&mut self) -> u64 { + pub fn read_gas(&mut self) -> Gas { self.reads += 1; match self.reads { - 0..=32 => 0, - 33..=128 => 2, - _ => 10, + 0..=32 => Gas(0), + 33..=128 => Gas(2), + _ => Gas(10), } } - pub fn write_gas(&mut self) -> u64 { + pub fn write_gas(&mut self) -> Gas { self.writes += 1; match self.writes { - 0..=8 => 0, - 9..=64 => 7, - _ => 10, + 0..=8 => Gas(0), + 9..=64 => Gas(7), + _ => Gas(10), } } } diff --git a/arbitrator/arbutil/src/pricing.rs b/arbitrator/arbutil/src/pricing.rs index 4614b02a2a..91de739303 100644 --- a/arbitrator/arbutil/src/pricing.rs +++ b/arbitrator/arbutil/src/pricing.rs @@ -1,20 +1,22 @@ // Copyright 2023, Offchain Labs, Inc. // For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE +use crate::evm::api::Ink; + /// For hostios that may return something. -pub const HOSTIO_INK: u64 = 8400; +pub const HOSTIO_INK: Ink = Ink(8400); /// For hostios that include pointers. -pub const PTR_INK: u64 = 13440 - HOSTIO_INK; +pub const PTR_INK: Ink = Ink(13440 - HOSTIO_INK.0); /// For hostios that involve an API cost. -pub const EVM_API_INK: u64 = 59673; +pub const EVM_API_INK: Ink = Ink(59673); /// For hostios that involve a div or mod. -pub const DIV_INK: u64 = 20000; +pub const DIV_INK: Ink = Ink(20000); /// For hostios that involve a mulmod. -pub const MUL_MOD_INK: u64 = 24100; +pub const MUL_MOD_INK: Ink = Ink(24100); /// For hostios that involve an addmod. -pub const ADD_MOD_INK: u64 = 21000; +pub const ADD_MOD_INK: Ink = Ink(21000); diff --git a/arbitrator/jit/src/program.rs b/arbitrator/jit/src/program.rs index 084afe96bc..f10a059748 100644 --- a/arbitrator/jit/src/program.rs +++ b/arbitrator/jit/src/program.rs @@ -6,6 +6,7 @@ use crate::caller_env::JitEnv; use crate::machine::{Escape, MaybeEscape, WasmEnvMut}; use crate::stylus_backend::exec_wasm; +use arbutil::evm::api::Gas; use arbutil::Bytes32; use arbutil::{evm::EvmData, format::DebugBytes, heapify}; use caller_env::{GuestPtr, MemAccess}; @@ -131,7 +132,7 @@ pub fn new_program( // buy ink let pricing = config.stylus.pricing; - let ink = pricing.gas_to_ink(gas); + let ink = pricing.gas_to_ink(Gas(gas)); let Some(module) = exec.module_asms.get(&compiled_hash).cloned() else { return Err(Escape::Failure(format!( @@ -217,7 +218,7 @@ pub fn set_response( let raw_data = mem.read_slice(raw_data_ptr, raw_data_len as usize); let thread = exec.threads.last_mut().unwrap(); - thread.set_response(id, result, raw_data, gas) + thread.set_response(id, result, raw_data, Gas(gas)) } /// sends previos response diff --git a/arbitrator/jit/src/stylus_backend.rs b/arbitrator/jit/src/stylus_backend.rs index 61dbf258d4..0d8c477c6c 100644 --- a/arbitrator/jit/src/stylus_backend.rs +++ b/arbitrator/jit/src/stylus_backend.rs @@ -4,7 +4,7 @@ #![allow(clippy::too_many_arguments)] use crate::machine::{Escape, MaybeEscape}; -use arbutil::evm::api::VecReader; +use arbutil::evm::api::{Gas, Ink, VecReader}; use arbutil::evm::{ api::{EvmApiMethod, EVM_API_METHOD_REQ_OFFSET}, req::EvmApiRequestor, @@ -28,7 +28,7 @@ use stylus::{native::NativeInstance, run::RunProgram}; struct MessageToCothread { result: Vec, raw_data: Vec, - cost: u64, + cost: Gas, } #[derive(Clone)] @@ -47,7 +47,7 @@ impl RequestHandler for CothreadRequestor { &mut self, req_type: EvmApiMethod, req_data: impl AsRef<[u8]>, - ) -> (Vec, VecReader, u64) { + ) -> (Vec, VecReader, Gas) { let msg = MessageFromCothread { req_type: req_type as u32 + EVM_API_METHOD_REQ_OFFSET, req_data: req_data.as_ref().to_vec(), @@ -104,7 +104,7 @@ impl CothreadHandler { id: u32, result: Vec, raw_data: Vec, - cost: u64, + cost: Gas, ) -> MaybeEscape { let Some(msg) = self.last_request.clone() else { return Escape::hostio("trying to set response but no message pending"); @@ -131,7 +131,7 @@ pub fn exec_wasm( compile: CompileConfig, config: StylusConfig, evm_data: EvmData, - ink: u64, + ink: Ink, ) -> Result { let (tothread_tx, tothread_rx) = mpsc::sync_channel::(0); let (fromthread_tx, fromthread_rx) = mpsc::sync_channel::(0); @@ -150,7 +150,7 @@ pub fn exec_wasm( let outcome = instance.run_main(&calldata, config, ink); let ink_left = match outcome.as_ref() { - Ok(UserOutcome::OutOfStack) => 0, // take all ink when out of stack + Ok(UserOutcome::OutOfStack) => Ink(0), // take all ink when out of stack _ => instance.ink_left().into(), }; diff --git a/arbitrator/prover/src/machine.rs b/arbitrator/prover/src/machine.rs index 4ece1f7bf2..66992019f9 100644 --- a/arbitrator/prover/src/machine.rs +++ b/arbitrator/prover/src/machine.rs @@ -1816,7 +1816,7 @@ impl Machine { } #[cfg(feature = "native")] - pub fn call_user_func(&mut self, func: &str, args: Vec, ink: u64) -> Result> { + pub fn call_user_func(&mut self, func: &str, args: Vec, ink: arbutil::evm::api::Ink) -> Result> { self.set_ink(ink); self.call_function("user", func, args) } diff --git a/arbitrator/prover/src/merkle.rs b/arbitrator/prover/src/merkle.rs index 4a1278b4cb..fbd704dfc6 100644 --- a/arbitrator/prover/src/merkle.rs +++ b/arbitrator/prover/src/merkle.rs @@ -549,8 +549,7 @@ mod test { let mut empty_node = Bytes32([ 57, 29, 211, 154, 252, 227, 18, 99, 65, 126, 203, 166, 252, 232, 32, 3, 98, 194, 254, 186, 118, 14, 139, 192, 101, 156, 55, 194, 101, 11, 11, 168, - ]) - .clone(); + ]); for _ in 0..64 { print!("Bytes32(["); for i in 0..32 { @@ -607,7 +606,7 @@ mod test { for layer in 0..64 { // empty_hash_at is just a lookup, but empty_hash is calculated iteratively. assert_eq!(empty_hash_at(ty, layer), &empty_hash); - empty_hash = hash_node(ty, &empty_hash, &empty_hash); + empty_hash = hash_node(ty, empty_hash, empty_hash); } } } diff --git a/arbitrator/prover/src/programs/config.rs b/arbitrator/prover/src/programs/config.rs index 0353589358..bd6fb3a843 100644 --- a/arbitrator/prover/src/programs/config.rs +++ b/arbitrator/prover/src/programs/config.rs @@ -4,6 +4,7 @@ #![allow(clippy::field_reassign_with_default)] use crate::{programs::meter, value::FunctionType}; +use arbutil::evm::api::{Gas, Ink}; use derivative::Derivative; use fnv::FnvHashMap as HashMap; use std::fmt::Debug; @@ -72,12 +73,12 @@ impl PricingParams { Self { ink_price } } - pub fn gas_to_ink(&self, gas: u64) -> u64 { - gas.saturating_mul(self.ink_price.into()) + pub fn gas_to_ink(&self, gas: Gas) -> Ink { + Ink(gas.0.saturating_mul(self.ink_price.into())) } - pub fn ink_to_gas(&self, ink: u64) -> u64 { - ink / self.ink_price as u64 // never 0 + pub fn ink_to_gas(&self, ink: Ink) -> Gas { + Gas(ink.0 / self.ink_price as u64) // ink_price is never 0 } } diff --git a/arbitrator/prover/src/programs/memory.rs b/arbitrator/prover/src/programs/memory.rs index 7253b59dc4..758f2f3e82 100644 --- a/arbitrator/prover/src/programs/memory.rs +++ b/arbitrator/prover/src/programs/memory.rs @@ -1,6 +1,8 @@ // Copyright 2023, Offchain Labs, Inc. // For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE +use arbutil::evm::api::Gas; + #[derive(Clone, Copy, Debug)] #[repr(C)] pub struct MemoryModel { @@ -28,20 +30,20 @@ impl MemoryModel { } /// Determines the gas cost of allocating `new` pages given `open` are active and `ever` have ever been. - pub fn gas_cost(&self, new: u16, open: u16, ever: u16) -> u64 { + pub fn gas_cost(&self, new: u16, open: u16, ever: u16) -> Gas { let new_open = open.saturating_add(new); let new_ever = ever.max(new_open); // free until expansion beyond the first few if new_ever <= self.free_pages { - return 0; + return Gas(0); } let credit = |pages: u16| pages.saturating_sub(self.free_pages); let adding = credit(new_open).saturating_sub(credit(open)) as u64; let linear = adding.saturating_mul(self.page_gas.into()); let expand = Self::exp(new_ever) - Self::exp(ever); - linear.saturating_add(expand) + Gas(linear.saturating_add(expand)) } fn exp(pages: u16) -> u64 { @@ -85,7 +87,7 @@ fn test_model() { let mut pages = 0; while pages < 128 { let jump = jump.min(128 - pages); - total += model.gas_cost(jump, pages, pages); + total += model.gas_cost(jump, pages, pages).0; pages += jump; } assert_eq!(total, 31999998); @@ -98,7 +100,7 @@ fn test_model() { let mut adds = 0; while ever < 128 { let jump = jump.min(128 - open); - total += model.gas_cost(jump, open, ever); + total += model.gas_cost(jump, open, ever).0; open += jump; ever = ever.max(open); @@ -114,12 +116,12 @@ fn test_model() { } // check saturation - assert_eq!(u64::MAX, model.gas_cost(129, 0, 0)); - assert_eq!(u64::MAX, model.gas_cost(u16::MAX, 0, 0)); + assert_eq!(Gas(u64::MAX), model.gas_cost(129, 0, 0)); + assert_eq!(Gas(u64::MAX), model.gas_cost(u16::MAX, 0, 0)); // check free pages let model = MemoryModel::new(128, 1000); - assert_eq!(0, model.gas_cost(128, 0, 0)); - assert_eq!(0, model.gas_cost(128, 0, 128)); - assert_eq!(u64::MAX, model.gas_cost(129, 0, 0)); + assert_eq!(Gas(0), model.gas_cost(128, 0, 0)); + assert_eq!(Gas(0), model.gas_cost(128, 0, 128)); + assert_eq!(Gas(u64::MAX), model.gas_cost(129, 0, 0)); } diff --git a/arbitrator/prover/src/programs/meter.rs b/arbitrator/prover/src/programs/meter.rs index ab069fd911..fe98bbcbc1 100644 --- a/arbitrator/prover/src/programs/meter.rs +++ b/arbitrator/prover/src/programs/meter.rs @@ -9,7 +9,7 @@ use crate::{ value::FunctionType, Machine, }; -use arbutil::{evm, operator::OperatorInfo, Bytes32}; +use arbutil::{evm::{self, api::{Gas, Ink}}, operator::OperatorInfo, Bytes32}; use derivative::Derivative; use eyre::Result; use fnv::FnvHashMap as HashMap; @@ -188,15 +188,15 @@ impl<'a, F: OpcodePricer> FuncMiddleware<'a> for FuncMeter<'a, F> { #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum MachineMeter { - Ready(u64), + Ready(Ink), Exhausted, } impl MachineMeter { - pub fn ink(self) -> u64 { + pub fn ink(self) -> Ink { match self { Self::Ready(ink) => ink, - Self::Exhausted => 0, + Self::Exhausted => Ink(0), } } @@ -210,8 +210,8 @@ impl MachineMeter { /// We don't implement `From` since it's unclear what 0 would map to #[allow(clippy::from_over_into)] -impl Into for MachineMeter { - fn into(self) -> u64 { +impl Into for MachineMeter { + fn into(self) -> Ink { self.ink() } } @@ -219,7 +219,7 @@ impl Into for MachineMeter { impl Display for MachineMeter { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Self::Ready(ink) => write!(f, "{ink} ink"), + Self::Ready(ink) => write!(f, "{} ink", ink.0), Self::Exhausted => write!(f, "exhausted"), } } @@ -241,7 +241,7 @@ pub trait MeteredMachine { fn ink_left(&self) -> MachineMeter; fn set_meter(&mut self, meter: MachineMeter); - fn set_ink(&mut self, ink: u64) { + fn set_ink(&mut self, ink: Ink) { self.set_meter(MachineMeter::Ready(ink)); } @@ -250,14 +250,14 @@ pub trait MeteredMachine { Err(OutOfInkError) } - fn ink_ready(&mut self) -> Result { + fn ink_ready(&mut self) -> Result { let MachineMeter::Ready(ink_left) = self.ink_left() else { return self.out_of_ink(); }; Ok(ink_left) } - fn buy_ink(&mut self, ink: u64) -> Result<(), OutOfInkError> { + fn buy_ink(&mut self, ink: Ink) -> Result<(), OutOfInkError> { let ink_left = self.ink_ready()?; if ink_left < ink { return self.out_of_ink(); @@ -267,7 +267,7 @@ pub trait MeteredMachine { } /// Checks if the user has enough ink, but doesn't burn any - fn require_ink(&mut self, ink: u64) -> Result<(), OutOfInkError> { + fn require_ink(&mut self, ink: Ink) -> Result<(), OutOfInkError> { let ink_left = self.ink_ready()?; if ink_left < ink { return self.out_of_ink(); @@ -277,18 +277,18 @@ pub trait MeteredMachine { /// Pays for a write into the client. fn pay_for_write(&mut self, bytes: u32) -> Result<(), OutOfInkError> { - self.buy_ink(sat_add_mul(5040, 30, bytes.saturating_sub(32))) + self.buy_ink(Ink(sat_add_mul(5040, 30, bytes.saturating_sub(32)))) } /// Pays for a read into the host. fn pay_for_read(&mut self, bytes: u32) -> Result<(), OutOfInkError> { - self.buy_ink(sat_add_mul(16381, 55, bytes.saturating_sub(32))) + self.buy_ink(Ink(sat_add_mul(16381, 55, bytes.saturating_sub(32)))) } /// Pays for both I/O and keccak. fn pay_for_keccak(&mut self, bytes: u32) -> Result<(), OutOfInkError> { let words = evm::evm_words(bytes).saturating_sub(2); - self.buy_ink(sat_add_mul(121800, 21000, words)) + self.buy_ink(Ink(sat_add_mul(121800, 21000, words))) } /// Pays for copying bytes from geth. @@ -305,14 +305,14 @@ pub trait MeteredMachine { false => break, } } - self.buy_ink(3000 + exp * 17500) + self.buy_ink(Ink(3000 + exp * 17500)) } } pub trait GasMeteredMachine: MeteredMachine { fn pricing(&self) -> PricingParams; - fn gas_left(&self) -> Result { + fn gas_left(&self) -> Result { let pricing = self.pricing(); match self.ink_left() { MachineMeter::Ready(ink) => Ok(pricing.ink_to_gas(ink)), @@ -320,13 +320,13 @@ pub trait GasMeteredMachine: MeteredMachine { } } - fn buy_gas(&mut self, gas: u64) -> Result<(), OutOfInkError> { + fn buy_gas(&mut self, gas: Gas) -> Result<(), OutOfInkError> { let pricing = self.pricing(); self.buy_ink(pricing.gas_to_ink(gas)) } /// Checks if the user has enough gas, but doesn't burn any - fn require_gas(&mut self, gas: u64) -> Result<(), OutOfInkError> { + fn require_gas(&mut self, gas: Gas) -> Result<(), OutOfInkError> { let pricing = self.pricing(); self.require_ink(pricing.gas_to_ink(gas)) } @@ -350,7 +350,7 @@ impl MeteredMachine for Machine { }}; } - let ink = || convert!(self.get_global(STYLUS_INK_LEFT)); + let ink = || Ink(convert!(self.get_global(STYLUS_INK_LEFT))); let status: u32 = convert!(self.get_global(STYLUS_INK_STATUS)); match status { @@ -362,7 +362,7 @@ impl MeteredMachine for Machine { fn set_meter(&mut self, meter: MachineMeter) { let ink = meter.ink(); let status = meter.status(); - self.set_global(STYLUS_INK_LEFT, ink.into()).unwrap(); + self.set_global(STYLUS_INK_LEFT, ink.0.into()).unwrap(); self.set_global(STYLUS_INK_STATUS, status.into()).unwrap(); } } diff --git a/arbitrator/prover/src/test.rs b/arbitrator/prover/src/test.rs index 97170441ff..4fd739342e 100644 --- a/arbitrator/prover/src/test.rs +++ b/arbitrator/prover/src/test.rs @@ -1,8 +1,6 @@ // Copyright 2022-2024, Offchain Labs, Inc. // For license information, see https://github.com/nitro/blob/master/LICENSE -#![cfg(test)] - use crate::binary; use brotli::Dictionary; use eyre::Result; @@ -64,7 +62,7 @@ pub fn test_compress() -> Result<()> { let deflate = brotli::compress(data, 11, 22, dict).unwrap(); let inflate = brotli::decompress(&deflate, dict).unwrap(); assert_eq!(hex::encode(inflate), hex::encode(data)); - assert!(&deflate != &last); + assert!(deflate != last); last = deflate; } Ok(()) diff --git a/arbitrator/stylus/src/env.rs b/arbitrator/stylus/src/env.rs index 69d542070d..a153fb5bf1 100644 --- a/arbitrator/stylus/src/env.rs +++ b/arbitrator/stylus/src/env.rs @@ -3,7 +3,7 @@ use arbutil::{ evm::{ - api::{DataReader, EvmApi}, + api::{DataReader, EvmApi, Ink}, EvmData, }, pricing, @@ -74,7 +74,7 @@ impl> WasmEnv { pub fn start<'a>( env: &'a mut WasmEnvMut<'_, D, E>, - ink: u64, + ink: Ink, ) -> Result, Escape> { let mut info = Self::program(env)?; info.buy_ink(pricing::HOSTIO_INK + ink)?; @@ -88,7 +88,7 @@ impl> WasmEnv { env, memory, store, - start_ink: 0, + start_ink: Ink(0), }; if info.env.evm_data.tracing { info.start_ink = info.ink_ready()?; @@ -114,16 +114,16 @@ pub struct MeterData { } impl MeterData { - pub fn ink(&self) -> u64 { - unsafe { self.ink_left.as_ref().val.u64 } + pub fn ink(&self) -> Ink { + Ink(unsafe { self.ink_left.as_ref().val.u64 }) } pub fn status(&self) -> u32 { unsafe { self.ink_status.as_ref().val.u32 } } - pub fn set_ink(&mut self, ink: u64) { - unsafe { self.ink_left.as_mut().val = RawValue { u64: ink } } + pub fn set_ink(&mut self, ink: Ink) { + unsafe { self.ink_left.as_mut().val = RawValue { u64: ink.0 } } } pub fn set_status(&mut self, status: u32) { @@ -140,7 +140,7 @@ pub struct HostioInfo<'a, D: DataReader, E: EvmApi> { pub env: &'a mut WasmEnv, pub memory: Memory, pub store: StoreMut<'a>, - pub start_ink: u64, + pub start_ink: Ink, } impl<'a, D: DataReader, E: EvmApi> HostioInfo<'a, D, E> { diff --git a/arbitrator/stylus/src/evm_api.rs b/arbitrator/stylus/src/evm_api.rs index d267372827..0dd27e3f8c 100644 --- a/arbitrator/stylus/src/evm_api.rs +++ b/arbitrator/stylus/src/evm_api.rs @@ -3,7 +3,7 @@ use crate::{GoSliceData, RustSlice}; use arbutil::evm::{ - api::{EvmApiMethod, EVM_API_METHOD_REQ_OFFSET}, + api::{EvmApiMethod, Gas, EVM_API_METHOD_REQ_OFFSET}, req::RequestHandler, }; @@ -31,7 +31,7 @@ impl RequestHandler for NativeRequestHandler { &mut self, req_type: EvmApiMethod, req_data: impl AsRef<[u8]>, - ) -> (Vec, GoSliceData, u64) { + ) -> (Vec, GoSliceData, Gas) { let mut result = GoSliceData::null(); let mut raw_data = GoSliceData::null(); let mut cost = 0; @@ -45,6 +45,6 @@ impl RequestHandler for NativeRequestHandler { ptr!(raw_data), ) }; - (result.slice().to_vec(), raw_data, cost) + (result.slice().to_vec(), raw_data, Gas(cost)) } } diff --git a/arbitrator/stylus/src/host.rs b/arbitrator/stylus/src/host.rs index 1afc1b4e51..c72cafc316 100644 --- a/arbitrator/stylus/src/host.rs +++ b/arbitrator/stylus/src/host.rs @@ -6,7 +6,7 @@ use crate::env::{Escape, HostioInfo, MaybeEscape, WasmEnv, WasmEnvMut}; use arbutil::{ evm::{ - api::{DataReader, EvmApi}, + api::{DataReader, EvmApi, Gas, Ink}, EvmData, }, Color, @@ -82,7 +82,7 @@ where println!("{} {text}", "Stylus says:".yellow()); } - fn trace(&mut self, name: &str, args: &[u8], outs: &[u8], end_ink: u64) { + fn trace(&mut self, name: &str, args: &[u8], outs: &[u8], end_ink: Ink) { let start_ink = self.start_ink; self.evm_api .capture_hostio(name, args, outs, start_ink, end_ink); @@ -168,7 +168,7 @@ pub(crate) fn call_contract>( ) -> Result { hostio!( env, - call_contract(contract, data, data_len, value, gas, ret_len) + call_contract(contract, data, data_len, value, Gas(gas), ret_len) ) } @@ -182,7 +182,7 @@ pub(crate) fn delegate_call_contract>( ) -> Result { hostio!( env, - delegate_call_contract(contract, data, data_len, gas, ret_len) + delegate_call_contract(contract, data, data_len, Gas(gas), ret_len) ) } @@ -196,7 +196,7 @@ pub(crate) fn static_call_contract>( ) -> Result { hostio!( env, - static_call_contract(contract, data, data_len, gas, ret_len) + static_call_contract(contract, data, data_len, Gas(gas), ret_len) ) } @@ -334,13 +334,13 @@ pub(crate) fn contract_address>( pub(crate) fn evm_gas_left>( mut env: WasmEnvMut, ) -> Result { - hostio!(env, evm_gas_left()) + hostio!(env, evm_gas_left()).map(|g| g.0) } pub(crate) fn evm_ink_left>( mut env: WasmEnvMut, ) -> Result { - hostio!(env, evm_ink_left()) + hostio!(env, evm_ink_left()).map(|i| i.0) } pub(crate) fn math_div>( diff --git a/arbitrator/stylus/src/lib.rs b/arbitrator/stylus/src/lib.rs index 5962817d78..e7f10c2400 100644 --- a/arbitrator/stylus/src/lib.rs +++ b/arbitrator/stylus/src/lib.rs @@ -3,7 +3,7 @@ use arbutil::{ evm::{ - api::DataReader, + api::{DataReader, Gas, Ink}, req::EvmApiRequestor, user::{UserOutcome, UserOutcomeKind}, EvmData, @@ -279,7 +279,7 @@ pub unsafe extern "C" fn stylus_call( let evm_api = EvmApiRequestor::new(req_handler); let pricing = config.pricing; let output = &mut *output; - let ink = pricing.gas_to_ink(*gas); + let ink = pricing.gas_to_ink(Gas(*gas)); // Safety: module came from compile_user_wasm and we've paid for memory expansion let instance = unsafe { @@ -302,10 +302,10 @@ pub unsafe extern "C" fn stylus_call( Ok(outcome) => output.write_outcome(outcome), }; let ink_left = match status { - UserOutcomeKind::OutOfStack => 0, // take all gas when out of stack + UserOutcomeKind::OutOfStack => Ink(0), // take all gas when out of stack _ => instance.ink_left().into(), }; - *gas = pricing.ink_to_gas(ink_left); + *gas = pricing.ink_to_gas(ink_left).0; status } diff --git a/arbitrator/stylus/src/native.rs b/arbitrator/stylus/src/native.rs index c751a670cc..0fbdb342f3 100644 --- a/arbitrator/stylus/src/native.rs +++ b/arbitrator/stylus/src/native.rs @@ -8,7 +8,7 @@ use crate::{ }; use arbutil::{ evm::{ - api::{DataReader, EvmApi}, + api::{DataReader, EvmApi, Ink}, EvmData, }, operator::OperatorCode, @@ -270,7 +270,7 @@ impl> NativeInstance { global.set(store, value.into()).map_err(ErrReport::msg) } - pub fn call_func(&mut self, func: TypedFunction<(), R>, ink: u64) -> Result + pub fn call_func(&mut self, func: TypedFunction<(), R>, ink: Ink) -> Result where R: WasmTypeList, { diff --git a/arbitrator/stylus/src/run.rs b/arbitrator/stylus/src/run.rs index 8e673a25e5..6cbb0cfb42 100644 --- a/arbitrator/stylus/src/run.rs +++ b/arbitrator/stylus/src/run.rs @@ -4,18 +4,18 @@ #![allow(clippy::redundant_closure_call)] use crate::{env::Escape, native::NativeInstance}; -use arbutil::evm::api::{DataReader, EvmApi}; +use arbutil::evm::api::{DataReader, EvmApi, Ink}; use arbutil::evm::user::UserOutcome; use eyre::{eyre, Result}; use prover::machine::Machine; use prover::programs::{prelude::*, STYLUS_ENTRY_POINT}; pub trait RunProgram { - fn run_main(&mut self, args: &[u8], config: StylusConfig, ink: u64) -> Result; + fn run_main(&mut self, args: &[u8], config: StylusConfig, ink: Ink) -> Result; } impl RunProgram for Machine { - fn run_main(&mut self, args: &[u8], config: StylusConfig, ink: u64) -> Result { + fn run_main(&mut self, args: &[u8], config: StylusConfig, ink: Ink) -> Result { macro_rules! call { ($module:expr, $func:expr, $args:expr) => { call!($module, $func, $args, |error| UserOutcome::Failure(error)) @@ -65,7 +65,7 @@ impl RunProgram for Machine { } impl> RunProgram for NativeInstance { - fn run_main(&mut self, args: &[u8], config: StylusConfig, ink: u64) -> Result { + fn run_main(&mut self, args: &[u8], config: StylusConfig, ink: Ink) -> Result { use UserOutcome::*; self.set_ink(ink); diff --git a/arbitrator/stylus/src/test/api.rs b/arbitrator/stylus/src/test/api.rs index 66d600a6f7..7a5af6f89f 100644 --- a/arbitrator/stylus/src/test/api.rs +++ b/arbitrator/stylus/src/test/api.rs @@ -4,7 +4,7 @@ use crate::{native, run::RunProgram}; use arbutil::{ evm::{ - api::{EvmApi, VecReader}, + api::{EvmApi, Gas, Ink, VecReader}, user::UserOutcomeKind, EvmData, }, @@ -68,24 +68,24 @@ impl TestEvmApi { } impl EvmApi for TestEvmApi { - fn get_bytes32(&mut self, key: Bytes32, _evm_api_gas_to_use: u64) -> (Bytes32, u64) { + fn get_bytes32(&mut self, key: Bytes32, _evm_api_gas_to_use: Gas) -> (Bytes32, Gas) { let storage = &mut self.storage.lock(); let storage = storage.get_mut(&self.program).unwrap(); let value = storage.get(&key).cloned().unwrap_or_default(); - (value, 2100) // pretend worst case + (value, Gas(2100)) // pretend worst case } - fn cache_bytes32(&mut self, key: Bytes32, value: Bytes32) -> u64 { + fn cache_bytes32(&mut self, key: Bytes32, value: Bytes32) -> Gas { let storage = &mut self.storage.lock(); let storage = storage.get_mut(&self.program).unwrap(); storage.insert(key, value); - 0 + Gas(0) } - fn flush_storage_cache(&mut self, _clear: bool, _gas_left: u64) -> Result { + fn flush_storage_cache(&mut self, _clear: bool, _gas_left: Gas) -> Result { let storage = &mut self.storage.lock(); let storage = storage.get_mut(&self.program).unwrap(); - Ok(22100 * storage.len() as u64) // pretend worst case + Ok(Gas(22100) * storage.len() as u64) // pretend worst case } fn get_transient_bytes32(&mut self, _key: Bytes32) -> Bytes32 { @@ -102,10 +102,10 @@ impl EvmApi for TestEvmApi { &mut self, contract: Bytes20, calldata: &[u8], - _gas_left: u64, - gas_req: u64, + _gas_left: Gas, + gas_req: Gas, _value: Bytes32, - ) -> (u32, u64, UserOutcomeKind) { + ) -> (u32, Gas, UserOutcomeKind) { let compile = self.compile.clone(); let evm_data = self.evm_data; let config = *self.configs.lock().get(&contract).unwrap(); @@ -122,7 +122,7 @@ impl EvmApi for TestEvmApi { let (status, outs) = outcome.into_data(); let outs_len = outs.len() as u32; - let ink_left: u64 = native.ink_left().into(); + let ink_left: Ink = native.ink_left().into(); let gas_left = config.pricing.ink_to_gas(ink_left); *self.write_result.lock() = outs; (outs_len, gas - gas_left, status) @@ -132,9 +132,9 @@ impl EvmApi for TestEvmApi { &mut self, _contract: Bytes20, _calldata: &[u8], - _gas_left: u64, - _gas_req: u64, - ) -> (u32, u64, UserOutcomeKind) { + _gas_left: Gas, + _gas_req: Gas, + ) -> (u32, Gas, UserOutcomeKind) { todo!("delegate call not yet supported") } @@ -142,9 +142,9 @@ impl EvmApi for TestEvmApi { &mut self, contract: Bytes20, calldata: &[u8], - gas_left: u64, - gas_req: u64, - ) -> (u32, u64, UserOutcomeKind) { + gas_left: Gas, + gas_req: Gas, + ) -> (u32, Gas, UserOutcomeKind) { println!("note: overriding static call with call"); self.contract_call(contract, calldata, gas_left, gas_req, Bytes32::default()) } @@ -153,8 +153,8 @@ impl EvmApi for TestEvmApi { &mut self, _code: Vec, _endowment: Bytes32, - _gas: u64, - ) -> (Result, u32, u64) { + _gas: Gas, + ) -> (Result, u32, Gas) { unimplemented!("create1 not supported") } @@ -163,8 +163,8 @@ impl EvmApi for TestEvmApi { _code: Vec, _endowment: Bytes32, _salt: Bytes32, - _gas: u64, - ) -> (Result, u32, u64) { + _gas: Gas, + ) -> (Result, u32, Gas) { unimplemented!("create2 not supported") } @@ -176,19 +176,19 @@ impl EvmApi for TestEvmApi { Ok(()) // pretend a log was emitted } - fn account_balance(&mut self, _address: Bytes20) -> (Bytes32, u64) { + fn account_balance(&mut self, _address: Bytes20) -> (Bytes32, Gas) { unimplemented!() } - fn account_code(&mut self, _address: Bytes20, _gas_left: u64) -> (VecReader, u64) { + fn account_code(&mut self, _address: Bytes20, _gas_left: Gas) -> (VecReader, Gas) { unimplemented!() } - fn account_codehash(&mut self, _address: Bytes20) -> (Bytes32, u64) { + fn account_codehash(&mut self, _address: Bytes20) -> (Bytes32, Gas) { unimplemented!() } - fn add_pages(&mut self, new: u16) -> u64 { + fn add_pages(&mut self, new: u16) -> Gas { let model = MemoryModel::new(2, 1000); let (open, ever) = *self.pages.lock(); @@ -203,8 +203,8 @@ impl EvmApi for TestEvmApi { _name: &str, _args: &[u8], _outs: &[u8], - _start_ink: u64, - _end_ink: u64, + _start_ink: Ink, + _end_ink: Ink, ) { unimplemented!() } diff --git a/arbitrator/stylus/src/test/mod.rs b/arbitrator/stylus/src/test/mod.rs index 00c9c62ae4..830b9cd23f 100644 --- a/arbitrator/stylus/src/test/mod.rs +++ b/arbitrator/stylus/src/test/mod.rs @@ -3,7 +3,7 @@ use crate::{env::WasmEnv, native::NativeInstance, run::RunProgram, test::api::TestEvmApi}; use arbutil::{ - evm::{api::VecReader, user::UserOutcome}, + evm::{api::{Ink, VecReader}, user::UserOutcome}, Bytes20, Bytes32, Color, }; use eyre::{bail, Result}; @@ -41,7 +41,7 @@ impl TestInstance { }; let mut native = Self::new_from_store(path, store, imports)?; native.set_meter_data(); - native.set_ink(u64::MAX); + native.set_ink(Ink(u64::MAX)); native.set_stack(u32::MAX); Ok(native) } @@ -107,8 +107,8 @@ fn expensive_add(op: &Operator, _tys: &SigMap) -> u64 { } } -pub fn random_ink(min: u64) -> u64 { - rand::thread_rng().gen_range(min..=u64::MAX) +pub fn random_ink(min: u64) -> Ink { + Ink(rand::thread_rng().gen_range(min..=u64::MAX)) } pub fn random_bytes20() -> Bytes20 { @@ -135,7 +135,7 @@ fn uniform_cost_config() -> StylusConfig { stylus_config } -fn test_configs() -> (CompileConfig, StylusConfig, u64) { +fn test_configs() -> (CompileConfig, StylusConfig, Ink) { ( test_compile_config(), uniform_cost_config(), @@ -165,12 +165,12 @@ fn new_test_machine(path: &str, compile: &CompileConfig) -> Result { Arc::new(|_, _, _| panic!("tried to read preimage")), Some(stylus_data), )?; - mach.set_ink(u64::MAX); + mach.set_ink(Ink(u64::MAX)); mach.set_stack(u32::MAX); Ok(mach) } -fn run_native(native: &mut TestInstance, args: &[u8], ink: u64) -> Result> { +fn run_native(native: &mut TestInstance, args: &[u8], ink: Ink) -> Result> { let config = native.env().config.expect("no config"); match native.run_main(args, config, ink)? { UserOutcome::Success(output) => Ok(output), @@ -182,7 +182,7 @@ fn run_machine( machine: &mut Machine, args: &[u8], config: StylusConfig, - ink: u64, + ink: Ink, ) -> Result> { match machine.run_main(args, config, ink)? { UserOutcome::Success(output) => Ok(output), diff --git a/arbitrator/stylus/src/test/native.rs b/arbitrator/stylus/src/test/native.rs index 9669932a03..672bdd179c 100644 --- a/arbitrator/stylus/src/test/native.rs +++ b/arbitrator/stylus/src/test/native.rs @@ -16,7 +16,7 @@ use crate::{ use arbutil::{ crypto, evm::{ - api::EvmApi, + api::{EvmApi, Gas, Ink}, user::{UserOutcome, UserOutcomeKind}, }, format, Bytes20, Bytes32, Color, @@ -48,8 +48,8 @@ fn test_ink() -> Result<()> { macro_rules! exhaust { ($ink:expr) => { - native.set_ink($ink); - assert_eq!(native.ink_left(), MachineMeter::Ready($ink)); + native.set_ink(Ink($ink)); + assert_eq!(native.ink_left(), MachineMeter::Ready(Ink($ink))); assert!(add_one.call(&mut native.store, 32).is_err()); assert_eq!(native.ink_left(), MachineMeter::Exhausted); }; @@ -59,12 +59,12 @@ fn test_ink() -> Result<()> { exhaust!(50); exhaust!(99); - let mut ink_left = 500; + let mut ink_left = Ink(500); native.set_ink(ink_left); - while ink_left > 0 { + while ink_left > Ink(0) { assert_eq!(native.ink_left(), MachineMeter::Ready(ink_left)); assert_eq!(add_one.call(&mut native.store, 64)?, 65); - ink_left -= 100; + ink_left -= Ink(100); } assert!(add_one.call(&mut native.store, 32).is_err()); assert_eq!(native.ink_left(), MachineMeter::Exhausted); @@ -198,7 +198,7 @@ fn test_import_export_safety() -> Result<()> { let mut bin = bin?; assert!(bin.clone().instrument(&compile, codehash).is_err()); compile.debug.debug_info = false; - assert!(bin.instrument(&compile, &codehash).is_err()); + assert!(bin.instrument(&compile, codehash).is_err()); if both { assert!(TestInstance::new_test(file, compile).is_err()); @@ -268,7 +268,7 @@ fn test_heap() -> Result<()> { assert_eq!(pages, 128); let used = config.pricing.ink_to_gas(ink - native.ink_ready()?); - ensure!((used as i64 - 32_000_000).abs() < 3_000, "wrong ink"); + ensure!((used.0 as i64 - 32_000_000).abs() < 3_000, "wrong ink"); assert_eq!(native.memory_size(), Pages(128)); if step == extra { @@ -283,7 +283,7 @@ fn test_heap() -> Result<()> { // the cost should exceed a maximum u32, consuming more gas than can ever be bought let (mut native, _) = TestInstance::new_with_evm("tests/memory2.wat", &compile, config)?; - let outcome = native.run_main(&[], config, config.pricing.ink_to_gas(u32::MAX.into()))?; + let outcome = native.run_main(&[], config, config.pricing.gas_to_ink(Gas(u32::MAX.into())))?; assert_eq!(outcome.kind(), UserOutcomeKind::OutOfInk); // ensure we reject programs with excessive footprints @@ -381,7 +381,7 @@ fn test_storage() -> Result<()> { let (mut native, mut evm) = TestInstance::new_with_evm(filename, &compile, config)?; run_native(&mut native, &store_args, ink)?; - assert_eq!(evm.get_bytes32(key.into(), 0).0, Bytes32(value)); + assert_eq!(evm.get_bytes32(key.into(), Gas(0)).0, Bytes32(value)); assert_eq!(run_native(&mut native, &load_args, ink)?, value); let mut machine = Machine::from_user_path(Path::new(filename), &compile)?; @@ -465,7 +465,7 @@ fn test_calls() -> Result<()> { run_native(&mut native, &args, ink)?; for (key, value) in slots { - assert_eq!(evm.get_bytes32(key, 0).0, value); + assert_eq!(evm.get_bytes32(key, Gas(0)).0, value); } Ok(()) } diff --git a/arbitrator/stylus/src/test/wavm.rs b/arbitrator/stylus/src/test/wavm.rs index e707cf490a..729cfebf2f 100644 --- a/arbitrator/stylus/src/test/wavm.rs +++ b/arbitrator/stylus/src/test/wavm.rs @@ -2,6 +2,7 @@ // For license information, see https://github.com/nitro/blob/master/LICENSE use crate::test::{new_test_machine, test_compile_config}; +use arbutil::evm::api::Ink; use eyre::Result; use prover::{programs::prelude::*, Machine}; @@ -15,8 +16,8 @@ fn test_ink() -> Result<()> { macro_rules! exhaust { ($ink:expr) => { - machine.set_ink($ink); - assert_eq!(machine.ink_left(), MachineMeter::Ready($ink)); + machine.set_ink(Ink($ink)); + assert_eq!(machine.ink_left(), MachineMeter::Ready(Ink($ink))); assert!(call(machine, 32).is_err()); assert_eq!(machine.ink_left(), MachineMeter::Exhausted); }; @@ -26,12 +27,12 @@ fn test_ink() -> Result<()> { exhaust!(50); exhaust!(99); - let mut ink_left = 500; + let mut ink_left = Ink(500); machine.set_ink(ink_left); - while ink_left > 0 { + while ink_left > Ink(0) { assert_eq!(machine.ink_left(), MachineMeter::Ready(ink_left)); assert_eq!(call(machine, 64)?, vec![65_u32.into()]); - ink_left -= 100; + ink_left -= Ink(100); } assert!(call(machine, 32).is_err()); assert_eq!(machine.ink_left(), MachineMeter::Exhausted); diff --git a/arbitrator/wasm-libraries/user-host-trait/src/lib.rs b/arbitrator/wasm-libraries/user-host-trait/src/lib.rs index 12a6bdbed2..35a4a31347 100644 --- a/arbitrator/wasm-libraries/user-host-trait/src/lib.rs +++ b/arbitrator/wasm-libraries/user-host-trait/src/lib.rs @@ -5,7 +5,7 @@ use arbutil::{ crypto, evm::{ self, - api::{DataReader, EvmApi}, + api::{DataReader, EvmApi, Gas, Ink}, storage::StorageCache, user::UserOutcomeKind, EvmData, ARBOS_VERSION_STYLUS_CHARGING_FIXES, @@ -88,7 +88,7 @@ pub trait UserHost: GasMeteredMachine { } fn say(&self, text: D); - fn trace(&mut self, name: &str, args: &[u8], outs: &[u8], end_ink: u64); + fn trace(&mut self, name: &str, args: &[u8], outs: &[u8], end_ink: Ink); fn write_bytes20(&self, ptr: GuestPtr, src: Bytes20) -> Result<(), Self::MemoryErr> { self.write_slice(ptr, &src.0) @@ -147,7 +147,7 @@ pub trait UserHost: GasMeteredMachine { // require for cache-miss case, preserve wrong behavior for old arbos let evm_api_gas_to_use = if arbos_version < ARBOS_VERSION_STYLUS_CHARGING_FIXES { - EVM_API_INK + Gas(EVM_API_INK.0) } else { self.pricing().ink_to_gas(EVM_API_INK) }; @@ -253,7 +253,7 @@ pub trait UserHost: GasMeteredMachine { data: GuestPtr, data_len: u32, value: GuestPtr, - gas: u64, + gas: Gas, ret_len: GuestPtr, ) -> Result { let value = Some(value); @@ -282,7 +282,7 @@ pub trait UserHost: GasMeteredMachine { contract: GuestPtr, data: GuestPtr, data_len: u32, - gas: u64, + gas: Gas, ret_len: GuestPtr, ) -> Result { let call = |api: &mut Self::A, contract, data: &_, left, req, _| { @@ -312,7 +312,7 @@ pub trait UserHost: GasMeteredMachine { contract: GuestPtr, data: GuestPtr, data_len: u32, - gas: u64, + gas: Gas, ret_len: GuestPtr, ) -> Result { let call = |api: &mut Self::A, contract, data: &_, left, req, _| { @@ -329,7 +329,7 @@ pub trait UserHost: GasMeteredMachine { calldata: GuestPtr, calldata_len: u32, value: Option, - gas: u64, + gas: Gas, return_data_len: GuestPtr, call: F, name: &str, @@ -339,10 +339,10 @@ pub trait UserHost: GasMeteredMachine { &mut Self::A, Address, &[u8], - u64, - u64, + Gas, + Gas, Option, - ) -> (u32, u64, UserOutcomeKind), + ) -> (u32, Gas, UserOutcomeKind), { self.buy_ink(HOSTIO_INK + 3 * PTR_INK + EVM_API_INK)?; self.pay_for_read(calldata_len)?; @@ -465,12 +465,12 @@ pub trait UserHost: GasMeteredMachine { salt: Option, contract: GuestPtr, revert_data_len: GuestPtr, - cost: u64, + cost: Ink, call: F, name: &str, ) -> Result<(), Self::Err> where - F: FnOnce(&mut Self::A, Vec, Bytes32, Option, u64) -> (Result
, u32, u64), + F: FnOnce(&mut Self::A, Vec, Bytes32, Option, Gas) -> (Result
, u32, Gas), { self.buy_ink(HOSTIO_INK + cost)?; self.pay_for_read(code_len)?; @@ -745,7 +745,7 @@ pub trait UserHost: GasMeteredMachine { /// equivalent to that of the EVM's [`GAS`] opcode. /// /// [`GAS`]: https://www.evm.codes/#5a - fn evm_gas_left(&mut self) -> Result { + fn evm_gas_left(&mut self) -> Result { self.buy_ink(HOSTIO_INK)?; let gas = self.gas_left()?; trace!("evm_gas_left", self, &[], be!(gas), gas) @@ -757,7 +757,7 @@ pub trait UserHost: GasMeteredMachine { /// /// [`GAS`]: https://www.evm.codes/#5a /// [`Ink and Gas`]: https://developer.arbitrum.io/TODO - fn evm_ink_left(&mut self) -> Result { + fn evm_ink_left(&mut self) -> Result { self.buy_ink(HOSTIO_INK)?; let ink = self.ink_ready()?; trace!("evm_ink_left", self, &[], be!(ink), ink) From 788de6ed78a4c2626302a4a65c6b287d9848bc85 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Mon, 14 Oct 2024 14:59:34 -0500 Subject: [PATCH 376/489] Fix user-host compilation --- arbitrator/arbutil/src/evm/api.rs | 4 +-- .../wasm-libraries/user-host/src/host.rs | 29 +++++++++++++++---- .../wasm-libraries/user-host/src/ink.rs | 5 ++-- .../wasm-libraries/user-host/src/link.rs | 18 +++++++----- .../wasm-libraries/user-host/src/program.rs | 12 ++++---- 5 files changed, 45 insertions(+), 23 deletions(-) diff --git a/arbitrator/arbutil/src/evm/api.rs b/arbitrator/arbutil/src/evm/api.rs index a9f8d927fe..8715f5ac8c 100644 --- a/arbitrator/arbutil/src/evm/api.rs +++ b/arbitrator/arbutil/src/evm/api.rs @@ -135,13 +135,13 @@ macro_rules! derive_math { }; } -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Default)] #[must_use] pub struct Gas(pub u64); derive_math!(Gas); -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Default)] #[must_use] pub struct Ink(pub u64); diff --git a/arbitrator/wasm-libraries/user-host/src/host.rs b/arbitrator/wasm-libraries/user-host/src/host.rs index abe55b8c12..5ec2ece2c3 100644 --- a/arbitrator/wasm-libraries/user-host/src/host.rs +++ b/arbitrator/wasm-libraries/user-host/src/host.rs @@ -2,7 +2,7 @@ // For license information, see https://github.com/nitro/blob/master/LICENSE use crate::program::Program; -use arbutil::evm::user::UserOutcomeKind; +use arbutil::evm::{api::Gas, user::UserOutcomeKind}; use caller_env::GuestPtr; use user_host_trait::UserHost; @@ -77,7 +77,14 @@ pub unsafe extern "C" fn user_host__call_contract( gas: u64, ret_len: GuestPtr, ) -> u8 { - hostio!(call_contract(contract, data, data_len, value, gas, ret_len)) + hostio!(call_contract( + contract, + data, + data_len, + value, + Gas(gas), + ret_len + )) } #[no_mangle] @@ -89,7 +96,11 @@ pub unsafe extern "C" fn user_host__delegate_call_contract( ret_len: GuestPtr, ) -> u8 { hostio!(delegate_call_contract( - contract, data, data_len, gas, ret_len + contract, + data, + data_len, + Gas(gas), + ret_len )) } @@ -101,7 +112,13 @@ pub unsafe extern "C" fn user_host__static_call_contract( gas: u64, ret_len: GuestPtr, ) -> u8 { - hostio!(static_call_contract(contract, data, data_len, gas, ret_len)) + hostio!(static_call_contract( + contract, + data, + data_len, + Gas(gas), + ret_len + )) } #[no_mangle] @@ -207,12 +224,12 @@ pub unsafe extern "C" fn user_host__contract_address(ptr: GuestPtr) { #[no_mangle] pub unsafe extern "C" fn user_host__evm_gas_left() -> u64 { - hostio!(evm_gas_left()) + hostio!(evm_gas_left()).0 } #[no_mangle] pub unsafe extern "C" fn user_host__evm_ink_left() -> u64 { - hostio!(evm_ink_left()) + hostio!(evm_ink_left()).0 } #[no_mangle] diff --git a/arbitrator/wasm-libraries/user-host/src/ink.rs b/arbitrator/wasm-libraries/user-host/src/ink.rs index e01e616e07..bde7cfc1c0 100644 --- a/arbitrator/wasm-libraries/user-host/src/ink.rs +++ b/arbitrator/wasm-libraries/user-host/src/ink.rs @@ -2,6 +2,7 @@ // For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE use crate::program::Program; +use arbutil::evm::api::Ink; use prover::programs::{ config::PricingParams, prelude::{GasMeteredMachine, MachineMeter, MeteredMachine}, @@ -18,7 +19,7 @@ impl MeteredMachine for Program { fn ink_left(&self) -> MachineMeter { unsafe { match user_ink_status() { - 0 => MachineMeter::Ready(user_ink_left()), + 0 => MachineMeter::Ready(Ink(user_ink_left())), _ => MachineMeter::Exhausted, } } @@ -26,7 +27,7 @@ impl MeteredMachine for Program { fn set_meter(&mut self, meter: MachineMeter) { unsafe { - user_set_ink(meter.ink(), meter.status()); + user_set_ink(meter.ink().0, meter.status()); } } } diff --git a/arbitrator/wasm-libraries/user-host/src/link.rs b/arbitrator/wasm-libraries/user-host/src/link.rs index f4c402fd97..cb9f046cdb 100644 --- a/arbitrator/wasm-libraries/user-host/src/link.rs +++ b/arbitrator/wasm-libraries/user-host/src/link.rs @@ -3,7 +3,11 @@ use crate::program::Program; use arbutil::{ - evm::{user::UserOutcomeKind, EvmData}, + evm::{ + api::{Gas, Ink}, + user::UserOutcomeKind, + EvmData, + }, format::DebugBytes, heapify, Bytes20, Bytes32, }; @@ -120,11 +124,11 @@ pub unsafe extern "C" fn programs__new_program( // buy ink let pricing = config.pricing; - let ink = pricing.gas_to_ink(gas); + let ink = pricing.gas_to_ink(Gas(gas)); // link the program and ready its instrumentation let module = wavm_link_module(&MemoryLeaf(*module_hash)); - program_set_ink(module, ink); + program_set_ink(module, ink.0); program_set_stack(module, config.max_depth); // provide arguments @@ -175,7 +179,7 @@ pub unsafe extern "C" fn programs__set_response( id, STATIC_MEM.read_slice(result_ptr, result_len), STATIC_MEM.read_slice(raw_data_ptr, raw_data_len), - gas, + Gas(gas), ); } @@ -207,7 +211,7 @@ pub unsafe extern "C" fn program_internal__set_done(mut status: UserOutcomeKind) let program = Program::current(); let module = program.module; let mut outs = program.outs.as_slice(); - let mut ink_left = program_ink_left(module); + let mut ink_left = Ink(program_ink_left(module)); // apply any early exit codes if let Some(early) = program.early_exit { @@ -218,12 +222,12 @@ pub unsafe extern "C" fn program_internal__set_done(mut status: UserOutcomeKind) if program_ink_status(module) != 0 { status = OutOfInk; outs = &[]; - ink_left = 0; + ink_left = Ink(0); } if program_stack_left(module) == 0 { status = OutOfStack; outs = &[]; - ink_left = 0; + ink_left = Ink(0); } let gas_left = program.config.pricing.ink_to_gas(ink_left); diff --git a/arbitrator/wasm-libraries/user-host/src/program.rs b/arbitrator/wasm-libraries/user-host/src/program.rs index 4199a691f7..7b3782b2e5 100644 --- a/arbitrator/wasm-libraries/user-host/src/program.rs +++ b/arbitrator/wasm-libraries/user-host/src/program.rs @@ -3,7 +3,7 @@ use arbutil::{ evm::{ - api::{EvmApiMethod, VecReader, EVM_API_METHOD_REQ_OFFSET}, + api::{EvmApiMethod, Gas, Ink, VecReader, EVM_API_METHOD_REQ_OFFSET}, req::{EvmApiRequestor, RequestHandler}, user::UserOutcomeKind, EvmData, @@ -49,7 +49,7 @@ static mut LAST_REQUEST_ID: u32 = 0x10000; #[derive(Clone)] pub(crate) struct UserHostRequester { data: Option>, - answer: Option<(Vec, VecReader, u64)>, + answer: Option<(Vec, VecReader, Gas)>, req_type: u32, id: u32, } @@ -95,7 +95,7 @@ impl UserHostRequester { req_id: u32, result: Vec, raw_data: Vec, - gas: u64, + gas: Gas, ) { self.answer = Some((result, VecReader::new(raw_data), gas)); if req_id != self.id { @@ -130,7 +130,7 @@ impl UserHostRequester { } #[no_mangle] - unsafe fn send_request(&mut self, req_type: u32, data: Vec) -> (Vec, VecReader, u64) { + unsafe fn send_request(&mut self, req_type: u32, data: Vec) -> (Vec, VecReader, Gas) { let req_id = self.set_request(req_type, &data); compiler_fence(Ordering::SeqCst); @@ -149,7 +149,7 @@ impl RequestHandler for UserHostRequester { &mut self, req_type: EvmApiMethod, req_data: impl AsRef<[u8]>, - ) -> (Vec, VecReader, u64) { + ) -> (Vec, VecReader, Gas) { unsafe { self.send_request( req_type as u32 + EVM_API_METHOD_REQ_OFFSET, @@ -265,7 +265,7 @@ impl UserHost for Program { println!("{} {text}", "Stylus says:".yellow()); } - fn trace(&mut self, name: &str, args: &[u8], outs: &[u8], _end_ink: u64) { + fn trace(&mut self, name: &str, args: &[u8], outs: &[u8], _end_ink: Ink) { let args = hex::encode(args); let outs = hex::encode(outs); println!("Error: unexpected hostio tracing info for {name} while proving: {args}, {outs}"); From faadf7968161c1f8b30757af4e6386be64343264 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Mon, 14 Oct 2024 16:49:58 -0500 Subject: [PATCH 377/489] Fix user-test compilation --- .../wasm-libraries/user-test/src/host.rs | 28 ++++++++-- .../wasm-libraries/user-test/src/ink.rs | 5 +- .../wasm-libraries/user-test/src/program.rs | 54 +++++++++---------- 3 files changed, 53 insertions(+), 34 deletions(-) diff --git a/arbitrator/wasm-libraries/user-test/src/host.rs b/arbitrator/wasm-libraries/user-test/src/host.rs index f2912eaae3..f1b4506414 100644 --- a/arbitrator/wasm-libraries/user-test/src/host.rs +++ b/arbitrator/wasm-libraries/user-test/src/host.rs @@ -2,6 +2,7 @@ // For license information, see https://github.com/nitro/blob/master/LICENSE use crate::program::Program; +use arbutil::evm::api::Gas; use caller_env::GuestPtr; use user_host_trait::UserHost; @@ -63,7 +64,14 @@ pub unsafe extern "C" fn vm_hooks__call_contract( gas: u64, ret_len: GuestPtr, ) -> u8 { - hostio!(call_contract(contract, data, data_len, value, gas, ret_len)) + hostio!(call_contract( + contract, + data, + data_len, + value, + Gas(gas), + ret_len + )) } #[no_mangle] @@ -75,7 +83,11 @@ pub unsafe extern "C" fn vm_hooks__delegate_call_contract( ret_len: GuestPtr, ) -> u8 { hostio!(delegate_call_contract( - contract, data, data_len, gas, ret_len + contract, + data, + data_len, + Gas(gas), + ret_len )) } @@ -87,7 +99,13 @@ pub unsafe extern "C" fn vm_hooks__static_call_contract( gas: u64, ret_len: GuestPtr, ) -> u8 { - hostio!(static_call_contract(contract, data, data_len, gas, ret_len)) + hostio!(static_call_contract( + contract, + data, + data_len, + Gas(gas), + ret_len + )) } #[no_mangle] @@ -189,12 +207,12 @@ pub unsafe extern "C" fn vm_hooks__contract_address(ptr: GuestPtr) { #[no_mangle] pub unsafe extern "C" fn vm_hooks__evm_gas_left() -> u64 { - hostio!(evm_gas_left()) + hostio!(evm_gas_left()).0 } #[no_mangle] pub unsafe extern "C" fn vm_hooks__evm_ink_left() -> u64 { - hostio!(evm_ink_left()) + hostio!(evm_ink_left()).0 } #[no_mangle] diff --git a/arbitrator/wasm-libraries/user-test/src/ink.rs b/arbitrator/wasm-libraries/user-test/src/ink.rs index fca658e59b..72ecfadd96 100644 --- a/arbitrator/wasm-libraries/user-test/src/ink.rs +++ b/arbitrator/wasm-libraries/user-test/src/ink.rs @@ -2,6 +2,7 @@ // For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE use crate::{program::Program, CONFIG}; +use arbutil::evm::api::Ink; use prover::programs::{ config::PricingParams, prelude::{GasMeteredMachine, MachineMeter, MeteredMachine}, @@ -18,7 +19,7 @@ impl MeteredMachine for Program { fn ink_left(&self) -> MachineMeter { unsafe { match user_ink_status() { - 0 => MachineMeter::Ready(user_ink_left()), + 0 => MachineMeter::Ready(Ink(user_ink_left())), _ => MachineMeter::Exhausted, } } @@ -26,7 +27,7 @@ impl MeteredMachine for Program { fn set_meter(&mut self, meter: MachineMeter) { unsafe { - user_set_ink(meter.ink(), meter.status()); + user_set_ink(meter.ink().0, meter.status()); } } } diff --git a/arbitrator/wasm-libraries/user-test/src/program.rs b/arbitrator/wasm-libraries/user-test/src/program.rs index 85b522ee74..299fca08c3 100644 --- a/arbitrator/wasm-libraries/user-test/src/program.rs +++ b/arbitrator/wasm-libraries/user-test/src/program.rs @@ -4,7 +4,7 @@ use crate::{ARGS, EVER_PAGES, EVM_DATA, KEYS, LOGS, OPEN_PAGES, OUTS}; use arbutil::{ evm::{ - api::{EvmApi, VecReader}, + api::{EvmApi, Gas, Ink, VecReader}, user::UserOutcomeKind, EvmData, }, @@ -80,7 +80,7 @@ impl UserHost for Program { println!("{} {text}", "Stylus says:".yellow()); } - fn trace(&mut self, name: &str, args: &[u8], outs: &[u8], _end_ink: u64) { + fn trace(&mut self, name: &str, args: &[u8], outs: &[u8], _end_ink: Ink) { let args = hex::encode(args); let outs = hex::encode(outs); println!("Error: unexpected hostio tracing info for {name} while proving: {args}, {outs}"); @@ -102,18 +102,18 @@ impl Program { pub struct MockEvmApi; impl EvmApi for MockEvmApi { - fn get_bytes32(&mut self, key: Bytes32, _evm_api_gas_to_use: u64) -> (Bytes32, u64) { + fn get_bytes32(&mut self, key: Bytes32, _evm_api_gas_to_use: Gas) -> (Bytes32, Gas) { let value = KEYS.lock().get(&key).cloned().unwrap_or_default(); - (value, 2100) // pretend worst case + (value, Gas(2100)) // pretend worst case } - fn cache_bytes32(&mut self, key: Bytes32, value: Bytes32) -> u64 { + fn cache_bytes32(&mut self, key: Bytes32, value: Bytes32) -> Gas { KEYS.lock().insert(key, value); - 0 + Gas(0) } - fn flush_storage_cache(&mut self, _clear: bool, _gas_left: u64) -> Result { - Ok(22100 * KEYS.lock().len() as u64) // pretend worst case + fn flush_storage_cache(&mut self, _clear: bool, _gas_left: Gas) -> Result { + Ok(Gas(22100) * KEYS.lock().len() as u64) // pretend worst case } fn get_transient_bytes32(&mut self, _key: Bytes32) -> Bytes32 { @@ -130,10 +130,10 @@ impl EvmApi for MockEvmApi { &mut self, _contract: Bytes20, _calldata: &[u8], - _gas_left: u64, - _gas_req: u64, + _gas_left: Gas, + _gas_req: Gas, _value: Bytes32, - ) -> (u32, u64, UserOutcomeKind) { + ) -> (u32, Gas, UserOutcomeKind) { unimplemented!() } @@ -141,9 +141,9 @@ impl EvmApi for MockEvmApi { &mut self, _contract: Bytes20, _calldata: &[u8], - _gas_left: u64, - _gas_req: u64, - ) -> (u32, u64, UserOutcomeKind) { + _gas_left: Gas, + _gas_req: Gas, + ) -> (u32, Gas, UserOutcomeKind) { unimplemented!() } @@ -151,9 +151,9 @@ impl EvmApi for MockEvmApi { &mut self, _contract: Bytes20, _calldata: &[u8], - _gas_left: u64, - _gas_req: u64, - ) -> (u32, u64, UserOutcomeKind) { + _gas_left: Gas, + _gas_req: Gas, + ) -> (u32, Gas, UserOutcomeKind) { unimplemented!() } @@ -161,8 +161,8 @@ impl EvmApi for MockEvmApi { &mut self, _code: Vec, _endowment: Bytes32, - _gas: u64, - ) -> (Result, u32, u64) { + _gas: Gas, + ) -> (Result, u32, Gas) { unimplemented!() } @@ -171,8 +171,8 @@ impl EvmApi for MockEvmApi { _code: Vec, _endowment: Bytes32, _salt: Bytes32, - _gas: u64, - ) -> (Result, u32, u64) { + _gas: Gas, + ) -> (Result, u32, Gas) { unimplemented!() } @@ -185,19 +185,19 @@ impl EvmApi for MockEvmApi { Ok(()) } - fn account_balance(&mut self, _address: Bytes20) -> (Bytes32, u64) { + fn account_balance(&mut self, _address: Bytes20) -> (Bytes32, Gas) { unimplemented!() } - fn account_code(&mut self, _address: Bytes20, _gas_left: u64) -> (VecReader, u64) { + fn account_code(&mut self, _address: Bytes20, _gas_left: Gas) -> (VecReader, Gas) { unimplemented!() } - fn account_codehash(&mut self, _address: Bytes20) -> (Bytes32, u64) { + fn account_codehash(&mut self, _address: Bytes20) -> (Bytes32, Gas) { unimplemented!() } - fn add_pages(&mut self, pages: u16) -> u64 { + fn add_pages(&mut self, pages: u16) -> Gas { let model = MemoryModel::new(2, 1000); unsafe { let (open, ever) = (OPEN_PAGES, EVER_PAGES); @@ -212,8 +212,8 @@ impl EvmApi for MockEvmApi { _name: &str, _args: &[u8], _outs: &[u8], - _start_ink: u64, - _end_ink: u64, + _start_ink: Ink, + _end_ink: Ink, ) { unimplemented!() } From 7bc39d3e0feb879d8ea0f1daacb1c2806d8babdf Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Mon, 14 Oct 2024 20:09:06 -0500 Subject: [PATCH 378/489] Fix formatting --- arbitrator/prover/src/machine.rs | 7 ++++++- arbitrator/prover/src/programs/meter.rs | 9 ++++++++- arbitrator/stylus/src/test/mod.rs | 5 ++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/arbitrator/prover/src/machine.rs b/arbitrator/prover/src/machine.rs index 66992019f9..dec355ac7c 100644 --- a/arbitrator/prover/src/machine.rs +++ b/arbitrator/prover/src/machine.rs @@ -1816,7 +1816,12 @@ impl Machine { } #[cfg(feature = "native")] - pub fn call_user_func(&mut self, func: &str, args: Vec, ink: arbutil::evm::api::Ink) -> Result> { + pub fn call_user_func( + &mut self, + func: &str, + args: Vec, + ink: arbutil::evm::api::Ink, + ) -> Result> { self.set_ink(ink); self.call_function("user", func, args) } diff --git a/arbitrator/prover/src/programs/meter.rs b/arbitrator/prover/src/programs/meter.rs index fe98bbcbc1..0d7b3151d7 100644 --- a/arbitrator/prover/src/programs/meter.rs +++ b/arbitrator/prover/src/programs/meter.rs @@ -9,7 +9,14 @@ use crate::{ value::FunctionType, Machine, }; -use arbutil::{evm::{self, api::{Gas, Ink}}, operator::OperatorInfo, Bytes32}; +use arbutil::{ + evm::{ + self, + api::{Gas, Ink}, + }, + operator::OperatorInfo, + Bytes32, +}; use derivative::Derivative; use eyre::Result; use fnv::FnvHashMap as HashMap; diff --git a/arbitrator/stylus/src/test/mod.rs b/arbitrator/stylus/src/test/mod.rs index 830b9cd23f..3fd0faede8 100644 --- a/arbitrator/stylus/src/test/mod.rs +++ b/arbitrator/stylus/src/test/mod.rs @@ -3,7 +3,10 @@ use crate::{env::WasmEnv, native::NativeInstance, run::RunProgram, test::api::TestEvmApi}; use arbutil::{ - evm::{api::{Ink, VecReader}, user::UserOutcome}, + evm::{ + api::{Ink, VecReader}, + user::UserOutcome, + }, Bytes20, Bytes32, Color, }; use eyre::{bail, Result}; From af4c7b53ade34583190961649d1922db8607497f Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Tue, 15 Oct 2024 11:31:33 +0530 Subject: [PATCH 379/489] Test precompile methods related to stylus --- system_tests/program_test.go | 128 +++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) diff --git a/system_tests/program_test.go b/system_tests/program_test.go index 4755096b26..61a157e5f8 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -1242,6 +1242,134 @@ func testSdkStorage(t *testing.T, jit bool) { check() } +func TestStylusPrecompileMethodsSimple(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, true) + cleanup := builder.Build(t) + defer cleanup() + + arbOwner, err := pgen.NewArbOwner(types.ArbOwnerAddress, builder.L2.Client) + Require(t, err) + arbDebug, err := pgen.NewArbDebug(types.ArbDebugAddress, builder.L2.Client) + Require(t, err) + arbWasm, err := pgen.NewArbWasm(types.ArbWasmAddress, builder.L2.Client) + Require(t, err) + + ensure := func(tx *types.Transaction, err error) *types.Receipt { + t.Helper() + Require(t, err) + receipt, err := EnsureTxSucceeded(ctx, builder.L2.Client, tx) + Require(t, err) + return receipt + } + + ownerAuth := builder.L2Info.GetDefaultTransactOpts("Owner", ctx) + ensure(arbDebug.BecomeChainOwner(&ownerAuth)) + + // ArbOwner precompile methods + testConst := uint16(10) + ensure(arbOwner.SetInkPrice(&ownerAuth, uint32(testConst))) + ensure(arbOwner.SetWasmMaxStackDepth(&ownerAuth, uint32(testConst))) + ensure(arbOwner.SetWasmFreePages(&ownerAuth, testConst)) + ensure(arbOwner.SetWasmPageGas(&ownerAuth, testConst)) + ensure(arbOwner.SetWasmPageLimit(&ownerAuth, testConst)) + // Setting low values of gas and cached parameters ensures when MinInitGas is called on ArbWasm precompile, + // the returned values would be programs.MinInitGasUnits and programs.MinCachedGasUnits + ensure(arbOwner.SetWasmMinInitGas(&ownerAuth, 1, 1)) + ensure(arbOwner.SetWasmInitCostScalar(&ownerAuth, uint64(testConst))) + expectedExpiryDays := uint16(1) + ensure(arbOwner.SetWasmExpiryDays(&ownerAuth, expectedExpiryDays)) + ensure(arbOwner.SetWasmKeepaliveDays(&ownerAuth, 0)) + ensure(arbOwner.SetWasmBlockCacheSize(&ownerAuth, testConst)) + + // ArbWasm precompile methods + wasm, _ := readWasmFile(t, rustFile("keccak")) + codehash := crypto.Keccak256Hash(wasm) + programAddress := deployContract(t, ctx, ownerAuth, builder.L2.Client, wasm) + + activateAuth := ownerAuth + activateAuth.Value = oneEth + ensure(arbWasm.ActivateProgram(&activateAuth, programAddress)) + + bcs, err := arbWasm.BlockCacheSize(nil) + Require(t, err) + if bcs != testConst { + t.Errorf("BlockCacheSize from arbWasm precompile didnt match the value set by arbowner. have: %d, want: %d", bcs, testConst) + } + ed, err := arbWasm.ExpiryDays(nil) + Require(t, err) + if ed != expectedExpiryDays { + t.Errorf("ExpiryDays from arbWasm precompile didnt match the value set by arbowner. have: %d, want: %d", ed, expectedExpiryDays) + } + fp, err := arbWasm.FreePages(nil) + Require(t, err) + if fp != testConst { + t.Errorf("FreePages from arbWasm precompile didnt match the value set by arbowner. have: %d, want: %d", fp, testConst) + } + ics, err := arbWasm.InitCostScalar(nil) + Require(t, err) + if ics != uint64(testConst) { + t.Errorf("InitCostScalar from arbWasm precompile didnt match the value set by arbowner. have: %d, want: %d", ics, testConst) + } + ip, err := arbWasm.InkPrice(nil) + Require(t, err) + if ip != uint32(testConst) { + t.Errorf("InkPrice from arbWasm precompile didnt match the value set by arbowner. have: %d, want: %d", ip, testConst) + } + kad, err := arbWasm.KeepaliveDays(nil) + Require(t, err) + if kad != 0 { + t.Errorf("KeepaliveDays from arbWasm precompile didnt match the value set by arbowner. have: %d, want: 0", kad) + } + msd, err := arbWasm.MaxStackDepth(nil) + Require(t, err) + if msd != uint32(testConst) { + t.Errorf("MaxStackDepth from arbWasm precompile didnt match the value set by arbowner. have: %d, want: %d", msd, testConst) + } + mig, err := arbWasm.MinInitGas(nil) + Require(t, err) + if mig.Gas != programs.MinInitGasUnits { + t.Errorf("MinInitGas from arbWasm precompile didnt match the Gas value set by arbowner. have: %d, want: %d", mig.Gas, programs.MinInitGasUnits) + } + if mig.Cached != programs.MinCachedGasUnits { + t.Errorf("MinInitGas from arbWasm precompile didnt match the Cached value set by arbowner. have: %d, want: %d", mig.Cached, programs.MinCachedGasUnits) + } + pg, err := arbWasm.PageGas(nil) + Require(t, err) + if pg != testConst { + t.Errorf("PageGas from arbWasm precompile didnt match the value set by arbowner. have: %d, want: %d", pg, testConst) + } + pl, err := arbWasm.PageLimit(nil) + Require(t, err) + if pl != testConst { + t.Errorf("PageLimit from arbWasm precompile didnt match the value set by arbowner. have: %d, want: %d", pl, testConst) + } + // pageramp currently is initialPageRamp = 620674314 value in programs package + _, err = arbWasm.PageRamp(nil) + Require(t, err) + + cas, err := arbWasm.CodehashAsmSize(nil, codehash) + Require(t, err) + if cas == 0 { + t.Error("CodehashAsmSize from arbWasm precompile returned 0 value") + } + ptl, err := arbWasm.ProgramTimeLeft(nil, programAddress) + Require(t, err) + expectedExpirySeconds := (uint64(expectedExpiryDays) * 24 * 3600) + // ProgramTimeLeft returns time in seconds to expiry and the current ExpiryDays is set to 1 day + // We expect the lag of 3600 seconds to exist because program.activatedAt uses hoursSinceArbitrum that + // rounds down (the current time since ArbitrumStartTime in hours)/3600 + if expectedExpirySeconds-ptl > 3600 { + t.Errorf("ProgramTimeLeft from arbWasm precompile returned value lesser than expected. %d <= want <= %d, have: %d", expectedExpirySeconds-3600, expectedExpirySeconds, ptl) + } + // Since ArbOwner has set wasm KeepaliveDays to 0, it enables us to do this, though this shouldn't have any effect + codehashKeepaliveAuth := ownerAuth + codehashKeepaliveAuth.Value = oneEth + ensure(arbWasm.CodehashKeepalive(&codehashKeepaliveAuth, codehash)) +} + func TestProgramActivationLogs(t *testing.T) { t.Parallel() builder, auth, cleanup := setupProgramTest(t, true) From eb2c5be47bca0f1089a61a1365884fbec17a3c02 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Tue, 15 Oct 2024 14:55:06 +0530 Subject: [PATCH 380/489] address PR comments --- .github/workflows/ci.yml | 4 ++-- system_tests/common_test.go | 28 ++++++++++++++++++++++------ system_tests/program_test.go | 8 +++----- validator/inputs/writer.go | 8 +++----- 4 files changed, 30 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 660f9f4dea..b4966a9e95 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -173,13 +173,13 @@ jobs: - name: create block input json file if: matrix.test-mode == 'defaults' run: | - gotestsum --format short-verbose -- -run TestProgramStorage$ ./system_tests/... --count 1 --validator.inputswriter.basedir="${{ github.workspace }}/target" + gotestsum --format short-verbose -- -run TestProgramStorage$ ./system_tests/... --count 1 --recordBlockInputs.WithBaseDir="${{ github.workspace }}/target" --recordBlockInputs.WithTimestampDirEnabled=false --recordBlockInputs.WithBlockIdInFileNameEnabled=false - name: run arbitrator prover on block input json if: matrix.test-mode == 'defaults' run: | make build-prover-bin - target/bin/prover target/machines/latest/machine.wavm.br -b --json-inputs="${{ github.workspace }}/target/block_inputs.json" + target/bin/prover target/machines/latest/machine.wavm.br -b --json-inputs="${{ github.workspace }}/target/TestProgramStorage/block_inputs.json" - name: run challenge tests if: matrix.test-mode == 'challenge' diff --git a/system_tests/common_test.go b/system_tests/common_test.go index f4e2edad42..7700ec28e4 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -9,6 +9,7 @@ import ( "encoding/binary" "encoding/hex" "encoding/json" + "flag" "io" "math/big" "net" @@ -1716,10 +1717,18 @@ func logParser[T any](t *testing.T, source string, name string) func(*types.Log) } } +var ( + recordBlockInputsEnable = flag.Bool("recordBlockInputs.enable", true, "Whether to record block inputs as a json file") + recordBlockInputsWithSlug = flag.String("recordBlockInputs.WithSlug", "", "Slug directory for validationInputsWriter") + recordBlockInputsWithBaseDir = flag.String("recordBlockInputs.WithBaseDir", "", "Base directory for validationInputsWriter") + recordBlockInputsWithTimestampDirEnabled = flag.Bool("recordBlockInputs.WithTimestampDirEnabled", true, "Whether to add timestamp directory while recording block inputs") + recordBlockInputsWithBlockIdInFileNameEnabled = flag.Bool("recordBlockInputs.WithBlockIdInFileNameEnabled", true, "Whether to record block inputs using test specific block_id") +) + // recordBlock writes a json file with all of the data needed to validate a block. // // This can be used as an input to the arbitrator prover to validate a block. -func recordBlock(t *testing.T, block uint64, builder *NodeBuilder, baseDirectory string) { +func recordBlock(t *testing.T, block uint64, builder *NodeBuilder) { t.Helper() ctx := builder.ctx inboxPos := arbutil.MessageIndex(block) @@ -1733,11 +1742,18 @@ func recordBlock(t *testing.T, block uint64, builder *NodeBuilder, baseDirectory break } } - validationInputsWriter, err := inputs.NewWriter( - inputs.WithBaseDir(baseDirectory), - inputs.WithTimestampDirEnabled(false), - inputs.WithBlockIdInFileNameEnabled(false), - ) + var options []inputs.WriterOption + options = append(options, inputs.WithTimestampDirEnabled(*recordBlockInputsWithTimestampDirEnabled)) + options = append(options, inputs.WithBlockIdInFileNameEnabled(*recordBlockInputsWithBlockIdInFileNameEnabled)) + if *recordBlockInputsWithBaseDir != "" { + options = append(options, inputs.WithBaseDir(*recordBlockInputsWithBaseDir)) + } + if *recordBlockInputsWithSlug != "" { + options = append(options, inputs.WithSlug(*recordBlockInputsWithSlug)) + } else { + options = append(options, inputs.WithSlug(t.Name())) + } + validationInputsWriter, err := inputs.NewWriter(options...) Require(t, err) inputJson, err := builder.L2.ConsensusNode.StatelessBlockValidator.ValidationInputsAt(ctx, inboxPos, rawdb.TargetWavm) if err != nil { diff --git a/system_tests/program_test.go b/system_tests/program_test.go index d007a40ddd..d6324df301 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -394,8 +394,6 @@ func errorTest(t *testing.T, jit bool) { validateBlocks(t, 7, jit, builder) } -var validatorInputsWriterBaseDir = flag.String("validator.inputswriter.basedir", "", "Base directory for validationInputsWriter") - func TestProgramStorage(t *testing.T) { t.Parallel() storageTest(t, true) @@ -426,11 +424,11 @@ func storageTest(t *testing.T, jit bool) { validateBlocks(t, 2, jit, builder) - // Captures a block_input_.json file for the block that included the + // Captures a block_inputs json file for the block that included the // storage write transaction. Include wasm targets necessary for arbitrator prover and jit binaries flag.Parse() - if *validatorInputsWriterBaseDir != "" { - recordBlock(t, receipt.BlockNumber.Uint64(), builder, *validatorInputsWriterBaseDir) + if *recordBlockInputsEnable { + recordBlock(t, receipt.BlockNumber.Uint64(), builder) } } diff --git a/validator/inputs/writer.go b/validator/inputs/writer.go index fc7456b8bf..1a476c52a3 100644 --- a/validator/inputs/writer.go +++ b/validator/inputs/writer.go @@ -146,13 +146,11 @@ func (w *Writer) Write(json *server_api.InputJSON) error { if err != nil { return err } - var fileName string + fileName := "block_inputs.json" if w.useBlockIdInFileName { - fileName = filepath.Join(dir, fmt.Sprintf("block_inputs_%d.json", json.Id)) - } else { - fileName = filepath.Join(dir, "block_inputs.json") + fileName = fmt.Sprintf("block_inputs_%d.json", json.Id) } - if err = os.WriteFile(fileName, contents, 0600); err != nil { + if err = os.WriteFile(filepath.Join(dir, fileName), contents, 0600); err != nil { return err } return nil From 2b1d7a26cd557333183f4358f7180ecb0d74aff5 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Tue, 15 Oct 2024 19:22:20 +0530 Subject: [PATCH 381/489] prioritize test flag over the test's decision to record a block --- system_tests/common_test.go | 4 ++++ system_tests/program_test.go | 6 +----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 7700ec28e4..274e368b74 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -1730,6 +1730,10 @@ var ( // This can be used as an input to the arbitrator prover to validate a block. func recordBlock(t *testing.T, block uint64, builder *NodeBuilder) { t.Helper() + flag.Parse() + if !*recordBlockInputsEnable { + return + } ctx := builder.ctx inboxPos := arbutil.MessageIndex(block) for { diff --git a/system_tests/program_test.go b/system_tests/program_test.go index d6324df301..9aac1eb589 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -7,7 +7,6 @@ import ( "bytes" "context" "encoding/binary" - "flag" "fmt" "math" "math/big" @@ -426,10 +425,7 @@ func storageTest(t *testing.T, jit bool) { // Captures a block_inputs json file for the block that included the // storage write transaction. Include wasm targets necessary for arbitrator prover and jit binaries - flag.Parse() - if *recordBlockInputsEnable { - recordBlock(t, receipt.BlockNumber.Uint64(), builder) - } + recordBlock(t, receipt.BlockNumber.Uint64(), builder) } func TestProgramTransientStorage(t *testing.T) { From 654efa95629ca83a1fc7441e5f761348a1cb3140 Mon Sep 17 00:00:00 2001 From: Nick Hovsmith Date: Tue, 15 Oct 2024 16:19:45 -0400 Subject: [PATCH 382/489] remove docker process --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 88bbd8dabe..4c5550d199 100644 --- a/Makefile +++ b/Makefile @@ -579,9 +579,9 @@ contracts/test/prover/proofs/%.json: $(arbitrator_cases)/%.wasm $(prover_bin) @touch $@ .make/cbrotli-wasm: $(DEP_PREDICATE) $(ORDER_ONLY_PREDICATE) .make - test -f target/lib-wasm/libbrotlicommon-static.a || ./scripts/build-brotli.sh -w -d - test -f target/lib-wasm/libbrotlienc-static.a || ./scripts/build-brotli.sh -w -d - test -f target/lib-wasm/libbrotlidec-static.a || ./scripts/build-brotli.sh -w -d + test -f target/lib-wasm/libbrotlicommon-static.a || ./scripts/build-brotli.sh -w + test -f target/lib-wasm/libbrotlienc-static.a || ./scripts/build-brotli.sh -w + test -f target/lib-wasm/libbrotlidec-static.a || ./scripts/build-brotli.sh -w @touch $@ .make/wasm-lib: $(DEP_PREDICATE) arbitrator/wasm-libraries/soft-float/SoftFloat/build/Wasm-Clang/softfloat.a $(ORDER_ONLY_PREDICATE) .make From 087e59b4982eaed7d517ae55368777ef27928812 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Tue, 15 Oct 2024 21:57:23 -0500 Subject: [PATCH 383/489] Address PR comments --- arbitrator/arbutil/src/evm/api.rs | 10 ++++++++++ arbitrator/arbutil/src/pricing.rs | 2 +- arbitrator/prover/src/programs/memory.rs | 6 +++--- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/arbitrator/arbutil/src/evm/api.rs b/arbitrator/arbutil/src/evm/api.rs index 8715f5ac8c..0a603a3bb2 100644 --- a/arbitrator/arbutil/src/evm/api.rs +++ b/arbitrator/arbutil/src/evm/api.rs @@ -120,6 +120,16 @@ macro_rules! derive_math { } impl $t { + /// Equivalent to the Add trait, but const. + pub const fn add(self, rhs: Self) -> Self { + Self(self.0 + rhs.0) + } + + /// Equivalent to the Sub trait, but const. + pub const fn sub(self, rhs: Self) -> Self { + Self(self.0 - rhs.0) + } + pub const fn saturating_add(self, rhs: Self) -> Self { Self(self.0.saturating_add(rhs.0)) } diff --git a/arbitrator/arbutil/src/pricing.rs b/arbitrator/arbutil/src/pricing.rs index 91de739303..4d6bf827be 100644 --- a/arbitrator/arbutil/src/pricing.rs +++ b/arbitrator/arbutil/src/pricing.rs @@ -7,7 +7,7 @@ use crate::evm::api::Ink; pub const HOSTIO_INK: Ink = Ink(8400); /// For hostios that include pointers. -pub const PTR_INK: Ink = Ink(13440 - HOSTIO_INK.0); +pub const PTR_INK: Ink = Ink(13440).sub(HOSTIO_INK); /// For hostios that involve an API cost. pub const EVM_API_INK: Ink = Ink(59673); diff --git a/arbitrator/prover/src/programs/memory.rs b/arbitrator/prover/src/programs/memory.rs index 758f2f3e82..82c4d4469d 100644 --- a/arbitrator/prover/src/programs/memory.rs +++ b/arbitrator/prover/src/programs/memory.rs @@ -83,14 +83,14 @@ fn test_model() { let model = MemoryModel::new(2, 1000); for jump in 1..=128 { - let mut total = 0; + let mut total = Gas(0); let mut pages = 0; while pages < 128 { let jump = jump.min(128 - pages); - total += model.gas_cost(jump, pages, pages).0; + total += model.gas_cost(jump, pages, pages); pages += jump; } - assert_eq!(total, 31999998); + assert_eq!(total, Gas(31999998)); } for jump in 1..=128 { From 8f961b0fc9f3ba5997c0d957759fb60a08552802 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Tue, 15 Oct 2024 22:23:10 -0500 Subject: [PATCH 384/489] Regenerate data-poster external-signer test certs --- arbnode/dataposter/testdata/client.crt | 45 ++++++++-------- arbnode/dataposter/testdata/client.key | 52 +++++++++---------- arbnode/dataposter/testdata/localhost.crt | 48 ++++++++--------- arbnode/dataposter/testdata/localhost.key | 52 +++++++++---------- .../dataposter/testdata/regenerate-certs.sh | 8 +++ 5 files changed, 103 insertions(+), 102 deletions(-) create mode 100755 arbnode/dataposter/testdata/regenerate-certs.sh diff --git a/arbnode/dataposter/testdata/client.crt b/arbnode/dataposter/testdata/client.crt index 3d494be820..9171094ba5 100644 --- a/arbnode/dataposter/testdata/client.crt +++ b/arbnode/dataposter/testdata/client.crt @@ -1,28 +1,25 @@ -----BEGIN CERTIFICATE----- -MIIE0jCCA7qgAwIBAgIUPaBB3/hHMpZfGB3VOw1+mHG4LnUwDQYJKoZIhvcNAQEL +MIIEIjCCAwqgAwIBAgIUV1axsouzA9h1Vgr2cPv17AvUrKswDQYJKoZIhvcNAQEL BQAwgYMxCzAJBgNVBAYTAkNIMQswCQYDVQQIDAJaSDEPMA0GA1UEBwwGWnVyaWNo MRYwFAYDVQQKDA1PZmZjaGFpbiBMYWJzMRIwEAYDVQQDDAlsb2NhbGhvc3QxKjAo -BgkqhkiG9w0BCQEWG25vdGFiaWdkZWFsQG9mZmNoYWlubGFicy5jaDAeFw0yMzEw -MTYxNDU2MjhaFw0yNDEwMTUxNDU2MjhaMIGDMQswCQYDVQQGEwJDSDELMAkGA1UE -CAwCWkgxDzANBgNVBAcMBlp1cmljaDEWMBQGA1UECgwNT2ZmY2hhaW4gTGFiczES -MBAGA1UEAwwJbG9jYWxob3N0MSowKAYJKoZIhvcNAQkBFhtub3RhYmlnZGVhbEBv -ZmZjaGFpbmxhYnMuY2gwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC1 -1asfUzv07QTVwlM4o3g51ilIFEApPkpdQej/GIItLEVRQW+GI9jYuEM07wdwMhSH -JPFNbZB3dmBuqDLx13hY03ufyeY+nab0/sO6x13kXChvIqgPRyJtkEAoYkMM3W0D -S6HeL/6DFoTQ2xAlZb/7i/9deuUwDL3MNVSjPCm9PjFzSOFgAQQud2uUT7aENGuG -Whw3oXz9gU/8gv3keLzcIa2PHyEW5M7jeGSYMjfW3wr0d+Z5mSNRc/U6kncKi06c -QrMKrgFfF7a5kHgxUL7bRCGgCMemXe7VfrW6oKT11JcLWDKhe+uo6bNXUptek55H -HfQi6x8cbM46/h3riZA3AgMBAAGjggE6MIIBNjAdBgNVHQ4EFgQUQD2BOems0+JQ -br234cW5noMmXRIwga0GA1UdIwSBpTCBoqGBiaSBhjCBgzELMAkGA1UEBhMCQ0gx -CzAJBgNVBAgMAlpIMQ8wDQYDVQQHDAZadXJpY2gxFjAUBgNVBAoMDU9mZmNoYWlu -IExhYnMxEjAQBgNVBAMMCWxvY2FsaG9zdDEqMCgGCSqGSIb3DQEJARYbbm90YWJp -Z2RlYWxAb2ZmY2hhaW5sYWJzLmNoghQ9oEHf+Ecyll8YHdU7DX6YcbgudTAJBgNV -HRMEAjAAMAsGA1UdDwQEAwIFoDAfBgNVHREEGDAWgglsb2NhbGhvc3SCCTEyNy4w -LjAuMTAsBglghkgBhvhCAQ0EHxYdT3BlblNTTCBHZW5lcmF0ZWQgQ2VydGlmaWNh -dGUwDQYJKoZIhvcNAQELBQADggEBAF4EVkOZZeMIvv0JViP7NsmIl2ke/935x6Hd -hQiLUw13XHYXzMa5/8Y5fnKjttBODpFoQlwjgI18vzuYzItYMBc2cabQJcpfG+Wq -M3m/wl1TC2XOuHj1E4RA/nU3tslntahtXG+vkks9RN+f9irHUhDRR6AGSnSB2Gi/ -B2OGmXn7S4Qge8+fGHAjN+tlu+tOoEWP6R3if/a9UIe5EGM8QTe4zw6lr+iPrOhC -M94pK5IEWn5IIGhr3zJIYkm/Dp+rFqhV1sqPOjjFLVCA7KJ3jVVVHlcm4Xa/+fyk -CIm7/VAmnbeUNlMbkXNOfQMeku8Iwsu80pvf3kjhU/PgO/5oojk= +BgkqhkiG9w0BCQEWG25vdGFiaWdkZWFsQG9mZmNoYWlubGFicy5jaDAgFw0yNDEw +MTYwMzI1NDdaGA8yMTI0MDkyMjAzMjU0N1owgYMxCzAJBgNVBAYTAkNIMQswCQYD +VQQIDAJaSDEPMA0GA1UEBwwGWnVyaWNoMRYwFAYDVQQKDA1PZmZjaGFpbiBMYWJz +MRIwEAYDVQQDDAlsb2NhbGhvc3QxKjAoBgkqhkiG9w0BCQEWG25vdGFiaWdkZWFs +QG9mZmNoYWlubGFicy5jaDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +AKU5q5iwYV4/gPeWcyys561pTGV4pk+sRY2q0znsZFOYcxrjaXEjj2HGNkvH1rKy +8Cv1ZoFW+1ejQZeLtd0qL9v5fDkdLsmCZaIYI5Bvo2CfY6KLUZ5c1q2K2GZgQk8i +eSbqBXq+F/EwziDfheXkhDoAE05hOg684titb21eJ0ZK7f7Koam7cmbQI0lqUCrt +MLp0cJzWnfW0SpCzahnCZ5h31BZeZIRLOxsTvg5N1wOivrdWLXGVbprNCGGhVg0E +ZxhwI00pU/E/K4mcKjtPy/5fqe71jH7/iLYNmhRp6PrA78GilxTT79rro8ooantD +GyQbm+Qkk2tMHHum3GOcjuECAwEAAaOBiTCBhjAdBgNVHQ4EFgQUIUFF6jA0NkRA +1kZhJKH0W/9zJWIwCQYDVR0TBAIwADALBgNVHQ8EBAMCBaAwHwYDVR0RBBgwFoIJ +bG9jYWxob3N0ggkxMjcuMC4wLjEwLAYJYIZIAYb4QgENBB8WHU9wZW5TU0wgR2Vu +ZXJhdGVkIENlcnRpZmljYXRlMA0GCSqGSIb3DQEBCwUAA4IBAQCkfknujeFa0yf4 +YX/3ltP9itq4hLtAYnQF7M/uC86QdyDPsrNqhvj54qC0BnR5wGeZP3c144J2mAUr +4j4Y/ztgFVBR4rLyatHgm0/tL/fy/UgjeSmpY4UOr1QnpNP3fIzL7hxacS4uO8v4 +wcc5KlG/xjHRcrzJaaWLldCogBMb8vlModcbeKrkvQ4hUF+zf138RtpRfcRf1X5c +EaAtUZk+BxVYS79qL7YyESRD8YYMhIImLuiyPt2V3HQRhrjqa3mzODBLhUbNRWPX +8/CH2UZ6TD9Hy4FVX0VZzLoDZjfi4KCTgXI3WGrDoL4FF26cSiK8HVx0qJzAWw4a +tkkj5jtd -----END CERTIFICATE----- diff --git a/arbnode/dataposter/testdata/client.key b/arbnode/dataposter/testdata/client.key index b14941dd9f..4313d0f124 100644 --- a/arbnode/dataposter/testdata/client.key +++ b/arbnode/dataposter/testdata/client.key @@ -1,28 +1,28 @@ -----BEGIN PRIVATE KEY----- -MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQC11asfUzv07QTV -wlM4o3g51ilIFEApPkpdQej/GIItLEVRQW+GI9jYuEM07wdwMhSHJPFNbZB3dmBu -qDLx13hY03ufyeY+nab0/sO6x13kXChvIqgPRyJtkEAoYkMM3W0DS6HeL/6DFoTQ -2xAlZb/7i/9deuUwDL3MNVSjPCm9PjFzSOFgAQQud2uUT7aENGuGWhw3oXz9gU/8 -gv3keLzcIa2PHyEW5M7jeGSYMjfW3wr0d+Z5mSNRc/U6kncKi06cQrMKrgFfF7a5 -kHgxUL7bRCGgCMemXe7VfrW6oKT11JcLWDKhe+uo6bNXUptek55HHfQi6x8cbM46 -/h3riZA3AgMBAAECggEADUboCYMCpm+LqIhzNCtqswQD6QsiSwCmqs8nuKZGk9ue -+hmZj5IpgMJZLrgvWY4s+PGfgiRR/28QCBrVXkETiZ5zirQFN4tvLlKcSK4xZf29 -FBRUCiPxck36NhiqrBNOi1Mn8BKedl4cESkvSu1cvcmeOh100HPcHfLDVqHx3qsl -D/5yMkT2+zdhtLa+X3nkAa+3aibOvgtyfkV679e20CG6h89N9GBKkTXO8ioLZZVm -84ksnd4FcpTo7ebJJxElEB+ZA4akPHbF6ArUmcpqtGso5GtwqqO2ZlguSn2XQT0d -jqvOG4DwfSXk6SpE/dpWvU92fmxWAxZvGrZNgDyJ2QKBgQDyQ8NN4b80Yza/YXar -LWx8A6B0eMc1dXgt9m3UUI+titt45jEcaXhCX01FRFTznWGmWFtJmcWBoaQVPVel -IcDYQSxEuBUrCeI75ocv/IQtENaiX3TK7Nlz5RHfpQpfDVJq45lpiD38CGkYkAif -9pSzC8aup4W3WR0JJZ1AOHUZaQKBgQDAJNJnaSNzB+eDWTKCIN5V9X3QMkmjsuir -Nf2lBXHYARnlYWAbtYFG12wLJQMTNX5ewVQQrWtsdPkGPpCnPLelUTxMssrsXjej -JlLzYUfzRBqEXMI3AA9bVdiauxId2RTcp2F81SM1keCMcuHYxrzVkBSOC9u3wCnb -Whb6+feInwKBgQCbzgC5AcoaQwReqKvNAvWV/C8hONvFAbs8tBOGTBlbHsZvRnun -Lh1tciUbuwp3cmvuszxiZUakS/RexIitZrvDWIbD2y+h8kVRCL1Am0HWSdH/syxF -pXVkF5obHuVApCyxGZb8S+axRCdy6I7jcY3IaHZqtMpGVEVcMJilSKnmoQKBgQCC -tEmgaMfhhx34nqOaG4vDA4T7LEolnh1h4g9RwztnCZC5FZ1QHA79xqrLhfjqhzgY -cwChe6aYl5WSptq1uLrgLTuMnQ8m7QyB4h8JSkKse8ZiBctjqJnJssLutpSjUzk6 -xG2vgjk6RqpuP/PcB40K5cDlw7FJ9OFEQqthPMsi1wKBgQC0/vv5bY3DQ+wV6gUy -nFoSa/XNHaa8y7jmmlCnWJqs6DAAQQ3VW0tPX03GYL/NDcI+PwzYDHDkSB6Qa/o8 -VzVGK1/kr/+bveNvqmi0vNb54fMFLveGgsY4Cu1cffiw8m6nYJ/V4eCsHfpF1B5L -5HDnt5rFKt1Mi9WsUSRtxipxBA== +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQClOauYsGFeP4D3 +lnMsrOetaUxleKZPrEWNqtM57GRTmHMa42lxI49hxjZLx9aysvAr9WaBVvtXo0GX +i7XdKi/b+Xw5HS7JgmWiGCOQb6Ngn2Oii1GeXNatithmYEJPInkm6gV6vhfxMM4g +34Xl5IQ6ABNOYToOvOLYrW9tXidGSu3+yqGpu3Jm0CNJalAq7TC6dHCc1p31tEqQ +s2oZwmeYd9QWXmSESzsbE74OTdcDor63Vi1xlW6azQhhoVYNBGcYcCNNKVPxPyuJ +nCo7T8v+X6nu9Yx+/4i2DZoUaej6wO/BopcU0+/a66PKKGp7QxskG5vkJJNrTBx7 +ptxjnI7hAgMBAAECggEAAagHWVuDTl+SmmjOtMby96ETm/zOpgPTGq14up7tDo17 +sexPtUum91L2XmIde+MhVz95jJhjoqhHUw6afyIaIrlojmYFfw2omSxmxt7no2NV +q158Lfs+R7UZoEUcxRBSaJp1/ZoEQW2800WKYRieXrp7dxCwdU9dctCiSlVkTWcU +w+f9xr084dUIKgtICbLWRdGDvGFmr99MBZXzHg5+x8MiAVtpiNcggRfQKIg2QYQv +xdUtfxrKHuRcbeo4QOgSR6fb772F5eO6hPfwgl0AqmSX9XyRaPox/vKcq531S95S +JvGeAdS47Qo8Elh9rIlC/pxVdJ/Gz4sbmlYCfcXDRQKBgQDbSFUrEaOnH5ITEDy/ +SDTCbdQ3bP1FmHVoLdCRoBohb0xHZrJoOn6cmxyyHWR6dwpv+rvi1bJCScDdphJM +zV8W5sG94PaM/dwCws4CFAwaAlMakrsNUXtMgIeub27mzX5OMTss8vjKdKbVDyAv +XCT4idJY1EOdLA3R+JTLSszxJwKBgQDA5CSKLn4HpZI6qmR5g7HRZ5g249BbX/q8 +oszAUIFfY0ME5aujWYRmTfdWno0Y2yG9x4g9QDVNK9fUH3Ii6pNRFGc/yF3HkbsP +kT6UW6rw9CeyyYPKjrFx7M+2kBWJ16+5noVvzWhLScMCt7IcVCKaqiJFapOkS75t +zBYH1IX0twKBgCtFBqlM/cIIlMZ2OcZ09RQ4n9ugAgotn11DTRivQvi+AYtFVIcE +o987LFppOl6ABus5ysFj8Zzq+MfD8XB+Rfk655gUQBJqNXPGBOicFBc9xjBEK+zg +2zepVRyymGuquPWs+URRXY51nkYEihFOWW1BpOQqXn0xKDj6mEHVLMOZAoGAc7Ol +k1ll8ZJIT3ZLxHPRYqmALVSjc1v0G9iPdsATijMRTUuyk84rU+5qcYOzYPh4mcyp +FQyBrGOjF7MxFG6epSDW+fRnBEGO8jyOTBFcTSI2+dBUhFjpaUvCIGD2+nLtDitf +IPwWFisNlYC4jrOM+jcZTYgrPX7NoDCt+k5pd6sCgYB7NMYEC9XjoXV8S3n2yni5 +y0oGLox39hh31LJ90yQ/l+oFJWn1BPnzTI6xeJTTzJjQf7padyvfw45b+3BywZHM +TI5LWcIaA6L+tiBlgBjYa7gE3CCxh0AdV+pUa8L/R6OWAK6+lg2zNt1/ommZ2sKg +LcbNAqMiMWH1a1el7idoBA== -----END PRIVATE KEY----- diff --git a/arbnode/dataposter/testdata/localhost.crt b/arbnode/dataposter/testdata/localhost.crt index ca33dfc8cc..8b7fb02c9a 100644 --- a/arbnode/dataposter/testdata/localhost.crt +++ b/arbnode/dataposter/testdata/localhost.crt @@ -1,28 +1,24 @@ -----BEGIN CERTIFICATE----- -MIIEwzCCA6ugAwIBAgIUHx3SdpCP5jXZE7USUqX5uRNFKPIwDQYJKoZIhvcNAQEL -BQAwfzELMAkGA1UEBhMCQ0gxCzAJBgNVBAgMAlpIMQ8wDQYDVQQHDAZadXJpY2gx -FjAUBgNVBAoMDU9mZmNoYWluIExhYnMxEjAQBgNVBAMMCWxvY2FsaG9zdDEmMCQG -CSqGSIb3DQEJARYXYmlnZGVhbEBvZmZjaGFpbmxhYnMuY2gwHhcNMjMxMDE2MTQ0 -MDA1WhcNMjQxMDE1MTQ0MDA1WjB/MQswCQYDVQQGEwJDSDELMAkGA1UECAwCWkgx -DzANBgNVBAcMBlp1cmljaDEWMBQGA1UECgwNT2ZmY2hhaW4gTGFiczESMBAGA1UE -AwwJbG9jYWxob3N0MSYwJAYJKoZIhvcNAQkBFhdiaWdkZWFsQG9mZmNoYWlubGFi -cy5jaDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALg7XwaIh4l2Fp8a -MfNMdTQSMPMR0zpnicVTn/eiozWsqlAKaxmQM3PxJ0oVWW3iJ89p4rv5m+UjK6Dr -vsUQOzl8isgyGCTMnkLtxFlyallDNRDawRcuTPuNI9NkdJm+Zz7HooLzFeBDeS13 -iRPEXr1T/4af9MjOxqFvbw5xBY9k4tc2hPp6q00948gPWKIB9Mz4thoB2Hl2rQBY -X/WhjSnre9o9qoyBO0XAsG0mssBs1vPa9/aEp7C5cDY0HCuM1RIjhXnRpb8lC9VQ -aC+FozDffmm23EGVpLmyPs590UOtVJdTUd6Q0TAT6d7fjCRUJ12DendQf2uMFV90 -u6Yj0zUCAwEAAaOCATUwggExMB0GA1UdDgQWBBT2B3FTGFQ49JyBgDGLoZREOIGD -DTCBqAYDVR0jBIGgMIGdoYGEpIGBMH8xCzAJBgNVBAYTAkNIMQswCQYDVQQIDAJa -SDEPMA0GA1UEBwwGWnVyaWNoMRYwFAYDVQQKDA1PZmZjaGFpbiBMYWJzMRIwEAYD -VQQDDAlsb2NhbGhvc3QxJjAkBgkqhkiG9w0BCQEWF2JpZ2RlYWxAb2ZmY2hhaW5s -YWJzLmNoghQfHdJ2kI/mNdkTtRJSpfm5E0Uo8jAJBgNVHRMEAjAAMAsGA1UdDwQE -AwIFoDAfBgNVHREEGDAWgglsb2NhbGhvc3SCCTEyNy4wLjAuMTAsBglghkgBhvhC -AQ0EHxYdT3BlblNTTCBHZW5lcmF0ZWQgQ2VydGlmaWNhdGUwDQYJKoZIhvcNAQEL -BQADggEBAIkhBcnLeeNwUwb+sSG4Qm8JdeplHPMeViNfFIflUfIIYS00JA2q9w8W -+6Nh8s6Dn20lQETUnesYj97BdqzLjFuJYAlblhE+zP8g/3Mkpu+wZAGvQjUIRyGT -C17BEtQQgAnv5pD22jr9hpLl2KowN6Oo1gzilCA+AtMkNZFIGDOxzuIv2u8rSD89 -R/V6UEDMCgusFJnZ/GzKkUNbsrAfNUezNUal+KzMhHGHBwg4jfCNhnAAB43eRtJA -0pSRMMLcUEQnVotXDXYC3DhJmkYp1uXOH/tWs6z9xForOkWFxNMVj+zUWBi7n3Jw -N2BXlb64D96uor13U0dmvQJ72ooJc+A= +MIIEFzCCAv+gAwIBAgITSiI3ITH8yVNHC4Bh412fjdSR2TANBgkqhkiG9w0BAQsF +ADB/MQswCQYDVQQGEwJDSDELMAkGA1UECAwCWkgxDzANBgNVBAcMBlp1cmljaDEW +MBQGA1UECgwNT2ZmY2hhaW4gTGFiczESMBAGA1UEAwwJbG9jYWxob3N0MSYwJAYJ +KoZIhvcNAQkBFhdiaWdkZWFsQG9mZmNoYWlubGFicy5jaDAgFw0yNDEwMTYwMzI1 +NDdaGA8yMTI0MDkyMjAzMjU0N1owfzELMAkGA1UEBhMCQ0gxCzAJBgNVBAgMAlpI +MQ8wDQYDVQQHDAZadXJpY2gxFjAUBgNVBAoMDU9mZmNoYWluIExhYnMxEjAQBgNV +BAMMCWxvY2FsaG9zdDEmMCQGCSqGSIb3DQEJARYXYmlnZGVhbEBvZmZjaGFpbmxh +YnMuY2gwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDK4BRVZ0nU98/f +l+QC4fF60oNtlMPCC/2R6GZoWz7VyhrrXBuol+F9vboAePxLgxeIr/pwHAbFWlW2 +ueAsN9dorC2waf5PDhfOE0gI6w7LysTkO5n7oMFf1KYPSpPJ15WxlobZR8qWeroR +we7z44tQ2F+es+HaqBrrk7jm0GS9AqaledN/ay9SP4CBu029F6nWDnK+VpNWuoN4 +A/pnwGFWrxDf0ftN7BxnxzzdsWs64+kYfz91Mojce2UKGuDTuk/oqOnHhX34bFDc +/e9KGAQqP1I+RuCJmQXW5b55+3WgpvT3u3Mp7478C+AK8GthPjja7go48nHp3uby +drNpTw+bAgMBAAGjgYkwgYYwHQYDVR0OBBYEFG09BO7OJcjB3fRFhPCsjQ6ICb2E +MAkGA1UdEwQCMAAwCwYDVR0PBAQDAgWgMB8GA1UdEQQYMBaCCWxvY2FsaG9zdIIJ +MTI3LjAuMC4xMCwGCWCGSAGG+EIBDQQfFh1PcGVuU1NMIEdlbmVyYXRlZCBDZXJ0 +aWZpY2F0ZTANBgkqhkiG9w0BAQsFAAOCAQEAJHG/5WOmpO7wg3BtTt4b0DoqDJjh +eQb9Woq5xbvliZ1RCp8E6m6BmOr66i4qu5r+31DEeAeQ2M9pG2nJKoayCVi2ygaQ +RAulxIH7o5+JUcZtX6FbRBsS7Go+SLmtkkJ89YVSIF40+2CAQs7loqQjHNeo9/iO +rVKt1Fa6rQhXmv4ItVOwRaMBvXRVw4gc3ObmH0ZBYZrvsE7uQkKX5f6sVKXOX3mm +ofyB+22QMYmx3XvEEQm8ELnjIr5Q8LxqQxHqjLFjyrcrXYVi4+3/PfjIdRr5+qes +H8JWJlAbF/SNncdXRb1jtkdxit56Qo7/Mz/c4Yuh1WLiYcQGJeBpr53dmg== -----END CERTIFICATE----- diff --git a/arbnode/dataposter/testdata/localhost.key b/arbnode/dataposter/testdata/localhost.key index aad9b40b3d..f56aef1e77 100644 --- a/arbnode/dataposter/testdata/localhost.key +++ b/arbnode/dataposter/testdata/localhost.key @@ -1,28 +1,28 @@ -----BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC4O18GiIeJdhaf -GjHzTHU0EjDzEdM6Z4nFU5/3oqM1rKpQCmsZkDNz8SdKFVlt4ifPaeK7+ZvlIyug -677FEDs5fIrIMhgkzJ5C7cRZcmpZQzUQ2sEXLkz7jSPTZHSZvmc+x6KC8xXgQ3kt -d4kTxF69U/+Gn/TIzsahb28OcQWPZOLXNoT6eqtNPePID1iiAfTM+LYaAdh5dq0A -WF/1oY0p63vaPaqMgTtFwLBtJrLAbNbz2vf2hKewuXA2NBwrjNUSI4V50aW/JQvV -UGgvhaMw335pttxBlaS5sj7OfdFDrVSXU1HekNEwE+ne34wkVCddg3p3UH9rjBVf -dLumI9M1AgMBAAECggEAHuc8oyKrQ5xmooUZHGP2pAeqJNfYXAtqoYpLwtUJ9hKy -1e7NdNIKw3fP/J4UrHk7btAm65us8hSCeMGatEErAhNZT0gR4zhcksMCBPQLkVIT -+HINYjdOzAJqoEbRRUnaVT5VDQy8HmyLCtyqhoGR18XbjshNnhKLYKCJ2z0Lrvf2 -3rU7bbt7/rvLitVhxVL8SIe2jWSfIgcEmEAZMigB9WAnUyQ/tAfbPy1I764LLfzD -nLXn7E2OH7GrxkLjOsH9kfERlur7V7IhC9NE/wI0q+rnILRa7Q3+ifRu8qla3bo1 -iyHl1ZmsYJ8Jnzbu9exzZaQmk42OoFPcMFm0mRe+2QKBgQDvRv0Q5JhBuVurkU98 -lzATwEO0uYmeWDMnHzrFSWAKr/x4LNQ9ytSCfe1aLxgOkZq6dQ3TyZiCYzpmwGz9 -K7/gghxmsVDKeCqiGVZOgFAWy7AhQyF6zM60oqqwSvJHhmGTsA/B5LPUiYe9lITW -ZSLVYkOzha7Coa++U8vPzI5VaQKBgQDFG4reFT79j8RKEm9jie6PdRdYMzOSDWty -Gjj5N9Jnlp1k/6RzCxjmp7w7yIorq/7fWZsQtt0UqgayOn25+I8dZeGC0BradUSB -tZbGElxPsF8Jg00ZvvK3G5mpZYDrJCud8Q05EaUZPXv9GuZhozEsTQgylVecVzsN -wyEK8VuZ7QKBgQChx9adUGIdtgzkILiknbh08j8U94mz1SCo5/WdpLHaKAlE29KZ -AQXUQP51Rng2iX4bab9yndCPADZheON3/debHX3EdUkRzFPPC+CN7TW5Y/jvVGtT -kxyDh6Ru1A2iDJr290iAKXjpUB/GL5/tMa5upiTuQYnasOWZgyC/nCf0WQKBgEwn -pRLDMLA1IMjhsInL3BEvU1KvjahLaQ0P1p1rlO6TAcLpBrewPPG5MwACLmhLLtFK -xJ/Dl02Jl8a61KLKxzi7iVLKZuWq00ouR8/FfkcHxOBfC6X74bkff9I0NogjVHrU -jKBVEe3blJEpGIP20mPka1tn2g68oUNi9dxNfm/NAoGAWj/Q0pgnNq0MQ8Lj6m99 -1baaXSo8biks3E3A3cqhHQm/j3SRnkf0lueQW8+r9yR9IWdYFXz5Waq13qK+lopE -KDmww0xr8dyMUYTP1vde7np2XKa/OX3iejDzbI3RcZN/DEV+dCBY8pqHHfaAaESu -fwBWvfD8wtwCZzB3lOZEi80= +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDK4BRVZ0nU98/f +l+QC4fF60oNtlMPCC/2R6GZoWz7VyhrrXBuol+F9vboAePxLgxeIr/pwHAbFWlW2 +ueAsN9dorC2waf5PDhfOE0gI6w7LysTkO5n7oMFf1KYPSpPJ15WxlobZR8qWeroR +we7z44tQ2F+es+HaqBrrk7jm0GS9AqaledN/ay9SP4CBu029F6nWDnK+VpNWuoN4 +A/pnwGFWrxDf0ftN7BxnxzzdsWs64+kYfz91Mojce2UKGuDTuk/oqOnHhX34bFDc +/e9KGAQqP1I+RuCJmQXW5b55+3WgpvT3u3Mp7478C+AK8GthPjja7go48nHp3uby +drNpTw+bAgMBAAECggEANE2Q8HOwlTdOYFbIcfXOS9v6BkZUMbLlrLg9rqnXiUaR +qhwVBWIiwEgpq/WFFfK2HodACacwF7EyZ+mD4eKDpni9Tr4E0lzPxlEyQRpYtjGQ +kUbMbBMFx68LIOYZM/Bgp2gnW90mXaVGU02sTTRctnsSK9g0Yir0xcdP5DHVxuRy +cReMIDjHu7/PN5X12oHpBxT1w7TKYZk3M2ZbmRcGfRSv1UIg06hXjxY4Tonqxv5b +Yv/R/r+RYsHE6cO3hLV7FE/ypaRFGt/qcHhqwmHKlpc+8/FNEpb5truXCbf5LhjM +YD2eMOpiBaiyufs+2BpUy/iDKToYz+fdusT5N3FgAQKBgQDuemOauY/ytbETUabB +Vt11fCvXvsx4L+45vqtVcRWl6jcy73rMnyA80D/ndzyxkuw2NRAllIUvsdVG26vM +8nWAxrsY7rrZ4kdRAAKAPYmTtT2O4mvhvJOYHA/ueEUamrKqZMgyLik1Wl1y2mt2 +Iak0zNB0GWoHsgDF20TTbxKTkwKBgQDZyAa/r3qhokKfvg/7LhwXH7vdiJd35zp1 +K1KZVeZf97FeReL4DLfTHZ92yFyVhWepp97Icd2WtQZ8jVVcesTP9C+mVoFEGXZf +nWx5y2WD92dBP/kKYNayXuBFQnRfS4AC7ALK8fKrU/Fzc7OctnkvHHWZfaKQ2sQ5 +xydj6Rso2QKBgBpgvT2zAsIU6MY7RNej1REWr/7IIvO0UYRfm7HytTNJ6dsfdBTI +ERfI7RicLsFxf+ErE2Mkv2qcH/wbdjBQLUEWOkGyvkY1ajACcURgCiSlam6wisBI +TIcJq5V0BijALbz9MsuiIXq+SRHYKQTDCmVFtlTxLrI1NTKtYzqD0akzAoGAB5fW +zGYk42/R3Nn2mq5f4lqD5VR224JfYmhxR9Fb5+qt73iGUlm3KxA0WCLiP4BYPe0R +cnGt5SxInp0a5c+N/yYnZyhK94Hfw7OsbY6u6mv82KSPXVJFChEOxrtrbUsnmnJ6 +InNPH7QcjgbxszwVe5QFcaWUvnIyN0V/VRdyj/kCgYEA0kPCIPR7t7OFEnQsw50W +AclrTlGRB0TzmCYZN4DTKC+1ZAV80XQUR4umJpHeyArdRZpFv+HBLF7fHsBnkmiU +ixBdbWgW4mxjAhyZkLMjcCkoiLmRPDUvELyFs4xpM+IEKoAk60PPbGi4CeIMHb5k +E47NZaw1bdG1tv2ya1FlQ/k= -----END PRIVATE KEY----- diff --git a/arbnode/dataposter/testdata/regenerate-certs.sh b/arbnode/dataposter/testdata/regenerate-certs.sh new file mode 100755 index 0000000000..6bcbd27b8e --- /dev/null +++ b/arbnode/dataposter/testdata/regenerate-certs.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -eu +cd "$(dirname "$0")" +for name in localhost client; do + openssl genrsa -out "$name.key" 2048 + csr="$(openssl req -new -key "$name.key" -config "$name.cnf" -batch)" + openssl x509 -req -signkey "$name.key" -out "$name.crt" -days 36500 -extensions req_ext -extfile "$name.cnf" <<< "$csr" +done From 247fb1f1a413113c5159e46d00254b85c85bbcb8 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Wed, 16 Oct 2024 13:20:28 +0530 Subject: [PATCH 385/489] Add log when switchover is complete --- arbnode/seq_coordinator.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arbnode/seq_coordinator.go b/arbnode/seq_coordinator.go index 4fbc31a108..930f42ffe4 100644 --- a/arbnode/seq_coordinator.go +++ b/arbnode/seq_coordinator.go @@ -653,6 +653,9 @@ func (c *SeqCoordinator) update(ctx context.Context) time.Duration { readUntil = min(readUntil, c.prevRedisMessageCount) client = c.prevRedisCoordinator.Client } + if c.prevRedisMessageCount != 0 && localMsgCount >= c.prevRedisMessageCount { + log.Info("coordinator caught up to prev redis coordinator", "msgcount", localMsgCount, "prevMsgCount", c.prevRedisMessageCount) + } var messages []arbostypes.MessageWithMetadata msgToRead := localMsgCount var msgReadErr error From 3c1daf15bc737c6f8e8ab2e7a0ad2f30d9fb5be6 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Wed, 16 Oct 2024 13:30:15 +0530 Subject: [PATCH 386/489] Changes based on PR comments --- util/check-build.sh | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/util/check-build.sh b/util/check-build.sh index 755cfda9ad..ddf540e45c 100755 --- a/util/check-build.sh +++ b/util/check-build.sh @@ -16,6 +16,8 @@ command_exists() { command -v "$1" >/dev/null 2>&1 } +EXIT_CODE=0 + # Detect operating system OS=$(uname -s) echo -e "${BLUE}Detected OS: $OS${NC}" @@ -26,16 +28,16 @@ if command_exists docker; then echo -e "${GREEN}Docker is installed.${NC}" else echo -e "${RED}Docker is not installed. $INSTALLATION_DOCS_URL${NC}" - exit 1 + EXIT_CODE=1 fi # Step 2: Check if Docker service is running if [[ "$OS" == "Linux" ]] && ! sudo service docker status >/dev/null; then echo -e "${YELLOW}Docker service is not running on Linux. Start it with: sudo service docker start${NC}" - exit 1 + EXIT_CODE=1 elif [[ "$OS" == "Darwin" ]] && ! docker info >/dev/null 2>&1; then echo -e "${YELLOW}Docker service is not running on macOS. Ensure Docker Desktop is started.${NC}" - exit 1 + EXIT_CODE=1 else echo -e "${GREEN}Docker service is running.${NC}" fi @@ -47,7 +49,7 @@ echo -e "${YELLOW}You are on the version tag: $VERSION_TAG${NC}" # Check if submodules are properly initialized and updated if git submodule status | grep -qE '^-|\+'; then echo -e "${YELLOW}Submodules are not properly initialized or updated. Run: git submodule update --init --recursive --force${NC}" - exit 1 + EXIT_CODE=1 else echo -e "${GREEN}All submodules are properly initialized and up to date.${NC}" fi @@ -57,7 +59,7 @@ if docker images | grep -q "nitro-node"; then echo -e "${GREEN}Nitro Docker image is built.${NC}" else echo -e "${RED}Nitro Docker image is not built. Build it using: docker build . --tag nitro-node${NC}" - exit 1 + EXIT_CODE=1 fi # Step 5: Check prerequisites for building binaries @@ -101,7 +103,7 @@ for pkg in "${prerequisites[@]}"; do [[ "$pkg" == "wasm2wat" ]] && pkg="wabt" [[ "$pkg" == "clang" ]] && pkg="llvm" echo -e "${RED}$pkg is not installed. Please install $pkg. $INSTALLATION_DOCS_URL${NC}" - exit 1 + EXIT_CODE=1 fi done @@ -110,7 +112,7 @@ if command_exists node && node -v | grep -q "v18"; then echo -e "${GREEN}Node.js version 18 is installed.${NC}" else echo -e "${RED}Node.js version 18 not installed. $INSTALLATION_DOCS_URL${NC}" - exit 1 + EXIT_CODE=1 fi # Step 7a: Check Rust version @@ -118,7 +120,7 @@ if command_exists rustc && rustc --version | grep -q "1.80.1"; then echo -e "${GREEN}Rust version 1.80.1 is installed.${NC}" else echo -e "${RED}Rust version 1.80.1 not installed. $INSTALLATION_DOCS_URL${NC}" - exit 1 + EXIT_CODE=1 fi # Step 7b: Check Rust nightly toolchain @@ -126,15 +128,16 @@ if rustup toolchain list | grep -q "nightly"; then echo -e "${GREEN}Rust nightly toolchain is installed.${NC}" else echo -e "${RED}Rust nightly toolchain is not installed. Install it using: rustup toolchain install nightly${NC}" - exit 1 + EXIT_CODE=1 fi # Step 8: Check Go version -if command_exists go && go version | grep -q "go1.23"; then - echo -e "${GREEN}Go version 1.23 is installed.${NC}" +go_version_needed=$(grep "^go " go.mod | awk '{print $2}') +if command_exists go && go version | grep -q "$go_version_needed"; then + echo -e "${GREEN}Go version $go_version_needed is installed.${NC}" else - echo -e "${RED}Go version 1.23 not installed. $INSTALLATION_DOCS_URL${NC}" - exit 1 + echo -e "${RED}Go version $go_version_needed not installed. $INSTALLATION_DOCS_URL${NC}" + EXIT_CODE=1 fi # Step 9: Check Foundry installation @@ -142,8 +145,10 @@ if command_exists foundryup; then echo -e "${GREEN}Foundry is installed.${NC}" else echo -e "${RED}Foundry is not installed. $INSTALLATION_DOCS_URL${NC}" - exit 1 + EXIT_CODE=1 fi echo -e "${BLUE}Verification complete.${NC}" -exit 0 + +exit $EXIT_CODE + From 79af5b4b04ad8eb41bad634473770b67a2468e67 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 24 Sep 2024 09:57:30 -0300 Subject: [PATCH 387/489] Adds comments on precompile methods without tests --- precompiles/ArbAddressTable.go | 1 + precompiles/ArbAggregator.go | 2 ++ precompiles/ArbDebug.go | 5 +++++ precompiles/ArbFunctionTable.go | 4 ++++ precompiles/ArbGasInfo.go | 25 +++++++++++++++++++++++++ precompiles/ArbInfo.go | 2 ++ precompiles/ArbOwner.go | 29 +++++++++++++++++++++++++++++ precompiles/ArbOwnerPublic.go | 4 ++++ precompiles/ArbRetryableTx.go | 8 ++++++++ precompiles/ArbStatistics.go | 1 + precompiles/ArbSys.go | 13 +++++++++++++ precompiles/ArbWasm.go | 20 ++++++++++++++++++++ precompiles/ArbWasmCache.go | 5 +++++ precompiles/ArbosActs.go | 2 ++ precompiles/ArbosTest.go | 1 + precompiles/context.go | 9 +++++++++ 16 files changed, 131 insertions(+) diff --git a/precompiles/ArbAddressTable.go b/precompiles/ArbAddressTable.go index 102fd55c3b..5ded229be6 100644 --- a/precompiles/ArbAddressTable.go +++ b/precompiles/ArbAddressTable.go @@ -13,6 +13,7 @@ type ArbAddressTable struct { Address addr // 0x66 } +// TODO: add test // AddressExists checks if an address exists in the table func (con ArbAddressTable) AddressExists(c ctx, evm mech, addr addr) (bool, error) { return c.State.AddressTable().AddressExists(addr) diff --git a/precompiles/ArbAggregator.go b/precompiles/ArbAggregator.go index b74e280fe8..438707d969 100644 --- a/precompiles/ArbAggregator.go +++ b/precompiles/ArbAggregator.go @@ -19,12 +19,14 @@ type ArbAggregator struct { var ErrNotOwner = errors.New("must be called by chain owner") +// TODO: add test // GetPreferredAggregator returns the preferred aggregator address. // Deprecated: Do not use this method. func (con ArbAggregator) GetPreferredAggregator(c ctx, evm mech, address addr) (prefAgg addr, isDefault bool, err error) { return l1pricing.BatchPosterAddress, true, err } +// TODO: add test // GetDefaultAggregator returns the default aggregator address. // Deprecated: Do not use this method. func (con ArbAggregator) GetDefaultAggregator(c ctx, evm mech) (addr, error) { diff --git a/precompiles/ArbDebug.go b/precompiles/ArbDebug.go index bf85d5e18f..3004e76447 100644 --- a/precompiles/ArbDebug.go +++ b/precompiles/ArbDebug.go @@ -24,6 +24,7 @@ type ArbDebug struct { UnusedError func() error } +// TODO: add test func (con ArbDebug) Events(c ctx, evm mech, paid huge, flag bool, value bytes32) (addr, huge, error) { // Emits 2 events that cover each case // Basic tests an index'd value & a normal value @@ -42,11 +43,13 @@ func (con ArbDebug) Events(c ctx, evm mech, paid huge, flag bool, value bytes32) return c.caller, paid, nil } +// TODO: add test func (con ArbDebug) EventsView(c ctx, evm mech) error { _, _, err := con.Events(c, evm, common.Big0, true, bytes32{}) return err } +// TODO: add test func (con ArbDebug) CustomRevert(c ctx, number uint64) error { return con.CustomError(number, "This spider family wards off bugs: /\\oo/\\ //\\(oo)//\\ /\\oo/\\", true) } @@ -56,11 +59,13 @@ func (con ArbDebug) BecomeChainOwner(c ctx, evm mech) error { return c.State.ChainOwners().Add(c.caller) } +// TODO: add test // Halts the chain by panicking in the STF func (con ArbDebug) Panic(c ctx, evm mech) error { panic("called ArbDebug's debug-only Panic method") } +// TODO: add test func (con ArbDebug) LegacyError(c ctx) error { return errors.New("example legacy error") } diff --git a/precompiles/ArbFunctionTable.go b/precompiles/ArbFunctionTable.go index a870995e0f..084d4d532b 100644 --- a/precompiles/ArbFunctionTable.go +++ b/precompiles/ArbFunctionTable.go @@ -8,6 +8,7 @@ import ( "math/big" ) +// TODO: add test // ArbFunctionTable precompile provided aggregator's the ability to manage function tables. // Aggregation works differently in Nitro, so these methods have been stubbed and their effects disabled. // They are kept for backwards compatibility. @@ -15,16 +16,19 @@ type ArbFunctionTable struct { Address addr // 0x68 } +// TODO: add test // Upload does nothing func (con ArbFunctionTable) Upload(c ctx, evm mech, buf []byte) error { return nil } +// TODO: add test // Size returns the empty table's size, which is 0 func (con ArbFunctionTable) Size(c ctx, evm mech, addr addr) (huge, error) { return big.NewInt(0), nil } +// TODO: add test // Get reverts since the table is empty func (con ArbFunctionTable) Get(c ctx, evm mech, addr addr, index huge) (huge, bool, huge, error) { return nil, false, nil, errors.New("table is empty") diff --git a/precompiles/ArbGasInfo.go b/precompiles/ArbGasInfo.go index b41dfda8a2..909ccb52b8 100644 --- a/precompiles/ArbGasInfo.go +++ b/precompiles/ArbGasInfo.go @@ -22,6 +22,7 @@ var storageArbGas = big.NewInt(int64(storage.StorageWriteCost)) const AssumedSimpleTxSize = 140 +// TODO: add test // GetPricesInWeiWithAggregator gets prices in wei when using the provided aggregator func (con ArbGasInfo) GetPricesInWeiWithAggregator( c ctx, @@ -65,6 +66,7 @@ func (con ArbGasInfo) GetPricesInWeiWithAggregator( return perL2Tx, weiForL1Calldata, weiForL2Storage, perArbGasBase, perArbGasCongestion, perArbGasTotal, nil } +// TODO: add test func (con ArbGasInfo) _preVersion4_GetPricesInWeiWithAggregator( c ctx, evm mech, @@ -97,11 +99,13 @@ func (con ArbGasInfo) _preVersion4_GetPricesInWeiWithAggregator( return perL2Tx, weiForL1Calldata, weiForL2Storage, perArbGasBase, perArbGasCongestion, perArbGasTotal, nil } +// TODO: add test // GetPricesInWei gets prices in wei when using the caller's preferred aggregator func (con ArbGasInfo) GetPricesInWei(c ctx, evm mech) (huge, huge, huge, huge, huge, huge, error) { return con.GetPricesInWeiWithAggregator(c, evm, addr{}) } +// TODO: add test // GetPricesInArbGasWithAggregator gets prices in ArbGas when using the provided aggregator func (con ArbGasInfo) GetPricesInArbGasWithAggregator(c ctx, evm mech, aggregator addr) (huge, huge, huge, error) { if c.State.ArbOSVersion() < 4 { @@ -131,6 +135,7 @@ func (con ArbGasInfo) GetPricesInArbGasWithAggregator(c ctx, evm mech, aggregato return gasPerL2Tx, gasForL1Calldata, storageArbGas, nil } +// TODO: add test func (con ArbGasInfo) _preVersion4_GetPricesInArbGasWithAggregator(c ctx, evm mech, aggregator addr) (huge, huge, huge, error) { l1GasPrice, err := c.State.L1PricingState().PricePerUnit() if err != nil { @@ -154,11 +159,13 @@ func (con ArbGasInfo) _preVersion4_GetPricesInArbGasWithAggregator(c ctx, evm me return perL2Tx, gasForL1Calldata, storageArbGas, nil } +// TODO: add test // GetPricesInArbGas gets prices in ArbGas when using the caller's preferred aggregator func (con ArbGasInfo) GetPricesInArbGas(c ctx, evm mech) (huge, huge, huge, error) { return con.GetPricesInArbGasWithAggregator(c, evm, addr{}) } +// TODO: add test // GetGasAccountingParams gets the rollup's speed limit, pool size, and tx gas limit func (con ArbGasInfo) GetGasAccountingParams(c ctx, evm mech) (huge, huge, huge, error) { l2pricing := c.State.L2PricingState() @@ -167,56 +174,67 @@ func (con ArbGasInfo) GetGasAccountingParams(c ctx, evm mech) (huge, huge, huge, return arbmath.UintToBig(speedLimit), arbmath.UintToBig(maxTxGasLimit), arbmath.UintToBig(maxTxGasLimit), err } +// TODO: add test // GetMinimumGasPrice gets the minimum gas price needed for a transaction to succeed func (con ArbGasInfo) GetMinimumGasPrice(c ctx, evm mech) (huge, error) { return c.State.L2PricingState().MinBaseFeeWei() } +// TODO: add test // GetL1BaseFeeEstimate gets the current estimate of the L1 basefee func (con ArbGasInfo) GetL1BaseFeeEstimate(c ctx, evm mech) (huge, error) { return c.State.L1PricingState().PricePerUnit() } +// TODO: add test // GetL1BaseFeeEstimateInertia gets how slowly ArbOS updates its estimate of the L1 basefee func (con ArbGasInfo) GetL1BaseFeeEstimateInertia(c ctx, evm mech) (uint64, error) { return c.State.L1PricingState().Inertia() } +// TODO: add test // GetL1RewardRate gets the L1 pricer reward rate func (con ArbGasInfo) GetL1RewardRate(c ctx, evm mech) (uint64, error) { return c.State.L1PricingState().PerUnitReward() } +// TODO: add test // GetL1RewardRecipient gets the L1 pricer reward recipient func (con ArbGasInfo) GetL1RewardRecipient(c ctx, evm mech) (common.Address, error) { return c.State.L1PricingState().PayRewardsTo() } +// TODO: add test // GetL1GasPriceEstimate gets the current estimate of the L1 basefee func (con ArbGasInfo) GetL1GasPriceEstimate(c ctx, evm mech) (huge, error) { return con.GetL1BaseFeeEstimate(c, evm) } +// TODO: add test // GetCurrentTxL1GasFees gets the fee paid to the aggregator for posting this tx func (con ArbGasInfo) GetCurrentTxL1GasFees(c ctx, evm mech) (huge, error) { return c.txProcessor.PosterFee, nil } +// TODO: add test // GetGasBacklog gets the backlogged amount of gas burnt in excess of the speed limit func (con ArbGasInfo) GetGasBacklog(c ctx, evm mech) (uint64, error) { return c.State.L2PricingState().GasBacklog() } +// TODO: add test // GetPricingInertia gets how slowly ArbOS updates the L2 basefee in response to backlogged gas func (con ArbGasInfo) GetPricingInertia(c ctx, evm mech) (uint64, error) { return c.State.L2PricingState().PricingInertia() } +// TODO: add test // GetGasBacklogTolerance gets the forgivable amount of backlogged gas ArbOS will ignore when raising the basefee func (con ArbGasInfo) GetGasBacklogTolerance(c ctx, evm mech) (uint64, error) { return c.State.L2PricingState().BacklogTolerance() } +// TODO: add test // GetL1PricingSurplus gets the surplus of funds for L1 batch posting payments (may be negative) func (con ArbGasInfo) GetL1PricingSurplus(c ctx, evm mech) (*big.Int, error) { if c.State.ArbOSVersion() < 10 { @@ -226,6 +244,7 @@ func (con ArbGasInfo) GetL1PricingSurplus(c ctx, evm mech) (*big.Int, error) { return ps.GetL1PricingSurplus() } +// TODO: add test func (con ArbGasInfo) _preversion10_GetL1PricingSurplus(c ctx, evm mech) (*big.Int, error) { ps := c.State.L1PricingState() fundsDueForRefunds, err := ps.BatchPosterTable().TotalFundsDue() @@ -241,6 +260,7 @@ func (con ArbGasInfo) _preversion10_GetL1PricingSurplus(c ctx, evm mech) (*big.I return arbmath.BigSub(haveFunds.ToBig(), needFunds), nil } +// TODO: add test // GetPerBatchGasCharge gets the base charge (in L1 gas) attributed to each data batch in the calldata pricer func (con ArbGasInfo) GetPerBatchGasCharge(c ctx, evm mech) (int64, error) { return c.State.L1PricingState().PerBatchGasCost() @@ -256,26 +276,31 @@ func (con ArbGasInfo) GetL1FeesAvailable(c ctx, evm mech) (huge, error) { return c.State.L1PricingState().L1FeesAvailable() } +// TODO: add test // GetL1PricingEquilibrationUnits gets the equilibration units parameter for L1 price adjustment algorithm func (con ArbGasInfo) GetL1PricingEquilibrationUnits(c ctx, evm mech) (*big.Int, error) { return c.State.L1PricingState().EquilibrationUnits() } +// TODO: add test // GetLastL1PricingUpdateTime gets the last time the L1 calldata pricer was updated func (con ArbGasInfo) GetLastL1PricingUpdateTime(c ctx, evm mech) (uint64, error) { return c.State.L1PricingState().LastUpdateTime() } +// TODO: add test // GetL1PricingFundsDueForRewards gets the amount of L1 calldata payments due for rewards (per the L1 reward rate) func (con ArbGasInfo) GetL1PricingFundsDueForRewards(c ctx, evm mech) (*big.Int, error) { return c.State.L1PricingState().FundsDueForRewards() } +// TODO: add test // GetL1PricingUnitsSinceUpdate gets the amount of L1 calldata posted since the last update func (con ArbGasInfo) GetL1PricingUnitsSinceUpdate(c ctx, evm mech) (uint64, error) { return c.State.L1PricingState().UnitsSinceUpdate() } +// TODO: add test // GetLastL1PricingSurplus gets the L1 pricing surplus as of the last update (may be negative) func (con ArbGasInfo) GetLastL1PricingSurplus(c ctx, evm mech) (*big.Int, error) { return c.State.L1PricingState().LastSurplus() diff --git a/precompiles/ArbInfo.go b/precompiles/ArbInfo.go index 9f8cf34532..c78dd84889 100644 --- a/precompiles/ArbInfo.go +++ b/precompiles/ArbInfo.go @@ -13,6 +13,7 @@ type ArbInfo struct { Address addr // 0x65 } +// TODO: add test // GetBalance retrieves an account's balance func (con ArbInfo) GetBalance(c ctx, evm mech, account addr) (huge, error) { if err := c.Burn(params.BalanceGasEIP1884); err != nil { @@ -21,6 +22,7 @@ func (con ArbInfo) GetBalance(c ctx, evm mech, account addr) (huge, error) { return evm.StateDB.GetBalance(account).ToBig(), nil } +// TODO: add test // GetCode retrieves a contract's deployed code func (con ArbInfo) GetCode(c ctx, evm mech, account addr) ([]byte, error) { if err := c.Burn(params.ColdSloadCostEIP2929); err != nil { diff --git a/precompiles/ArbOwner.go b/precompiles/ArbOwner.go index 8b87445e0e..8bb69fff0a 100644 --- a/precompiles/ArbOwner.go +++ b/precompiles/ArbOwner.go @@ -57,16 +57,19 @@ func (con ArbOwner) GetAllChainOwners(c ctx, evm mech) ([]common.Address, error) return c.State.ChainOwners().AllMembers(65536) } +// TODO: add test // SetL1BaseFeeEstimateInertia sets how slowly ArbOS updates its estimate of the L1 basefee func (con ArbOwner) SetL1BaseFeeEstimateInertia(c ctx, evm mech, inertia uint64) error { return c.State.L1PricingState().SetInertia(inertia) } +// TODO: add test // SetL2BaseFee sets the L2 gas price directly, bypassing the pool calculus func (con ArbOwner) SetL2BaseFee(c ctx, evm mech, priceInWei huge) error { return c.State.L2PricingState().SetBaseFeeWei(priceInWei) } +// TODO: add test // SetMinimumL2BaseFee sets the minimum base fee needed for a transaction to succeed func (con ArbOwner) SetMinimumL2BaseFee(c ctx, evm mech, priceInWei huge) error { if c.txProcessor.MsgIsNonMutating() && priceInWei.Sign() == 0 { @@ -75,26 +78,31 @@ func (con ArbOwner) SetMinimumL2BaseFee(c ctx, evm mech, priceInWei huge) error return c.State.L2PricingState().SetMinBaseFeeWei(priceInWei) } +// TODO: add test // SetSpeedLimit sets the computational speed limit for the chain func (con ArbOwner) SetSpeedLimit(c ctx, evm mech, limit uint64) error { return c.State.L2PricingState().SetSpeedLimitPerSecond(limit) } +// TODO: add test // SetMaxTxGasLimit sets the maximum size a tx (and block) can be func (con ArbOwner) SetMaxTxGasLimit(c ctx, evm mech, limit uint64) error { return c.State.L2PricingState().SetMaxPerBlockGasLimit(limit) } +// TODO: add test // SetL2GasPricingInertia sets the L2 gas pricing inertia func (con ArbOwner) SetL2GasPricingInertia(c ctx, evm mech, sec uint64) error { return c.State.L2PricingState().SetPricingInertia(sec) } +// TODO: add test // SetL2GasBacklogTolerance sets the L2 gas backlog tolerance func (con ArbOwner) SetL2GasBacklogTolerance(c ctx, evm mech, sec uint64) error { return c.State.L2PricingState().SetBacklogTolerance(sec) } +// TODO: add test // GetNetworkFeeAccount gets the network fee collector func (con ArbOwner) GetNetworkFeeAccount(c ctx, evm mech) (addr, error) { return c.State.NetworkFeeAccount() @@ -105,6 +113,7 @@ func (con ArbOwner) GetInfraFeeAccount(c ctx, evm mech) (addr, error) { return c.State.InfraFeeAccount() } +// TODO: add test // SetNetworkFeeAccount sets the network fee collector to the new network fee account func (con ArbOwner) SetNetworkFeeAccount(c ctx, evm mech, newNetworkFeeAccount addr) error { return c.State.SetNetworkFeeAccount(newNetworkFeeAccount) @@ -115,31 +124,38 @@ func (con ArbOwner) SetInfraFeeAccount(c ctx, evm mech, newNetworkFeeAccount add return c.State.SetInfraFeeAccount(newNetworkFeeAccount) } +// TODO: add test // ScheduleArbOSUpgrade to the requested version at the requested timestamp func (con ArbOwner) ScheduleArbOSUpgrade(c ctx, evm mech, newVersion uint64, timestamp uint64) error { return c.State.ScheduleArbOSUpgrade(newVersion, timestamp) } +// TODO: add test func (con ArbOwner) SetL1PricingEquilibrationUnits(c ctx, evm mech, equilibrationUnits huge) error { return c.State.L1PricingState().SetEquilibrationUnits(equilibrationUnits) } +// TODO: add test func (con ArbOwner) SetL1PricingInertia(c ctx, evm mech, inertia uint64) error { return c.State.L1PricingState().SetInertia(inertia) } +// TODO: add test func (con ArbOwner) SetL1PricingRewardRecipient(c ctx, evm mech, recipient addr) error { return c.State.L1PricingState().SetPayRewardsTo(recipient) } +// TODO: add test func (con ArbOwner) SetL1PricingRewardRate(c ctx, evm mech, weiPerUnit uint64) error { return c.State.L1PricingState().SetPerUnitReward(weiPerUnit) } +// TODO: add test func (con ArbOwner) SetL1PricePerUnit(c ctx, evm mech, pricePerUnit *big.Int) error { return c.State.L1PricingState().SetPricePerUnit(pricePerUnit) } +// TODO: add test func (con ArbOwner) SetPerBatchGasCharge(c ctx, evm mech, cost int64) error { return c.State.L1PricingState().SetPerBatchGasCost(cost) } @@ -148,6 +164,7 @@ func (con ArbOwner) SetAmortizedCostCapBips(c ctx, evm mech, cap uint64) error { return c.State.L1PricingState().SetAmortizedCostCapBips(cap) } +// TODO: add test func (con ArbOwner) SetBrotliCompressionLevel(c ctx, evm mech, level uint64) error { return c.State.SetBrotliCompressionLevel(level) } @@ -172,6 +189,7 @@ func (con ArbOwner) ReleaseL1PricerSurplusFunds(c ctx, evm mech, maxWeiToRelease return weiToTransfer, nil } +// TODO: add test // Sets the amount of ink 1 gas buys func (con ArbOwner) SetInkPrice(c ctx, evm mech, inkPrice uint32) error { params, err := c.State.Programs().Params() @@ -186,6 +204,7 @@ func (con ArbOwner) SetInkPrice(c ctx, evm mech, inkPrice uint32) error { return params.Save() } +// TODO: add test // Sets the maximum depth (in wasm words) a wasm stack may grow func (con ArbOwner) SetWasmMaxStackDepth(c ctx, evm mech, depth uint32) error { params, err := c.State.Programs().Params() @@ -196,6 +215,7 @@ func (con ArbOwner) SetWasmMaxStackDepth(c ctx, evm mech, depth uint32) error { return params.Save() } +// TODO: add test // Gets the number of free wasm pages a tx gets func (con ArbOwner) SetWasmFreePages(c ctx, evm mech, pages uint16) error { params, err := c.State.Programs().Params() @@ -206,6 +226,7 @@ func (con ArbOwner) SetWasmFreePages(c ctx, evm mech, pages uint16) error { return params.Save() } +// TODO: add test // Sets the base cost of each additional wasm page func (con ArbOwner) SetWasmPageGas(c ctx, evm mech, gas uint16) error { params, err := c.State.Programs().Params() @@ -216,6 +237,7 @@ func (con ArbOwner) SetWasmPageGas(c ctx, evm mech, gas uint16) error { return params.Save() } +// TODO: add test // Sets the initial number of pages a wasm may allocate func (con ArbOwner) SetWasmPageLimit(c ctx, evm mech, limit uint16) error { params, err := c.State.Programs().Params() @@ -226,6 +248,7 @@ func (con ArbOwner) SetWasmPageLimit(c ctx, evm mech, limit uint16) error { return params.Save() } +// TODO: add test // Sets the minimum costs to invoke a program func (con ArbOwner) SetWasmMinInitGas(c ctx, _ mech, gas, cached uint64) error { params, err := c.State.Programs().Params() @@ -237,6 +260,7 @@ func (con ArbOwner) SetWasmMinInitGas(c ctx, _ mech, gas, cached uint64) error { return params.Save() } +// TODO: add test // Sets the linear adjustment made to program init costs func (con ArbOwner) SetWasmInitCostScalar(c ctx, _ mech, percent uint64) error { params, err := c.State.Programs().Params() @@ -247,6 +271,7 @@ func (con ArbOwner) SetWasmInitCostScalar(c ctx, _ mech, percent uint64) error { return params.Save() } +// TODO: add test // Sets the number of days after which programs deactivate func (con ArbOwner) SetWasmExpiryDays(c ctx, _ mech, days uint16) error { params, err := c.State.Programs().Params() @@ -257,6 +282,7 @@ func (con ArbOwner) SetWasmExpiryDays(c ctx, _ mech, days uint16) error { return params.Save() } +// TODO: add test // Sets the age a program must be to perform a keepalive func (con ArbOwner) SetWasmKeepaliveDays(c ctx, _ mech, days uint16) error { params, err := c.State.Programs().Params() @@ -267,6 +293,7 @@ func (con ArbOwner) SetWasmKeepaliveDays(c ctx, _ mech, days uint16) error { return params.Save() } +// TODO: add test // Sets the number of extra programs ArbOS caches during a given block func (con ArbOwner) SetWasmBlockCacheSize(c ctx, _ mech, count uint16) error { params, err := c.State.Programs().Params() @@ -277,11 +304,13 @@ func (con ArbOwner) SetWasmBlockCacheSize(c ctx, _ mech, count uint16) error { return params.Save() } +// TODO: add test // Adds account as a wasm cache manager func (con ArbOwner) AddWasmCacheManager(c ctx, _ mech, manager addr) error { return c.State.Programs().CacheManagers().Add(manager) } +// TODO: add test // Removes account from the list of wasm cache managers func (con ArbOwner) RemoveWasmCacheManager(c ctx, _ mech, manager addr) error { managers := c.State.Programs().CacheManagers() diff --git a/precompiles/ArbOwnerPublic.go b/precompiles/ArbOwnerPublic.go index 451e18e1cc..2356c855ee 100644 --- a/precompiles/ArbOwnerPublic.go +++ b/precompiles/ArbOwnerPublic.go @@ -21,6 +21,7 @@ func (con ArbOwnerPublic) GetAllChainOwners(c ctx, evm mech) ([]common.Address, return c.State.ChainOwners().AllMembers(65536) } +// TODO: add test // RectifyChainOwner checks if the account is a chain owner func (con ArbOwnerPublic) RectifyChainOwner(c ctx, evm mech, addr addr) error { err := c.State.ChainOwners().RectifyMapping(addr) @@ -35,6 +36,7 @@ func (con ArbOwnerPublic) IsChainOwner(c ctx, evm mech, addr addr) (bool, error) return c.State.ChainOwners().IsMember(addr) } +// TODO: add test // GetNetworkFeeAccount gets the network fee collector func (con ArbOwnerPublic) GetNetworkFeeAccount(c ctx, evm mech) (addr, error) { return c.State.NetworkFeeAccount() @@ -48,11 +50,13 @@ func (con ArbOwnerPublic) GetInfraFeeAccount(c ctx, evm mech) (addr, error) { return c.State.InfraFeeAccount() } +// TODO: add test // GetBrotliCompressionLevel gets the current brotli compression level used for fast compression func (con ArbOwnerPublic) GetBrotliCompressionLevel(c ctx, evm mech) (uint64, error) { return c.State.BrotliCompressionLevel() } +// TODO: add test // GetScheduledUpgrade gets the next scheduled ArbOS version upgrade and its activation timestamp. // Returns (0, 0, nil) if no ArbOS upgrade is scheduled. func (con ArbOwnerPublic) GetScheduledUpgrade(c ctx, evm mech) (uint64, uint64, error) { diff --git a/precompiles/ArbRetryableTx.go b/precompiles/ArbRetryableTx.go index 93e8023603..abfdc94807 100644 --- a/precompiles/ArbRetryableTx.go +++ b/precompiles/ArbRetryableTx.go @@ -45,6 +45,7 @@ func (con ArbRetryableTx) oldNotFoundError(c ctx) error { return errors.New("ticketId not found") } +// TODO: add test // Redeem schedules an attempt to redeem the retryable, donating all of the call's gas to the redeem attempt func (con ArbRetryableTx) Redeem(c ctx, evm mech, ticketId bytes32) (bytes32, error) { if c.txProcessor.CurrentRetryable != nil && ticketId == *c.txProcessor.CurrentRetryable { @@ -130,11 +131,13 @@ func (con ArbRetryableTx) Redeem(c ctx, evm mech, ticketId bytes32) (bytes32, er return retryTxHash, c.State.L2PricingState().AddToGasPool(arbmath.SaturatingCast[int64](gasToDonate)) } +// TODO: add test // GetLifetime gets the default lifetime period a retryable has at creation func (con ArbRetryableTx) GetLifetime(c ctx, evm mech) (huge, error) { return big.NewInt(retryables.RetryableLifetimeSeconds), nil } +// TODO: add test // GetTimeout gets the timestamp for when ticket will expire func (con ArbRetryableTx) GetTimeout(c ctx, evm mech, ticketId bytes32) (huge, error) { retryableState := c.State.RetryableState() @@ -152,6 +155,7 @@ func (con ArbRetryableTx) GetTimeout(c ctx, evm mech, ticketId bytes32) (huge, e return new(big.Int).SetUint64(timeout), nil } +// TODO: add test // Keepalive adds one lifetime period to the ticket's expiry func (con ArbRetryableTx) Keepalive(c ctx, evm mech, ticketId bytes32) (huge, error) { @@ -181,6 +185,7 @@ func (con ArbRetryableTx) Keepalive(c ctx, evm mech, ticketId bytes32) (huge, er return bigNewTimeout, err } +// TODO: add test // GetBeneficiary gets the beneficiary of the ticket func (con ArbRetryableTx) GetBeneficiary(c ctx, evm mech, ticketId bytes32) (addr, error) { retryableState := c.State.RetryableState() @@ -194,6 +199,7 @@ func (con ArbRetryableTx) GetBeneficiary(c ctx, evm mech, ticketId bytes32) (add return retryable.Beneficiary() } +// TODO: add test // Cancel the ticket and refund its callvalue to its beneficiary func (con ArbRetryableTx) Cancel(c ctx, evm mech, ticketId bytes32) error { if c.txProcessor.CurrentRetryable != nil && ticketId == *c.txProcessor.CurrentRetryable { @@ -223,6 +229,7 @@ func (con ArbRetryableTx) Cancel(c ctx, evm mech, ticketId bytes32) error { return con.Canceled(c, evm, ticketId) } +// TODO: add test func (con ArbRetryableTx) GetCurrentRedeemer(c ctx, evm mech) (common.Address, error) { if c.txProcessor.CurrentRefundTo != nil { return *c.txProcessor.CurrentRefundTo, nil @@ -230,6 +237,7 @@ func (con ArbRetryableTx) GetCurrentRedeemer(c ctx, evm mech) (common.Address, e return common.Address{}, nil } +// TODO: add test func (con ArbRetryableTx) SubmitRetryable( c ctx, evm mech, requestId bytes32, l1BaseFee, deposit, callvalue, gasFeeCap huge, gasLimit uint64, maxSubmissionFee huge, diff --git a/precompiles/ArbStatistics.go b/precompiles/ArbStatistics.go index 83f867d09d..74d1c2dd15 100644 --- a/precompiles/ArbStatistics.go +++ b/precompiles/ArbStatistics.go @@ -14,6 +14,7 @@ type ArbStatistics struct { Address addr // 0x6e } +// TODO: add test // GetStats returns the current block number and some statistics about the rollup's pre-Nitro state func (con ArbStatistics) GetStats(c ctx, evm mech) (huge, huge, huge, huge, huge, huge, error) { blockNum := evm.Context.BlockNumber diff --git a/precompiles/ArbSys.go b/precompiles/ArbSys.go index d55067a09c..295085a3c5 100644 --- a/precompiles/ArbSys.go +++ b/precompiles/ArbSys.go @@ -28,11 +28,13 @@ type ArbSys struct { L2ToL1TransactionGasCost func(addr, addr, huge, huge, huge, huge, huge, huge, huge, []byte) (uint64, error) } +// TODO: add test // ArbBlockNumber gets the current L2 block number func (con *ArbSys) ArbBlockNumber(c ctx, evm mech) (huge, error) { return evm.Context.BlockNumber, nil } +// TODO: add test // ArbBlockHash gets the L2 block hash, if sufficiently recent func (con *ArbSys) ArbBlockHash(c ctx, evm mech, arbBlockNumber *big.Int) (bytes32, error) { if !arbBlockNumber.IsUint64() { @@ -54,32 +56,38 @@ func (con *ArbSys) ArbBlockHash(c ctx, evm mech, arbBlockNumber *big.Int) (bytes return evm.Context.GetHash(requestedBlockNum), nil } +// TODO: add test // ArbChainID gets the rollup's unique chain identifier func (con *ArbSys) ArbChainID(c ctx, evm mech) (huge, error) { return evm.ChainConfig().ChainID, nil } +// TODO: add test // ArbOSVersion gets the current ArbOS version func (con *ArbSys) ArbOSVersion(c ctx, evm mech) (huge, error) { version := new(big.Int).SetUint64(55 + c.State.ArbOSVersion()) // Nitro starts at version 56 return version, nil } +// TODO: add test // GetStorageGasAvailable returns 0 since Nitro has no concept of storage gas func (con *ArbSys) GetStorageGasAvailable(c ctx, evm mech) (huge, error) { return big.NewInt(0), nil } +// TODO: add test // IsTopLevelCall checks if the call is top-level (deprecated) func (con *ArbSys) IsTopLevelCall(c ctx, evm mech) (bool, error) { return evm.Depth() <= 2, nil } +// TODO: add test // MapL1SenderContractAddressToL2Alias gets the contract's L2 alias func (con *ArbSys) MapL1SenderContractAddressToL2Alias(c ctx, sender addr, dest addr) (addr, error) { return util.RemapL1Address(sender), nil } +// TODO: add test // WasMyCallersAddressAliased checks if the caller's caller was aliased func (con *ArbSys) WasMyCallersAddressAliased(c ctx, evm mech) (bool, error) { topLevel := con.isTopLevel(c, evm) @@ -90,6 +98,7 @@ func (con *ArbSys) WasMyCallersAddressAliased(c ctx, evm mech) (bool, error) { return aliased, nil } +// TODO: add test // MyCallersAddressWithoutAliasing gets the caller's caller without any potential aliasing func (con *ArbSys) MyCallersAddressWithoutAliasing(c ctx, evm mech) (addr, error) { @@ -106,6 +115,7 @@ func (con *ArbSys) MyCallersAddressWithoutAliasing(c ctx, evm mech) (addr, error return address, err } +// TODO: add test // SendTxToL1 sends a transaction to L1, adding it to the outbox func (con *ArbSys) SendTxToL1(c ctx, evm mech, value huge, destination addr, calldataForL1 []byte) (huge, error) { l1BlockNum, err := c.txProcessor.L1BlockNumber(vm.BlockContext{}) @@ -186,6 +196,7 @@ func (con *ArbSys) SendTxToL1(c ctx, evm mech, value huge, destination addr, cal return sendHash.Big(), err } +// TODO: add test // SendMerkleTreeState gets the root, size, and partials of the outbox Merkle tree state (caller must be the 0 address) func (con ArbSys) SendMerkleTreeState(c ctx, evm mech) (huge, bytes32, []bytes32, error) { if c.caller != (addr{}) { @@ -202,11 +213,13 @@ func (con ArbSys) SendMerkleTreeState(c ctx, evm mech) (huge, bytes32, []bytes32 return new(big.Int).SetUint64(size), rootHash, partials, nil } +// TODO: add test // WithdrawEth send paid eth to the destination on L1 func (con ArbSys) WithdrawEth(c ctx, evm mech, value huge, destination addr) (huge, error) { return con.SendTxToL1(c, evm, value, destination, []byte{}) } +// TODO: add test func (con ArbSys) isTopLevel(c ctx, evm mech) bool { depth := evm.Depth() return depth < 2 || evm.Origin == c.txProcessor.Contracts[depth-2].Caller() diff --git a/precompiles/ArbWasm.go b/precompiles/ArbWasm.go index bc24c8a6e8..f8ece25600 100644 --- a/precompiles/ArbWasm.go +++ b/precompiles/ArbWasm.go @@ -29,6 +29,7 @@ type ArbWasm struct { ProgramInsufficientValueError func(have, want huge) error } +// TODO: add test // Compile a wasm program with the latest instrumentation func (con ArbWasm) ActivateProgram(c ctx, evm mech, value huge, program addr) (uint16, huge, error) { debug := evm.ChainConfig().DebugMode() @@ -53,6 +54,7 @@ func (con ArbWasm) ActivateProgram(c ctx, evm mech, value huge, program addr) (u return version, dataFee, con.ProgramActivated(c, evm, codeHash, moduleHash, program, dataFee, version) } +// TODO: add test // Extends a program's expiration date (reverts if too soon) func (con ArbWasm) CodehashKeepalive(c ctx, evm mech, value huge, codehash bytes32) error { params, err := c.State.Programs().Params() @@ -89,48 +91,56 @@ func (con ArbWasm) payActivationDataFee(c ctx, evm mech, value, dataFee huge) er return util.TransferBalance(&con.Address, &c.caller, repay, evm, scenario, "reimburse") } +// TODO: add test // Gets the latest stylus version func (con ArbWasm) StylusVersion(c ctx, evm mech) (uint16, error) { params, err := c.State.Programs().Params() return params.Version, err } +// TODO: add test // Gets the amount of ink 1 gas buys func (con ArbWasm) InkPrice(c ctx, _ mech) (uint32, error) { params, err := c.State.Programs().Params() return params.InkPrice.ToUint32(), err } +// TODO: add test // Gets the wasm stack size limit func (con ArbWasm) MaxStackDepth(c ctx, _ mech) (uint32, error) { params, err := c.State.Programs().Params() return params.MaxStackDepth, err } +// TODO: add test // Gets the number of free wasm pages a tx gets func (con ArbWasm) FreePages(c ctx, _ mech) (uint16, error) { params, err := c.State.Programs().Params() return params.FreePages, err } +// TODO: add test // Gets the base cost of each additional wasm page func (con ArbWasm) PageGas(c ctx, _ mech) (uint16, error) { params, err := c.State.Programs().Params() return params.PageGas, err } +// TODO: add test // Gets the ramp that drives exponential memory costs func (con ArbWasm) PageRamp(c ctx, _ mech) (uint64, error) { params, err := c.State.Programs().Params() return params.PageRamp, err } +// TODO: add test // Gets the maximum initial number of pages a wasm may allocate func (con ArbWasm) PageLimit(c ctx, _ mech) (uint16, error) { params, err := c.State.Programs().Params() return params.PageLimit, err } +// TODO: add test // Gets the minimum costs to invoke a program func (con ArbWasm) MinInitGas(c ctx, _ mech) (uint64, uint64, error) { params, err := c.State.Programs().Params() @@ -142,30 +152,35 @@ func (con ArbWasm) MinInitGas(c ctx, _ mech) (uint64, uint64, error) { return init, cached, err } +// TODO: add test // Gets the linear adjustment made to program init costs func (con ArbWasm) InitCostScalar(c ctx, _ mech) (uint64, error) { params, err := c.State.Programs().Params() return uint64(params.InitCostScalar) * programs.CostScalarPercent, err } +// TODO: add test // Gets the number of days after which programs deactivate func (con ArbWasm) ExpiryDays(c ctx, _ mech) (uint16, error) { params, err := c.State.Programs().Params() return params.ExpiryDays, err } +// TODO: add test // Gets the age a program must be to perform a keepalive func (con ArbWasm) KeepaliveDays(c ctx, _ mech) (uint16, error) { params, err := c.State.Programs().Params() return params.KeepaliveDays, err } +// TODO: add test // Gets the number of extra programs ArbOS caches during a given block. func (con ArbWasm) BlockCacheSize(c ctx, _ mech) (uint16, error) { params, err := c.State.Programs().Params() return params.BlockCacheSize, err } +// TODO: add test // Gets the stylus version that program with codehash was most recently compiled with func (con ArbWasm) CodehashVersion(c ctx, evm mech, codehash bytes32) (uint16, error) { params, err := c.State.Programs().Params() @@ -175,6 +190,7 @@ func (con ArbWasm) CodehashVersion(c ctx, evm mech, codehash bytes32) (uint16, e return c.State.Programs().CodehashVersion(codehash, evm.Context.Time, params) } +// TODO: add test // Gets a program's asm size in bytes func (con ArbWasm) CodehashAsmSize(c ctx, evm mech, codehash bytes32) (uint32, error) { params, err := c.State.Programs().Params() @@ -184,6 +200,7 @@ func (con ArbWasm) CodehashAsmSize(c ctx, evm mech, codehash bytes32) (uint32, e return c.State.Programs().ProgramAsmSize(codehash, evm.Context.Time, params) } +// TODO: add test // Gets the stylus version that program at addr was most recently compiled with func (con ArbWasm) ProgramVersion(c ctx, evm mech, program addr) (uint16, error) { codehash, err := c.GetCodeHash(program) @@ -193,6 +210,7 @@ func (con ArbWasm) ProgramVersion(c ctx, evm mech, program addr) (uint16, error) return con.CodehashVersion(c, evm, codehash) } +// TODO: add test // Gets the cost to invoke the program func (con ArbWasm) ProgramInitGas(c ctx, evm mech, program addr) (uint64, uint64, error) { codehash, params, err := con.getCodeHash(c, program) @@ -202,6 +220,7 @@ func (con ArbWasm) ProgramInitGas(c ctx, evm mech, program addr) (uint64, uint64 return c.State.Programs().ProgramInitGas(codehash, evm.Context.Time, params) } +// TODO: add test // Gets the footprint of program at addr func (con ArbWasm) ProgramMemoryFootprint(c ctx, evm mech, program addr) (uint16, error) { codehash, params, err := con.getCodeHash(c, program) @@ -211,6 +230,7 @@ func (con ArbWasm) ProgramMemoryFootprint(c ctx, evm mech, program addr) (uint16 return c.State.Programs().ProgramMemoryFootprint(codehash, evm.Context.Time, params) } +// TODO: add test // Gets returns the amount of time remaining until the program expires func (con ArbWasm) ProgramTimeLeft(c ctx, evm mech, program addr) (uint64, error) { codehash, params, err := con.getCodeHash(c, program) diff --git a/precompiles/ArbWasmCache.go b/precompiles/ArbWasmCache.go index 3cada9dd70..053cfb3792 100644 --- a/precompiles/ArbWasmCache.go +++ b/precompiles/ArbWasmCache.go @@ -12,21 +12,25 @@ type ArbWasmCache struct { UpdateProgramCacheGasCost func(addr, bytes32, bool) (uint64, error) } +// TODO: add test // See if the user is a cache manager owner. func (con ArbWasmCache) IsCacheManager(c ctx, _ mech, addr addr) (bool, error) { return c.State.Programs().CacheManagers().IsMember(addr) } +// TODO: add test // Retrieve all authorized address managers. func (con ArbWasmCache) AllCacheManagers(c ctx, _ mech) ([]addr, error) { return c.State.Programs().CacheManagers().AllMembers(65536) } +// TODO: add test // Deprecated: replaced with CacheProgram. func (con ArbWasmCache) CacheCodehash(c ctx, evm mech, codehash hash) error { return con.setProgramCached(c, evm, common.Address{}, codehash, true) } +// TODO: add test // Caches all programs with a codehash equal to the given address. Caller must be a cache manager or chain owner. func (con ArbWasmCache) CacheProgram(c ctx, evm mech, address addr) error { codehash, err := c.GetCodeHash(address) @@ -36,6 +40,7 @@ func (con ArbWasmCache) CacheProgram(c ctx, evm mech, address addr) error { return con.setProgramCached(c, evm, address, codehash, true) } +// TODO: add test // Evicts all programs with the given codehash. Caller must be a cache manager or chain owner. func (con ArbWasmCache) EvictCodehash(c ctx, evm mech, codehash hash) error { return con.setProgramCached(c, evm, common.Address{}, codehash, false) diff --git a/precompiles/ArbosActs.go b/precompiles/ArbosActs.go index e18aa43efb..475faf0e85 100644 --- a/precompiles/ArbosActs.go +++ b/precompiles/ArbosActs.go @@ -11,10 +11,12 @@ type ArbosActs struct { CallerNotArbOSError func() error } +// TODO: add test func (con ArbosActs) StartBlock(c ctx, evm mech, l1BaseFee huge, l1BlockNumber, l2BlockNumber, timeLastBlock uint64) error { return con.CallerNotArbOSError() } +// TODO: add test func (con ArbosActs) BatchPostingReport(c ctx, evm mech, batchTimestamp huge, batchPosterAddress addr, batchNumber uint64, batchDataGas uint64, l1BaseFeeWei huge) error { return con.CallerNotArbOSError() } diff --git a/precompiles/ArbosTest.go b/precompiles/ArbosTest.go index 6e988f533a..d34bc740c7 100644 --- a/precompiles/ArbosTest.go +++ b/precompiles/ArbosTest.go @@ -12,6 +12,7 @@ type ArbosTest struct { Address addr // 0x69 } +// TODO: add test // BurnArbGas unproductively burns the amount of L2 ArbGas func (con ArbosTest) BurnArbGas(c ctx, gasAmount huge) error { if !gasAmount.IsUint64() { diff --git a/precompiles/context.go b/precompiles/context.go index 670ffa7443..25aceba8ec 100644 --- a/precompiles/context.go +++ b/precompiles/context.go @@ -35,6 +35,7 @@ type Context struct { readOnly bool } +// TODO: add test func (c *Context) Burn(amount uint64) error { if c.gasLeft < amount { return c.BurnOut() @@ -43,36 +44,44 @@ func (c *Context) Burn(amount uint64) error { return nil } +// TODO: add test //nolint:unused func (c *Context) Burned() uint64 { return c.gasSupplied - c.gasLeft } +// TODO: add test func (c *Context) BurnOut() error { c.gasLeft = 0 return vm.ErrOutOfGas } +// TODO: add test func (c *Context) GasLeft() *uint64 { return &c.gasLeft } +// TODO: add test func (c *Context) Restrict(err error) { log.Crit("A metered burner was used for access-controlled work", "error", err) } +// TODO: add test func (c *Context) HandleError(err error) error { return err } +// TODO: add test func (c *Context) ReadOnly() bool { return c.readOnly } +// TODO: add test func (c *Context) TracingInfo() *util.TracingInfo { return c.tracingInfo } +// TODO: add test func (c *Context) GetCodeHash(address common.Address) (common.Hash, error) { return c.State.BackingStorage().GetCodeHash(address) } From bcb464d77e1d280be5fa0c8c9a7e7ebe55214c23 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 24 Sep 2024 15:19:08 -0300 Subject: [PATCH 388/489] Updates comments on precompile methods without tests --- precompiles/ArbDebug.go | 3 --- precompiles/ArbFunctionTable.go | 1 - precompiles/ArbGasInfo.go | 4 ---- precompiles/ArbOwner.go | 29 ----------------------------- precompiles/ArbOwnerPublic.go | 2 -- precompiles/ArbSys.go | 9 --------- precompiles/ArbosTest.go | 1 - precompiles/context.go | 9 --------- 8 files changed, 58 deletions(-) diff --git a/precompiles/ArbDebug.go b/precompiles/ArbDebug.go index 3004e76447..f20532f6e6 100644 --- a/precompiles/ArbDebug.go +++ b/precompiles/ArbDebug.go @@ -24,7 +24,6 @@ type ArbDebug struct { UnusedError func() error } -// TODO: add test func (con ArbDebug) Events(c ctx, evm mech, paid huge, flag bool, value bytes32) (addr, huge, error) { // Emits 2 events that cover each case // Basic tests an index'd value & a normal value @@ -43,13 +42,11 @@ func (con ArbDebug) Events(c ctx, evm mech, paid huge, flag bool, value bytes32) return c.caller, paid, nil } -// TODO: add test func (con ArbDebug) EventsView(c ctx, evm mech) error { _, _, err := con.Events(c, evm, common.Big0, true, bytes32{}) return err } -// TODO: add test func (con ArbDebug) CustomRevert(c ctx, number uint64) error { return con.CustomError(number, "This spider family wards off bugs: /\\oo/\\ //\\(oo)//\\ /\\oo/\\", true) } diff --git a/precompiles/ArbFunctionTable.go b/precompiles/ArbFunctionTable.go index 084d4d532b..c3a545136a 100644 --- a/precompiles/ArbFunctionTable.go +++ b/precompiles/ArbFunctionTable.go @@ -8,7 +8,6 @@ import ( "math/big" ) -// TODO: add test // ArbFunctionTable precompile provided aggregator's the ability to manage function tables. // Aggregation works differently in Nitro, so these methods have been stubbed and their effects disabled. // They are kept for backwards compatibility. diff --git a/precompiles/ArbGasInfo.go b/precompiles/ArbGasInfo.go index 909ccb52b8..e3e0029d9e 100644 --- a/precompiles/ArbGasInfo.go +++ b/precompiles/ArbGasInfo.go @@ -99,7 +99,6 @@ func (con ArbGasInfo) _preVersion4_GetPricesInWeiWithAggregator( return perL2Tx, weiForL1Calldata, weiForL2Storage, perArbGasBase, perArbGasCongestion, perArbGasTotal, nil } -// TODO: add test // GetPricesInWei gets prices in wei when using the caller's preferred aggregator func (con ArbGasInfo) GetPricesInWei(c ctx, evm mech) (huge, huge, huge, huge, huge, huge, error) { return con.GetPricesInWeiWithAggregator(c, evm, addr{}) @@ -174,13 +173,11 @@ func (con ArbGasInfo) GetGasAccountingParams(c ctx, evm mech) (huge, huge, huge, return arbmath.UintToBig(speedLimit), arbmath.UintToBig(maxTxGasLimit), arbmath.UintToBig(maxTxGasLimit), err } -// TODO: add test // GetMinimumGasPrice gets the minimum gas price needed for a transaction to succeed func (con ArbGasInfo) GetMinimumGasPrice(c ctx, evm mech) (huge, error) { return c.State.L2PricingState().MinBaseFeeWei() } -// TODO: add test // GetL1BaseFeeEstimate gets the current estimate of the L1 basefee func (con ArbGasInfo) GetL1BaseFeeEstimate(c ctx, evm mech) (huge, error) { return c.State.L1PricingState().PricePerUnit() @@ -234,7 +231,6 @@ func (con ArbGasInfo) GetGasBacklogTolerance(c ctx, evm mech) (uint64, error) { return c.State.L2PricingState().BacklogTolerance() } -// TODO: add test // GetL1PricingSurplus gets the surplus of funds for L1 batch posting payments (may be negative) func (con ArbGasInfo) GetL1PricingSurplus(c ctx, evm mech) (*big.Int, error) { if c.State.ArbOSVersion() < 10 { diff --git a/precompiles/ArbOwner.go b/precompiles/ArbOwner.go index 8bb69fff0a..8b87445e0e 100644 --- a/precompiles/ArbOwner.go +++ b/precompiles/ArbOwner.go @@ -57,19 +57,16 @@ func (con ArbOwner) GetAllChainOwners(c ctx, evm mech) ([]common.Address, error) return c.State.ChainOwners().AllMembers(65536) } -// TODO: add test // SetL1BaseFeeEstimateInertia sets how slowly ArbOS updates its estimate of the L1 basefee func (con ArbOwner) SetL1BaseFeeEstimateInertia(c ctx, evm mech, inertia uint64) error { return c.State.L1PricingState().SetInertia(inertia) } -// TODO: add test // SetL2BaseFee sets the L2 gas price directly, bypassing the pool calculus func (con ArbOwner) SetL2BaseFee(c ctx, evm mech, priceInWei huge) error { return c.State.L2PricingState().SetBaseFeeWei(priceInWei) } -// TODO: add test // SetMinimumL2BaseFee sets the minimum base fee needed for a transaction to succeed func (con ArbOwner) SetMinimumL2BaseFee(c ctx, evm mech, priceInWei huge) error { if c.txProcessor.MsgIsNonMutating() && priceInWei.Sign() == 0 { @@ -78,31 +75,26 @@ func (con ArbOwner) SetMinimumL2BaseFee(c ctx, evm mech, priceInWei huge) error return c.State.L2PricingState().SetMinBaseFeeWei(priceInWei) } -// TODO: add test // SetSpeedLimit sets the computational speed limit for the chain func (con ArbOwner) SetSpeedLimit(c ctx, evm mech, limit uint64) error { return c.State.L2PricingState().SetSpeedLimitPerSecond(limit) } -// TODO: add test // SetMaxTxGasLimit sets the maximum size a tx (and block) can be func (con ArbOwner) SetMaxTxGasLimit(c ctx, evm mech, limit uint64) error { return c.State.L2PricingState().SetMaxPerBlockGasLimit(limit) } -// TODO: add test // SetL2GasPricingInertia sets the L2 gas pricing inertia func (con ArbOwner) SetL2GasPricingInertia(c ctx, evm mech, sec uint64) error { return c.State.L2PricingState().SetPricingInertia(sec) } -// TODO: add test // SetL2GasBacklogTolerance sets the L2 gas backlog tolerance func (con ArbOwner) SetL2GasBacklogTolerance(c ctx, evm mech, sec uint64) error { return c.State.L2PricingState().SetBacklogTolerance(sec) } -// TODO: add test // GetNetworkFeeAccount gets the network fee collector func (con ArbOwner) GetNetworkFeeAccount(c ctx, evm mech) (addr, error) { return c.State.NetworkFeeAccount() @@ -113,7 +105,6 @@ func (con ArbOwner) GetInfraFeeAccount(c ctx, evm mech) (addr, error) { return c.State.InfraFeeAccount() } -// TODO: add test // SetNetworkFeeAccount sets the network fee collector to the new network fee account func (con ArbOwner) SetNetworkFeeAccount(c ctx, evm mech, newNetworkFeeAccount addr) error { return c.State.SetNetworkFeeAccount(newNetworkFeeAccount) @@ -124,38 +115,31 @@ func (con ArbOwner) SetInfraFeeAccount(c ctx, evm mech, newNetworkFeeAccount add return c.State.SetInfraFeeAccount(newNetworkFeeAccount) } -// TODO: add test // ScheduleArbOSUpgrade to the requested version at the requested timestamp func (con ArbOwner) ScheduleArbOSUpgrade(c ctx, evm mech, newVersion uint64, timestamp uint64) error { return c.State.ScheduleArbOSUpgrade(newVersion, timestamp) } -// TODO: add test func (con ArbOwner) SetL1PricingEquilibrationUnits(c ctx, evm mech, equilibrationUnits huge) error { return c.State.L1PricingState().SetEquilibrationUnits(equilibrationUnits) } -// TODO: add test func (con ArbOwner) SetL1PricingInertia(c ctx, evm mech, inertia uint64) error { return c.State.L1PricingState().SetInertia(inertia) } -// TODO: add test func (con ArbOwner) SetL1PricingRewardRecipient(c ctx, evm mech, recipient addr) error { return c.State.L1PricingState().SetPayRewardsTo(recipient) } -// TODO: add test func (con ArbOwner) SetL1PricingRewardRate(c ctx, evm mech, weiPerUnit uint64) error { return c.State.L1PricingState().SetPerUnitReward(weiPerUnit) } -// TODO: add test func (con ArbOwner) SetL1PricePerUnit(c ctx, evm mech, pricePerUnit *big.Int) error { return c.State.L1PricingState().SetPricePerUnit(pricePerUnit) } -// TODO: add test func (con ArbOwner) SetPerBatchGasCharge(c ctx, evm mech, cost int64) error { return c.State.L1PricingState().SetPerBatchGasCost(cost) } @@ -164,7 +148,6 @@ func (con ArbOwner) SetAmortizedCostCapBips(c ctx, evm mech, cap uint64) error { return c.State.L1PricingState().SetAmortizedCostCapBips(cap) } -// TODO: add test func (con ArbOwner) SetBrotliCompressionLevel(c ctx, evm mech, level uint64) error { return c.State.SetBrotliCompressionLevel(level) } @@ -189,7 +172,6 @@ func (con ArbOwner) ReleaseL1PricerSurplusFunds(c ctx, evm mech, maxWeiToRelease return weiToTransfer, nil } -// TODO: add test // Sets the amount of ink 1 gas buys func (con ArbOwner) SetInkPrice(c ctx, evm mech, inkPrice uint32) error { params, err := c.State.Programs().Params() @@ -204,7 +186,6 @@ func (con ArbOwner) SetInkPrice(c ctx, evm mech, inkPrice uint32) error { return params.Save() } -// TODO: add test // Sets the maximum depth (in wasm words) a wasm stack may grow func (con ArbOwner) SetWasmMaxStackDepth(c ctx, evm mech, depth uint32) error { params, err := c.State.Programs().Params() @@ -215,7 +196,6 @@ func (con ArbOwner) SetWasmMaxStackDepth(c ctx, evm mech, depth uint32) error { return params.Save() } -// TODO: add test // Gets the number of free wasm pages a tx gets func (con ArbOwner) SetWasmFreePages(c ctx, evm mech, pages uint16) error { params, err := c.State.Programs().Params() @@ -226,7 +206,6 @@ func (con ArbOwner) SetWasmFreePages(c ctx, evm mech, pages uint16) error { return params.Save() } -// TODO: add test // Sets the base cost of each additional wasm page func (con ArbOwner) SetWasmPageGas(c ctx, evm mech, gas uint16) error { params, err := c.State.Programs().Params() @@ -237,7 +216,6 @@ func (con ArbOwner) SetWasmPageGas(c ctx, evm mech, gas uint16) error { return params.Save() } -// TODO: add test // Sets the initial number of pages a wasm may allocate func (con ArbOwner) SetWasmPageLimit(c ctx, evm mech, limit uint16) error { params, err := c.State.Programs().Params() @@ -248,7 +226,6 @@ func (con ArbOwner) SetWasmPageLimit(c ctx, evm mech, limit uint16) error { return params.Save() } -// TODO: add test // Sets the minimum costs to invoke a program func (con ArbOwner) SetWasmMinInitGas(c ctx, _ mech, gas, cached uint64) error { params, err := c.State.Programs().Params() @@ -260,7 +237,6 @@ func (con ArbOwner) SetWasmMinInitGas(c ctx, _ mech, gas, cached uint64) error { return params.Save() } -// TODO: add test // Sets the linear adjustment made to program init costs func (con ArbOwner) SetWasmInitCostScalar(c ctx, _ mech, percent uint64) error { params, err := c.State.Programs().Params() @@ -271,7 +247,6 @@ func (con ArbOwner) SetWasmInitCostScalar(c ctx, _ mech, percent uint64) error { return params.Save() } -// TODO: add test // Sets the number of days after which programs deactivate func (con ArbOwner) SetWasmExpiryDays(c ctx, _ mech, days uint16) error { params, err := c.State.Programs().Params() @@ -282,7 +257,6 @@ func (con ArbOwner) SetWasmExpiryDays(c ctx, _ mech, days uint16) error { return params.Save() } -// TODO: add test // Sets the age a program must be to perform a keepalive func (con ArbOwner) SetWasmKeepaliveDays(c ctx, _ mech, days uint16) error { params, err := c.State.Programs().Params() @@ -293,7 +267,6 @@ func (con ArbOwner) SetWasmKeepaliveDays(c ctx, _ mech, days uint16) error { return params.Save() } -// TODO: add test // Sets the number of extra programs ArbOS caches during a given block func (con ArbOwner) SetWasmBlockCacheSize(c ctx, _ mech, count uint16) error { params, err := c.State.Programs().Params() @@ -304,13 +277,11 @@ func (con ArbOwner) SetWasmBlockCacheSize(c ctx, _ mech, count uint16) error { return params.Save() } -// TODO: add test // Adds account as a wasm cache manager func (con ArbOwner) AddWasmCacheManager(c ctx, _ mech, manager addr) error { return c.State.Programs().CacheManagers().Add(manager) } -// TODO: add test // Removes account from the list of wasm cache managers func (con ArbOwner) RemoveWasmCacheManager(c ctx, _ mech, manager addr) error { managers := c.State.Programs().CacheManagers() diff --git a/precompiles/ArbOwnerPublic.go b/precompiles/ArbOwnerPublic.go index 2356c855ee..4c83646aa9 100644 --- a/precompiles/ArbOwnerPublic.go +++ b/precompiles/ArbOwnerPublic.go @@ -36,7 +36,6 @@ func (con ArbOwnerPublic) IsChainOwner(c ctx, evm mech, addr addr) (bool, error) return c.State.ChainOwners().IsMember(addr) } -// TODO: add test // GetNetworkFeeAccount gets the network fee collector func (con ArbOwnerPublic) GetNetworkFeeAccount(c ctx, evm mech) (addr, error) { return c.State.NetworkFeeAccount() @@ -56,7 +55,6 @@ func (con ArbOwnerPublic) GetBrotliCompressionLevel(c ctx, evm mech) (uint64, er return c.State.BrotliCompressionLevel() } -// TODO: add test // GetScheduledUpgrade gets the next scheduled ArbOS version upgrade and its activation timestamp. // Returns (0, 0, nil) if no ArbOS upgrade is scheduled. func (con ArbOwnerPublic) GetScheduledUpgrade(c ctx, evm mech) (uint64, uint64, error) { diff --git a/precompiles/ArbSys.go b/precompiles/ArbSys.go index 295085a3c5..a609f55dcd 100644 --- a/precompiles/ArbSys.go +++ b/precompiles/ArbSys.go @@ -28,13 +28,11 @@ type ArbSys struct { L2ToL1TransactionGasCost func(addr, addr, huge, huge, huge, huge, huge, huge, huge, []byte) (uint64, error) } -// TODO: add test // ArbBlockNumber gets the current L2 block number func (con *ArbSys) ArbBlockNumber(c ctx, evm mech) (huge, error) { return evm.Context.BlockNumber, nil } -// TODO: add test // ArbBlockHash gets the L2 block hash, if sufficiently recent func (con *ArbSys) ArbBlockHash(c ctx, evm mech, arbBlockNumber *big.Int) (bytes32, error) { if !arbBlockNumber.IsUint64() { @@ -56,7 +54,6 @@ func (con *ArbSys) ArbBlockHash(c ctx, evm mech, arbBlockNumber *big.Int) (bytes return evm.Context.GetHash(requestedBlockNum), nil } -// TODO: add test // ArbChainID gets the rollup's unique chain identifier func (con *ArbSys) ArbChainID(c ctx, evm mech) (huge, error) { return evm.ChainConfig().ChainID, nil @@ -75,7 +72,6 @@ func (con *ArbSys) GetStorageGasAvailable(c ctx, evm mech) (huge, error) { return big.NewInt(0), nil } -// TODO: add test // IsTopLevelCall checks if the call is top-level (deprecated) func (con *ArbSys) IsTopLevelCall(c ctx, evm mech) (bool, error) { return evm.Depth() <= 2, nil @@ -87,7 +83,6 @@ func (con *ArbSys) MapL1SenderContractAddressToL2Alias(c ctx, sender addr, dest return util.RemapL1Address(sender), nil } -// TODO: add test // WasMyCallersAddressAliased checks if the caller's caller was aliased func (con *ArbSys) WasMyCallersAddressAliased(c ctx, evm mech) (bool, error) { topLevel := con.isTopLevel(c, evm) @@ -98,7 +93,6 @@ func (con *ArbSys) WasMyCallersAddressAliased(c ctx, evm mech) (bool, error) { return aliased, nil } -// TODO: add test // MyCallersAddressWithoutAliasing gets the caller's caller without any potential aliasing func (con *ArbSys) MyCallersAddressWithoutAliasing(c ctx, evm mech) (addr, error) { @@ -196,7 +190,6 @@ func (con *ArbSys) SendTxToL1(c ctx, evm mech, value huge, destination addr, cal return sendHash.Big(), err } -// TODO: add test // SendMerkleTreeState gets the root, size, and partials of the outbox Merkle tree state (caller must be the 0 address) func (con ArbSys) SendMerkleTreeState(c ctx, evm mech) (huge, bytes32, []bytes32, error) { if c.caller != (addr{}) { @@ -213,13 +206,11 @@ func (con ArbSys) SendMerkleTreeState(c ctx, evm mech) (huge, bytes32, []bytes32 return new(big.Int).SetUint64(size), rootHash, partials, nil } -// TODO: add test // WithdrawEth send paid eth to the destination on L1 func (con ArbSys) WithdrawEth(c ctx, evm mech, value huge, destination addr) (huge, error) { return con.SendTxToL1(c, evm, value, destination, []byte{}) } -// TODO: add test func (con ArbSys) isTopLevel(c ctx, evm mech) bool { depth := evm.Depth() return depth < 2 || evm.Origin == c.txProcessor.Contracts[depth-2].Caller() diff --git a/precompiles/ArbosTest.go b/precompiles/ArbosTest.go index d34bc740c7..6e988f533a 100644 --- a/precompiles/ArbosTest.go +++ b/precompiles/ArbosTest.go @@ -12,7 +12,6 @@ type ArbosTest struct { Address addr // 0x69 } -// TODO: add test // BurnArbGas unproductively burns the amount of L2 ArbGas func (con ArbosTest) BurnArbGas(c ctx, gasAmount huge) error { if !gasAmount.IsUint64() { diff --git a/precompiles/context.go b/precompiles/context.go index 25aceba8ec..670ffa7443 100644 --- a/precompiles/context.go +++ b/precompiles/context.go @@ -35,7 +35,6 @@ type Context struct { readOnly bool } -// TODO: add test func (c *Context) Burn(amount uint64) error { if c.gasLeft < amount { return c.BurnOut() @@ -44,44 +43,36 @@ func (c *Context) Burn(amount uint64) error { return nil } -// TODO: add test //nolint:unused func (c *Context) Burned() uint64 { return c.gasSupplied - c.gasLeft } -// TODO: add test func (c *Context) BurnOut() error { c.gasLeft = 0 return vm.ErrOutOfGas } -// TODO: add test func (c *Context) GasLeft() *uint64 { return &c.gasLeft } -// TODO: add test func (c *Context) Restrict(err error) { log.Crit("A metered burner was used for access-controlled work", "error", err) } -// TODO: add test func (c *Context) HandleError(err error) error { return err } -// TODO: add test func (c *Context) ReadOnly() bool { return c.readOnly } -// TODO: add test func (c *Context) TracingInfo() *util.TracingInfo { return c.tracingInfo } -// TODO: add test func (c *Context) GetCodeHash(address common.Address) (common.Hash, error) { return c.State.BackingStorage().GetCodeHash(address) } From c3b011ea25c94412eaf6ec1d326739323c5eee26 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 24 Sep 2024 16:08:45 -0300 Subject: [PATCH 389/489] Test for ArbAddressTable.AddressExists --- precompiles/ArbAddressTable.go | 1 - precompiles/ArbAddressTable_test.go | 12 ++++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/precompiles/ArbAddressTable.go b/precompiles/ArbAddressTable.go index 5ded229be6..102fd55c3b 100644 --- a/precompiles/ArbAddressTable.go +++ b/precompiles/ArbAddressTable.go @@ -13,7 +13,6 @@ type ArbAddressTable struct { Address addr // 0x66 } -// TODO: add test // AddressExists checks if an address exists in the table func (con ArbAddressTable) AddressExists(c ctx, evm mech, addr addr) (bool, error) { return c.State.AddressTable().AddressExists(addr) diff --git a/precompiles/ArbAddressTable_test.go b/precompiles/ArbAddressTable_test.go index b01a460636..62ce177480 100644 --- a/precompiles/ArbAddressTable_test.go +++ b/precompiles/ArbAddressTable_test.go @@ -47,6 +47,12 @@ func TestAddressTable1(t *testing.T) { addr := common.BytesToAddress(crypto.Keccak256([]byte{})[:20]) + exists, err := atab.AddressExists(context, evm, addr) + Require(t, err) + if exists { + t.Fatal("Address shouldn't exist") + } + // register addr slot, err := atab.Register(context, evm, addr) Require(t, err) @@ -61,6 +67,12 @@ func TestAddressTable1(t *testing.T) { t.Fatal() } + exists, err = atab.AddressExists(context, evm, addr) + Require(t, err) + if !exists { + t.Fatal("Address should exist") + } + // verify Lookup of addr returns 0 index, err := atab.Lookup(context, evm, addr) Require(t, err) From 32975b921ecf3faa3f1fc7f01e9598926ae95f92 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 25 Sep 2024 07:16:28 -0300 Subject: [PATCH 390/489] Do not test deprecated methods in ArbAggregator --- precompiles/ArbAggregator.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/precompiles/ArbAggregator.go b/precompiles/ArbAggregator.go index 438707d969..b74e280fe8 100644 --- a/precompiles/ArbAggregator.go +++ b/precompiles/ArbAggregator.go @@ -19,14 +19,12 @@ type ArbAggregator struct { var ErrNotOwner = errors.New("must be called by chain owner") -// TODO: add test // GetPreferredAggregator returns the preferred aggregator address. // Deprecated: Do not use this method. func (con ArbAggregator) GetPreferredAggregator(c ctx, evm mech, address addr) (prefAgg addr, isDefault bool, err error) { return l1pricing.BatchPosterAddress, true, err } -// TODO: add test // GetDefaultAggregator returns the default aggregator address. // Deprecated: Do not use this method. func (con ArbAggregator) GetDefaultAggregator(c ctx, evm mech) (addr, error) { From 6615fac1110de6739cea1c220f8f77fbb0636579 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 25 Sep 2024 08:18:56 -0300 Subject: [PATCH 391/489] Do not test simple ArbDebug methods --- precompiles/ArbDebug.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/precompiles/ArbDebug.go b/precompiles/ArbDebug.go index f20532f6e6..bf85d5e18f 100644 --- a/precompiles/ArbDebug.go +++ b/precompiles/ArbDebug.go @@ -56,13 +56,11 @@ func (con ArbDebug) BecomeChainOwner(c ctx, evm mech) error { return c.State.ChainOwners().Add(c.caller) } -// TODO: add test // Halts the chain by panicking in the STF func (con ArbDebug) Panic(c ctx, evm mech) error { panic("called ArbDebug's debug-only Panic method") } -// TODO: add test func (con ArbDebug) LegacyError(c ctx) error { return errors.New("example legacy error") } From 94243c8b71553a12c45d300bfdfc33e2ab7bbeba Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 25 Sep 2024 08:41:50 -0300 Subject: [PATCH 392/489] Tests for ArbFunctionTable --- precompiles/ArbFunctionTable.go | 3 --- precompiles/ArbFunctionTable_test.go | 35 ++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 precompiles/ArbFunctionTable_test.go diff --git a/precompiles/ArbFunctionTable.go b/precompiles/ArbFunctionTable.go index c3a545136a..a870995e0f 100644 --- a/precompiles/ArbFunctionTable.go +++ b/precompiles/ArbFunctionTable.go @@ -15,19 +15,16 @@ type ArbFunctionTable struct { Address addr // 0x68 } -// TODO: add test // Upload does nothing func (con ArbFunctionTable) Upload(c ctx, evm mech, buf []byte) error { return nil } -// TODO: add test // Size returns the empty table's size, which is 0 func (con ArbFunctionTable) Size(c ctx, evm mech, addr addr) (huge, error) { return big.NewInt(0), nil } -// TODO: add test // Get reverts since the table is empty func (con ArbFunctionTable) Get(c ctx, evm mech, addr addr, index huge) (huge, bool, huge, error) { return nil, false, nil, errors.New("table is empty") diff --git a/precompiles/ArbFunctionTable_test.go b/precompiles/ArbFunctionTable_test.go new file mode 100644 index 0000000000..6a68bffd22 --- /dev/null +++ b/precompiles/ArbFunctionTable_test.go @@ -0,0 +1,35 @@ +// Copyright 2021-2024, Offchain Labs, Inc. +// For license information, see https://github.com/nitro/blob/master/LICENSE + +package precompiles + +import ( + "math/big" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" +) + +func TestArbFunctionTable(t *testing.T) { + evm := newMockEVMForTesting() + ftab := ArbFunctionTable{} + context := testContext(common.Address{}, evm) + + addr := common.BytesToAddress(crypto.Keccak256([]byte{})[:20]) + + // should be a noop + err := ftab.Upload(context, evm, []byte{0, 0, 0, 0}) + Require(t, err) + + size, err := ftab.Size(context, evm, addr) + Require(t, err) + if size.Cmp(big.NewInt(0)) != 0 { + t.Fatal("Size should be 0") + } + + _, _, _, err = ftab.Get(context, evm, addr, big.NewInt(10)) + if err == nil { + t.Fatal("Should error") + } +} From ac7f18e97c8b2be14632ae0f1c4dece1b903fada Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 25 Sep 2024 09:32:11 -0300 Subject: [PATCH 393/489] Tests for ArbInfo --- precompiles/ArbInfo.go | 2 -- system_tests/program_test.go | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/precompiles/ArbInfo.go b/precompiles/ArbInfo.go index c78dd84889..9f8cf34532 100644 --- a/precompiles/ArbInfo.go +++ b/precompiles/ArbInfo.go @@ -13,7 +13,6 @@ type ArbInfo struct { Address addr // 0x65 } -// TODO: add test // GetBalance retrieves an account's balance func (con ArbInfo) GetBalance(c ctx, evm mech, account addr) (huge, error) { if err := c.Burn(params.BalanceGasEIP1884); err != nil { @@ -22,7 +21,6 @@ func (con ArbInfo) GetBalance(c ctx, evm mech, account addr) (huge, error) { return evm.StateDB.GetBalance(account).ToBig(), nil } -// TODO: add test // GetCode retrieves a contract's deployed code func (con ArbInfo) GetCode(c ctx, evm mech, account addr) ([]byte, error) { if err := c.Burn(params.ColdSloadCostEIP2929); err != nil { diff --git a/system_tests/program_test.go b/system_tests/program_test.go index 4c896d1791..cda592ee06 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -535,6 +535,16 @@ func testCalls(t *testing.T, jit bool) { defer cleanup() callsAddr := deployWasm(t, ctx, auth, l2client, rustFile("multicall")) + // checks ArbInfo.GetCode works properly + codeFromFile, _ := readWasmFile(t, rustFile("multicall")) + arbInfo, err := pgen.NewArbInfo(types.ArbInfoAddress, l2client) + Require(t, err) + codeFromArbInfo, err := arbInfo.GetCode(nil, callsAddr) + Require(t, err) + if !bytes.Equal(codeFromFile, codeFromArbInfo) { + t.Fatal("ArbInfo.GetCode returned wrong code") + } + ensure := func(tx *types.Transaction, err error) *types.Receipt { t.Helper() Require(t, err) @@ -716,6 +726,13 @@ func testCalls(t *testing.T, jit bool) { Fatal(t, balance, value) } + // checks ArbInfo.GetBalance works properly + balance, err = arbInfo.GetBalance(nil, eoa) + Require(t, err) + if !arbmath.BigEquals(balance, value) { + Fatal(t, balance, value) + } + blocks := []uint64{10} validateBlockRange(t, blocks, jit, builder) } From 18c3336d7fc211cf6a1fcffb5212ae98837dc234 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 25 Sep 2024 09:44:05 -0300 Subject: [PATCH 394/489] Removes TODOs from ArbWasmCache --- precompiles/ArbWasmCache.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/precompiles/ArbWasmCache.go b/precompiles/ArbWasmCache.go index 053cfb3792..3cada9dd70 100644 --- a/precompiles/ArbWasmCache.go +++ b/precompiles/ArbWasmCache.go @@ -12,25 +12,21 @@ type ArbWasmCache struct { UpdateProgramCacheGasCost func(addr, bytes32, bool) (uint64, error) } -// TODO: add test // See if the user is a cache manager owner. func (con ArbWasmCache) IsCacheManager(c ctx, _ mech, addr addr) (bool, error) { return c.State.Programs().CacheManagers().IsMember(addr) } -// TODO: add test // Retrieve all authorized address managers. func (con ArbWasmCache) AllCacheManagers(c ctx, _ mech) ([]addr, error) { return c.State.Programs().CacheManagers().AllMembers(65536) } -// TODO: add test // Deprecated: replaced with CacheProgram. func (con ArbWasmCache) CacheCodehash(c ctx, evm mech, codehash hash) error { return con.setProgramCached(c, evm, common.Address{}, codehash, true) } -// TODO: add test // Caches all programs with a codehash equal to the given address. Caller must be a cache manager or chain owner. func (con ArbWasmCache) CacheProgram(c ctx, evm mech, address addr) error { codehash, err := c.GetCodeHash(address) @@ -40,7 +36,6 @@ func (con ArbWasmCache) CacheProgram(c ctx, evm mech, address addr) error { return con.setProgramCached(c, evm, address, codehash, true) } -// TODO: add test // Evicts all programs with the given codehash. Caller must be a cache manager or chain owner. func (con ArbWasmCache) EvictCodehash(c ctx, evm mech, codehash hash) error { return con.setProgramCached(c, evm, common.Address{}, codehash, false) From 844b5ea0a4f0d73c0f27c69cea7ffa19909da7ca Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 25 Sep 2024 09:44:53 -0300 Subject: [PATCH 395/489] Removes TODOs from ArbWasm --- precompiles/ArbWasm.go | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/precompiles/ArbWasm.go b/precompiles/ArbWasm.go index f8ece25600..bc24c8a6e8 100644 --- a/precompiles/ArbWasm.go +++ b/precompiles/ArbWasm.go @@ -29,7 +29,6 @@ type ArbWasm struct { ProgramInsufficientValueError func(have, want huge) error } -// TODO: add test // Compile a wasm program with the latest instrumentation func (con ArbWasm) ActivateProgram(c ctx, evm mech, value huge, program addr) (uint16, huge, error) { debug := evm.ChainConfig().DebugMode() @@ -54,7 +53,6 @@ func (con ArbWasm) ActivateProgram(c ctx, evm mech, value huge, program addr) (u return version, dataFee, con.ProgramActivated(c, evm, codeHash, moduleHash, program, dataFee, version) } -// TODO: add test // Extends a program's expiration date (reverts if too soon) func (con ArbWasm) CodehashKeepalive(c ctx, evm mech, value huge, codehash bytes32) error { params, err := c.State.Programs().Params() @@ -91,56 +89,48 @@ func (con ArbWasm) payActivationDataFee(c ctx, evm mech, value, dataFee huge) er return util.TransferBalance(&con.Address, &c.caller, repay, evm, scenario, "reimburse") } -// TODO: add test // Gets the latest stylus version func (con ArbWasm) StylusVersion(c ctx, evm mech) (uint16, error) { params, err := c.State.Programs().Params() return params.Version, err } -// TODO: add test // Gets the amount of ink 1 gas buys func (con ArbWasm) InkPrice(c ctx, _ mech) (uint32, error) { params, err := c.State.Programs().Params() return params.InkPrice.ToUint32(), err } -// TODO: add test // Gets the wasm stack size limit func (con ArbWasm) MaxStackDepth(c ctx, _ mech) (uint32, error) { params, err := c.State.Programs().Params() return params.MaxStackDepth, err } -// TODO: add test // Gets the number of free wasm pages a tx gets func (con ArbWasm) FreePages(c ctx, _ mech) (uint16, error) { params, err := c.State.Programs().Params() return params.FreePages, err } -// TODO: add test // Gets the base cost of each additional wasm page func (con ArbWasm) PageGas(c ctx, _ mech) (uint16, error) { params, err := c.State.Programs().Params() return params.PageGas, err } -// TODO: add test // Gets the ramp that drives exponential memory costs func (con ArbWasm) PageRamp(c ctx, _ mech) (uint64, error) { params, err := c.State.Programs().Params() return params.PageRamp, err } -// TODO: add test // Gets the maximum initial number of pages a wasm may allocate func (con ArbWasm) PageLimit(c ctx, _ mech) (uint16, error) { params, err := c.State.Programs().Params() return params.PageLimit, err } -// TODO: add test // Gets the minimum costs to invoke a program func (con ArbWasm) MinInitGas(c ctx, _ mech) (uint64, uint64, error) { params, err := c.State.Programs().Params() @@ -152,35 +142,30 @@ func (con ArbWasm) MinInitGas(c ctx, _ mech) (uint64, uint64, error) { return init, cached, err } -// TODO: add test // Gets the linear adjustment made to program init costs func (con ArbWasm) InitCostScalar(c ctx, _ mech) (uint64, error) { params, err := c.State.Programs().Params() return uint64(params.InitCostScalar) * programs.CostScalarPercent, err } -// TODO: add test // Gets the number of days after which programs deactivate func (con ArbWasm) ExpiryDays(c ctx, _ mech) (uint16, error) { params, err := c.State.Programs().Params() return params.ExpiryDays, err } -// TODO: add test // Gets the age a program must be to perform a keepalive func (con ArbWasm) KeepaliveDays(c ctx, _ mech) (uint16, error) { params, err := c.State.Programs().Params() return params.KeepaliveDays, err } -// TODO: add test // Gets the number of extra programs ArbOS caches during a given block. func (con ArbWasm) BlockCacheSize(c ctx, _ mech) (uint16, error) { params, err := c.State.Programs().Params() return params.BlockCacheSize, err } -// TODO: add test // Gets the stylus version that program with codehash was most recently compiled with func (con ArbWasm) CodehashVersion(c ctx, evm mech, codehash bytes32) (uint16, error) { params, err := c.State.Programs().Params() @@ -190,7 +175,6 @@ func (con ArbWasm) CodehashVersion(c ctx, evm mech, codehash bytes32) (uint16, e return c.State.Programs().CodehashVersion(codehash, evm.Context.Time, params) } -// TODO: add test // Gets a program's asm size in bytes func (con ArbWasm) CodehashAsmSize(c ctx, evm mech, codehash bytes32) (uint32, error) { params, err := c.State.Programs().Params() @@ -200,7 +184,6 @@ func (con ArbWasm) CodehashAsmSize(c ctx, evm mech, codehash bytes32) (uint32, e return c.State.Programs().ProgramAsmSize(codehash, evm.Context.Time, params) } -// TODO: add test // Gets the stylus version that program at addr was most recently compiled with func (con ArbWasm) ProgramVersion(c ctx, evm mech, program addr) (uint16, error) { codehash, err := c.GetCodeHash(program) @@ -210,7 +193,6 @@ func (con ArbWasm) ProgramVersion(c ctx, evm mech, program addr) (uint16, error) return con.CodehashVersion(c, evm, codehash) } -// TODO: add test // Gets the cost to invoke the program func (con ArbWasm) ProgramInitGas(c ctx, evm mech, program addr) (uint64, uint64, error) { codehash, params, err := con.getCodeHash(c, program) @@ -220,7 +202,6 @@ func (con ArbWasm) ProgramInitGas(c ctx, evm mech, program addr) (uint64, uint64 return c.State.Programs().ProgramInitGas(codehash, evm.Context.Time, params) } -// TODO: add test // Gets the footprint of program at addr func (con ArbWasm) ProgramMemoryFootprint(c ctx, evm mech, program addr) (uint16, error) { codehash, params, err := con.getCodeHash(c, program) @@ -230,7 +211,6 @@ func (con ArbWasm) ProgramMemoryFootprint(c ctx, evm mech, program addr) (uint16 return c.State.Programs().ProgramMemoryFootprint(codehash, evm.Context.Time, params) } -// TODO: add test // Gets returns the amount of time remaining until the program expires func (con ArbWasm) ProgramTimeLeft(c ctx, evm mech, program addr) (uint64, error) { codehash, params, err := con.getCodeHash(c, program) From a06b470abc5373ee330415d5c89adb6118eb8fe9 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 25 Sep 2024 09:56:02 -0300 Subject: [PATCH 396/489] Tests for ArbStatistics --- precompiles/ArbStatistics.go | 1 - precompiles/ArbStatistics_test.go | 22 ++++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 precompiles/ArbStatistics_test.go diff --git a/precompiles/ArbStatistics.go b/precompiles/ArbStatistics.go index 74d1c2dd15..83f867d09d 100644 --- a/precompiles/ArbStatistics.go +++ b/precompiles/ArbStatistics.go @@ -14,7 +14,6 @@ type ArbStatistics struct { Address addr // 0x6e } -// TODO: add test // GetStats returns the current block number and some statistics about the rollup's pre-Nitro state func (con ArbStatistics) GetStats(c ctx, evm mech) (huge, huge, huge, huge, huge, huge, error) { blockNum := evm.Context.BlockNumber diff --git a/precompiles/ArbStatistics_test.go b/precompiles/ArbStatistics_test.go new file mode 100644 index 0000000000..9787752906 --- /dev/null +++ b/precompiles/ArbStatistics_test.go @@ -0,0 +1,22 @@ +// Copyright 2021-2024, Offchain Labs, Inc. +// For license information, see https://github.com/nitro/blob/master/LICENSE + +package precompiles + +import ( + "testing" + + "github.com/ethereum/go-ethereum/common" +) + +func TestArbStatistics(t *testing.T) { + evm := newMockEVMForTesting() + stats := ArbStatistics{} + context := testContext(common.Address{}, evm) + + blockNum, _, _, _, _, _, err := stats.GetStats(context, evm) + Require(t, err) + if blockNum.Cmp(evm.Context.BlockNumber) != 0 { + t.Error("Unexpected block number") + } +} From da7d145cdc370475f91d37a029808c7f88b2d64a Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 25 Sep 2024 09:58:23 -0300 Subject: [PATCH 397/489] Do not test simple ArbosActs methods --- precompiles/ArbosActs.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/precompiles/ArbosActs.go b/precompiles/ArbosActs.go index 475faf0e85..e18aa43efb 100644 --- a/precompiles/ArbosActs.go +++ b/precompiles/ArbosActs.go @@ -11,12 +11,10 @@ type ArbosActs struct { CallerNotArbOSError func() error } -// TODO: add test func (con ArbosActs) StartBlock(c ctx, evm mech, l1BaseFee huge, l1BlockNumber, l2BlockNumber, timeLastBlock uint64) error { return con.CallerNotArbOSError() } -// TODO: add test func (con ArbosActs) BatchPostingReport(c ctx, evm mech, batchTimestamp huge, batchPosterAddress addr, batchNumber uint64, batchDataGas uint64, l1BaseFeeWei huge) error { return con.CallerNotArbOSError() } From 90ab8d0ac24f548b57f8d0b7c1de80459e68c697 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 25 Sep 2024 10:40:58 -0300 Subject: [PATCH 398/489] Tests for ArbSys.GetStorageGasAvailable and ArbSys.ArbOSVersion --- precompiles/ArbSys.go | 3 --- system_tests/precompile_test.go | 17 ++++++++++++++++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/precompiles/ArbSys.go b/precompiles/ArbSys.go index a609f55dcd..bdbf9066f9 100644 --- a/precompiles/ArbSys.go +++ b/precompiles/ArbSys.go @@ -59,14 +59,12 @@ func (con *ArbSys) ArbChainID(c ctx, evm mech) (huge, error) { return evm.ChainConfig().ChainID, nil } -// TODO: add test // ArbOSVersion gets the current ArbOS version func (con *ArbSys) ArbOSVersion(c ctx, evm mech) (huge, error) { version := new(big.Int).SetUint64(55 + c.State.ArbOSVersion()) // Nitro starts at version 56 return version, nil } -// TODO: add test // GetStorageGasAvailable returns 0 since Nitro has no concept of storage gas func (con *ArbSys) GetStorageGasAvailable(c ctx, evm mech) (huge, error) { return big.NewInt(0), nil @@ -109,7 +107,6 @@ func (con *ArbSys) MyCallersAddressWithoutAliasing(c ctx, evm mech) (addr, error return address, err } -// TODO: add test // SendTxToL1 sends a transaction to L1, adding it to the outbox func (con *ArbSys) SendTxToL1(c ctx, evm mech, value huge, destination addr, calldataForL1 []byte) (huge, error) { l1BlockNum, err := c.txProcessor.L1BlockNumber(vm.BlockContext{}) diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index 9e829124ee..68c630ff33 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -22,7 +22,10 @@ func TestPurePrecompileMethodCalls(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - builder := NewNodeBuilder(ctx).DefaultConfig(t, false) + arbosVersion := uint64(31) + builder := NewNodeBuilder(ctx). + DefaultConfig(t, false). + WithArbOSVersion(arbosVersion) cleanup := builder.Build(t) defer cleanup() @@ -33,6 +36,18 @@ func TestPurePrecompileMethodCalls(t *testing.T) { if chainId.Uint64() != params.ArbitrumDevTestChainConfig().ChainID.Uint64() { Fatal(t, "Wrong ChainID", chainId.Uint64()) } + + arbSysArbosVersion, err := arbSys.ArbOSVersion(&bind.CallOpts{}) + Require(t, err) + if arbSysArbosVersion.Uint64() != 55+arbosVersion { // Nitro versios start at 56 + Fatal(t, "Expected ArbOSVersion 86, got", arbosVersion) + } + + storageGasAvailable, err := arbSys.GetStorageGasAvailable(&bind.CallOpts{}) + Require(t, err) + if storageGasAvailable.Cmp(big.NewInt(0)) != 0 { + Fatal(t, "Expected 0 storage gas available, got", storageGasAvailable) + } } func TestViewLogReverts(t *testing.T) { From 8e742746fa19fad560669288d7cd574f6dce6c64 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 25 Sep 2024 10:41:32 -0300 Subject: [PATCH 399/489] Do not test ArbSys.MapL1SenderContractAddressToL2Alias It only calls util.RemapL1Address, that is covered in other test flows --- precompiles/ArbSys.go | 1 - 1 file changed, 1 deletion(-) diff --git a/precompiles/ArbSys.go b/precompiles/ArbSys.go index bdbf9066f9..d55067a09c 100644 --- a/precompiles/ArbSys.go +++ b/precompiles/ArbSys.go @@ -75,7 +75,6 @@ func (con *ArbSys) IsTopLevelCall(c ctx, evm mech) (bool, error) { return evm.Depth() <= 2, nil } -// TODO: add test // MapL1SenderContractAddressToL2Alias gets the contract's L2 alias func (con *ArbSys) MapL1SenderContractAddressToL2Alias(c ctx, sender addr, dest addr) (addr, error) { return util.RemapL1Address(sender), nil From d712416b20518928dab40892c119ac61ae43e1d3 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 25 Sep 2024 11:24:34 -0300 Subject: [PATCH 400/489] Test for ArbOwnerPublic.GetBrotliCompressionLevel --- precompiles/ArbOwnerPublic.go | 1 - system_tests/precompile_test.go | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/precompiles/ArbOwnerPublic.go b/precompiles/ArbOwnerPublic.go index 4c83646aa9..177268d9bb 100644 --- a/precompiles/ArbOwnerPublic.go +++ b/precompiles/ArbOwnerPublic.go @@ -49,7 +49,6 @@ func (con ArbOwnerPublic) GetInfraFeeAccount(c ctx, evm mech) (addr, error) { return c.State.InfraFeeAccount() } -// TODO: add test // GetBrotliCompressionLevel gets the current brotli compression level used for fast compression func (con ArbOwnerPublic) GetBrotliCompressionLevel(c ctx, evm mech) (uint64, error) { return c.State.BrotliCompressionLevel() diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index 68c630ff33..0b63d72e4d 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -140,6 +140,39 @@ func TestPrecompileErrorGasLeft(t *testing.T) { assertNotAllGasConsumed(common.HexToAddress("0xff"), arbDebug.Methods["legacyError"].ID) } +func TestGetBrotliCompressionLevel(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, false) + cleanup := builder.Build(t) + defer cleanup() + + auth := builder.L2Info.GetDefaultTransactOpts("Owner", ctx) + + arbOwnerPublic, err := precompilesgen.NewArbOwnerPublic(common.HexToAddress("0x6b"), builder.L2.Client) + Require(t, err, "could not bind ArbOwner contract") + + arbOwner, err := precompilesgen.NewArbOwner(common.HexToAddress("0x70"), builder.L2.Client) + Require(t, err, "could not bind ArbOwner contract") + + brotliCompressionLevel := uint64(11) + + // sets brotli compression level + tx, err := arbOwner.SetBrotliCompressionLevel(&auth, brotliCompressionLevel) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + + // retrieves brotli compression level + callOpts := &bind.CallOpts{Context: ctx} + retrievedBrotliCompressionLevel, err := arbOwnerPublic.GetBrotliCompressionLevel(callOpts) + Require(t, err, "failed to call GetBrotliCompressionLevel") + if retrievedBrotliCompressionLevel != brotliCompressionLevel { + Fatal(t, "expected brotli compression level to be", brotliCompressionLevel, "got", retrievedBrotliCompressionLevel) + } +} + func TestScheduleArbosUpgrade(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() From 115bda4eff1780ee075fa03dd18f475c2254eb02 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 25 Sep 2024 11:25:14 -0300 Subject: [PATCH 401/489] Do not test ArbOwnerPublic.RectifyChainOwner It basically uses AddressSet.RectifyMapping that is already covered in other tests --- precompiles/ArbOwnerPublic.go | 1 - 1 file changed, 1 deletion(-) diff --git a/precompiles/ArbOwnerPublic.go b/precompiles/ArbOwnerPublic.go index 177268d9bb..451e18e1cc 100644 --- a/precompiles/ArbOwnerPublic.go +++ b/precompiles/ArbOwnerPublic.go @@ -21,7 +21,6 @@ func (con ArbOwnerPublic) GetAllChainOwners(c ctx, evm mech) ([]common.Address, return c.State.ChainOwners().AllMembers(65536) } -// TODO: add test // RectifyChainOwner checks if the account is a chain owner func (con ArbOwnerPublic) RectifyChainOwner(c ctx, evm mech, addr addr) error { err := c.State.ChainOwners().RectifyMapping(addr) From 3f32c0f99031342797b5bcb60935359866118523 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 25 Sep 2024 11:42:18 -0300 Subject: [PATCH 402/489] Comment to check ArbOwner later --- precompiles/ArbOwner.go | 1 + 1 file changed, 1 insertion(+) diff --git a/precompiles/ArbOwner.go b/precompiles/ArbOwner.go index 8b87445e0e..040ce7054b 100644 --- a/precompiles/ArbOwner.go +++ b/precompiles/ArbOwner.go @@ -1,3 +1,4 @@ +// TODO: check // Copyright 2021-2024, Offchain Labs, Inc. // For license information, see https://github.com/nitro/blob/master/LICENSE From 5a7ca32045e753540faf5e0e08c4f0cabaa4170f Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 25 Sep 2024 12:44:40 -0300 Subject: [PATCH 403/489] TestGetLifetime --- precompiles/ArbRetryableTx.go | 3 --- precompiles/ArbRetryableTx_test.go | 13 +++++++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/precompiles/ArbRetryableTx.go b/precompiles/ArbRetryableTx.go index abfdc94807..3410754b63 100644 --- a/precompiles/ArbRetryableTx.go +++ b/precompiles/ArbRetryableTx.go @@ -45,7 +45,6 @@ func (con ArbRetryableTx) oldNotFoundError(c ctx) error { return errors.New("ticketId not found") } -// TODO: add test // Redeem schedules an attempt to redeem the retryable, donating all of the call's gas to the redeem attempt func (con ArbRetryableTx) Redeem(c ctx, evm mech, ticketId bytes32) (bytes32, error) { if c.txProcessor.CurrentRetryable != nil && ticketId == *c.txProcessor.CurrentRetryable { @@ -131,13 +130,11 @@ func (con ArbRetryableTx) Redeem(c ctx, evm mech, ticketId bytes32) (bytes32, er return retryTxHash, c.State.L2PricingState().AddToGasPool(arbmath.SaturatingCast[int64](gasToDonate)) } -// TODO: add test // GetLifetime gets the default lifetime period a retryable has at creation func (con ArbRetryableTx) GetLifetime(c ctx, evm mech) (huge, error) { return big.NewInt(retryables.RetryableLifetimeSeconds), nil } -// TODO: add test // GetTimeout gets the timestamp for when ticket will expire func (con ArbRetryableTx) GetTimeout(c ctx, evm mech, ticketId bytes32) (huge, error) { retryableState := c.State.RetryableState() diff --git a/precompiles/ArbRetryableTx_test.go b/precompiles/ArbRetryableTx_test.go index 9ccb437abc..af52926e75 100644 --- a/precompiles/ArbRetryableTx_test.go +++ b/precompiles/ArbRetryableTx_test.go @@ -7,12 +7,25 @@ import ( "math/big" "testing" + "github.com/offchainlabs/nitro/arbos/retryables" "github.com/offchainlabs/nitro/arbos/storage" "github.com/ethereum/go-ethereum/common" templates "github.com/offchainlabs/nitro/solgen/go/precompilesgen" ) +func TestGetLifetime(t *testing.T) { + evm := newMockEVMForTesting() + retryableTx := ArbRetryableTx{} + context := testContext(common.Address{}, evm) + + lifetime, err := retryableTx.GetLifetime(context, evm) + Require(t, err) + if lifetime.Cmp(big.NewInt(retryables.RetryableLifetimeSeconds)) != 0 { + t.Fatal("Expected to be ", retryables.RetryableLifetimeSeconds, " but got ", lifetime) + } +} + func TestRetryableRedeem(t *testing.T) { evm := newMockEVMForTesting() precompileCtx := testContext(common.Address{}, evm) From 4b6d4a033b514bf30c7f1ad523280d0dbb205b3a Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 26 Sep 2024 08:33:29 -0300 Subject: [PATCH 404/489] TestArbGasInfoAndArbOwner --- precompiles/ArbGasInfo.go | 11 ---- precompiles/ArbOwner.go | 5 +- system_tests/precompile_test.go | 101 ++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 12 deletions(-) diff --git a/precompiles/ArbGasInfo.go b/precompiles/ArbGasInfo.go index e3e0029d9e..cd3ca6458d 100644 --- a/precompiles/ArbGasInfo.go +++ b/precompiles/ArbGasInfo.go @@ -22,7 +22,6 @@ var storageArbGas = big.NewInt(int64(storage.StorageWriteCost)) const AssumedSimpleTxSize = 140 -// TODO: add test // GetPricesInWeiWithAggregator gets prices in wei when using the provided aggregator func (con ArbGasInfo) GetPricesInWeiWithAggregator( c ctx, @@ -66,7 +65,6 @@ func (con ArbGasInfo) GetPricesInWeiWithAggregator( return perL2Tx, weiForL1Calldata, weiForL2Storage, perArbGasBase, perArbGasCongestion, perArbGasTotal, nil } -// TODO: add test func (con ArbGasInfo) _preVersion4_GetPricesInWeiWithAggregator( c ctx, evm mech, @@ -134,7 +132,6 @@ func (con ArbGasInfo) GetPricesInArbGasWithAggregator(c ctx, evm mech, aggregato return gasPerL2Tx, gasForL1Calldata, storageArbGas, nil } -// TODO: add test func (con ArbGasInfo) _preVersion4_GetPricesInArbGasWithAggregator(c ctx, evm mech, aggregator addr) (huge, huge, huge, error) { l1GasPrice, err := c.State.L1PricingState().PricePerUnit() if err != nil { @@ -183,19 +180,16 @@ func (con ArbGasInfo) GetL1BaseFeeEstimate(c ctx, evm mech) (huge, error) { return c.State.L1PricingState().PricePerUnit() } -// TODO: add test // GetL1BaseFeeEstimateInertia gets how slowly ArbOS updates its estimate of the L1 basefee func (con ArbGasInfo) GetL1BaseFeeEstimateInertia(c ctx, evm mech) (uint64, error) { return c.State.L1PricingState().Inertia() } -// TODO: add test // GetL1RewardRate gets the L1 pricer reward rate func (con ArbGasInfo) GetL1RewardRate(c ctx, evm mech) (uint64, error) { return c.State.L1PricingState().PerUnitReward() } -// TODO: add test // GetL1RewardRecipient gets the L1 pricer reward recipient func (con ArbGasInfo) GetL1RewardRecipient(c ctx, evm mech) (common.Address, error) { return c.State.L1PricingState().PayRewardsTo() @@ -219,13 +213,11 @@ func (con ArbGasInfo) GetGasBacklog(c ctx, evm mech) (uint64, error) { return c.State.L2PricingState().GasBacklog() } -// TODO: add test // GetPricingInertia gets how slowly ArbOS updates the L2 basefee in response to backlogged gas func (con ArbGasInfo) GetPricingInertia(c ctx, evm mech) (uint64, error) { return c.State.L2PricingState().PricingInertia() } -// TODO: add test // GetGasBacklogTolerance gets the forgivable amount of backlogged gas ArbOS will ignore when raising the basefee func (con ArbGasInfo) GetGasBacklogTolerance(c ctx, evm mech) (uint64, error) { return c.State.L2PricingState().BacklogTolerance() @@ -240,7 +232,6 @@ func (con ArbGasInfo) GetL1PricingSurplus(c ctx, evm mech) (*big.Int, error) { return ps.GetL1PricingSurplus() } -// TODO: add test func (con ArbGasInfo) _preversion10_GetL1PricingSurplus(c ctx, evm mech) (*big.Int, error) { ps := c.State.L1PricingState() fundsDueForRefunds, err := ps.BatchPosterTable().TotalFundsDue() @@ -256,7 +247,6 @@ func (con ArbGasInfo) _preversion10_GetL1PricingSurplus(c ctx, evm mech) (*big.I return arbmath.BigSub(haveFunds.ToBig(), needFunds), nil } -// TODO: add test // GetPerBatchGasCharge gets the base charge (in L1 gas) attributed to each data batch in the calldata pricer func (con ArbGasInfo) GetPerBatchGasCharge(c ctx, evm mech) (int64, error) { return c.State.L1PricingState().PerBatchGasCost() @@ -272,7 +262,6 @@ func (con ArbGasInfo) GetL1FeesAvailable(c ctx, evm mech) (huge, error) { return c.State.L1PricingState().L1FeesAvailable() } -// TODO: add test // GetL1PricingEquilibrationUnits gets the equilibration units parameter for L1 price adjustment algorithm func (con ArbGasInfo) GetL1PricingEquilibrationUnits(c ctx, evm mech) (*big.Int, error) { return c.State.L1PricingState().EquilibrationUnits() diff --git a/precompiles/ArbOwner.go b/precompiles/ArbOwner.go index 040ce7054b..52096c1f26 100644 --- a/precompiles/ArbOwner.go +++ b/precompiles/ArbOwner.go @@ -1,4 +1,3 @@ -// TODO: check // Copyright 2021-2024, Offchain Labs, Inc. // For license information, see https://github.com/nitro/blob/master/LICENSE @@ -63,6 +62,7 @@ func (con ArbOwner) SetL1BaseFeeEstimateInertia(c ctx, evm mech, inertia uint64) return c.State.L1PricingState().SetInertia(inertia) } +// TODO: add test // SetL2BaseFee sets the L2 gas price directly, bypassing the pool calculus func (con ArbOwner) SetL2BaseFee(c ctx, evm mech, priceInWei huge) error { return c.State.L2PricingState().SetBaseFeeWei(priceInWei) @@ -76,6 +76,7 @@ func (con ArbOwner) SetMinimumL2BaseFee(c ctx, evm mech, priceInWei huge) error return c.State.L2PricingState().SetMinBaseFeeWei(priceInWei) } +// TODO: add test // SetSpeedLimit sets the computational speed limit for the chain func (con ArbOwner) SetSpeedLimit(c ctx, evm mech, limit uint64) error { return c.State.L2PricingState().SetSpeedLimitPerSecond(limit) @@ -96,6 +97,7 @@ func (con ArbOwner) SetL2GasBacklogTolerance(c ctx, evm mech, sec uint64) error return c.State.L2PricingState().SetBacklogTolerance(sec) } +// TODO: add test // GetNetworkFeeAccount gets the network fee collector func (con ArbOwner) GetNetworkFeeAccount(c ctx, evm mech) (addr, error) { return c.State.NetworkFeeAccount() @@ -125,6 +127,7 @@ func (con ArbOwner) SetL1PricingEquilibrationUnits(c ctx, evm mech, equilibratio return c.State.L1PricingState().SetEquilibrationUnits(equilibrationUnits) } +// TODO: add test func (con ArbOwner) SetL1PricingInertia(c ctx, evm mech, inertia uint64) error { return c.State.L1PricingState().SetInertia(inertia) } diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index 0b63d72e4d..9f47013133 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -11,6 +11,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" "github.com/offchainlabs/nitro/arbos" "github.com/offchainlabs/nitro/solgen/go/mocksgen" @@ -140,6 +141,106 @@ func TestPrecompileErrorGasLeft(t *testing.T) { assertNotAllGasConsumed(common.HexToAddress("0xff"), arbDebug.Methods["legacyError"].ID) } +func TestArbGasInfoAndArbOwner(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, false) + cleanup := builder.Build(t) + defer cleanup() + + auth := builder.L2Info.GetDefaultTransactOpts("Owner", ctx) + + arbOwner, err := precompilesgen.NewArbOwner(common.HexToAddress("0x70"), builder.L2.Client) + Require(t, err) + arbGasInfo, err := precompilesgen.NewArbGasInfo(common.HexToAddress("0x6c"), builder.L2.Client) + Require(t, err) + + // GetL1BaseFeeEstimateInertia test + inertia := uint64(11) + tx, err := arbOwner.SetL1BaseFeeEstimateInertia(&auth, inertia) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + arbGasInfoInertia, err := arbGasInfo.GetL1BaseFeeEstimateInertia(&bind.CallOpts{Context: ctx}) + Require(t, err) + if arbGasInfoInertia != inertia { + Fatal(t, "expected inertia to be", inertia, "got", arbGasInfoInertia) + } + + // GetL1RewardRate test + perUnitReward := uint64(11) + tx, err = arbOwner.SetL1PricingRewardRate(&auth, perUnitReward) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + arbGasInfoPerUnitReward, err := arbGasInfo.GetL1RewardRate(&bind.CallOpts{Context: ctx}) + Require(t, err) + if arbGasInfoPerUnitReward != perUnitReward { + Fatal(t, "expected per unit reward to be", perUnitReward, "got", arbGasInfoPerUnitReward) + } + + // GetL1RewardRecipient test + rewardRecipient := common.BytesToAddress(crypto.Keccak256([]byte{})[:20]) + tx, err = arbOwner.SetL1PricingRewardRecipient(&auth, rewardRecipient) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + arbGasInfoRewardRecipient, err := arbGasInfo.GetL1RewardRecipient(&bind.CallOpts{Context: ctx}) + Require(t, err) + if arbGasInfoRewardRecipient.Cmp(rewardRecipient) != 0 { + Fatal(t, "expected reward recipient to be", rewardRecipient, "got", arbGasInfoRewardRecipient) + } + + // GetPricingInertia + inertia = uint64(11) + tx, err = arbOwner.SetL2GasPricingInertia(&auth, inertia) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + arbGasInfoInertia, err = arbGasInfo.GetPricingInertia(&bind.CallOpts{Context: ctx}) + Require(t, err) + if arbGasInfoInertia != inertia { + Fatal(t, "expected inertia to be", inertia, "got", arbGasInfoInertia) + } + + // GetGasBacklogTolerance + gasTolerance := uint64(11) + tx, err = arbOwner.SetL2GasBacklogTolerance(&auth, inertia) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + arbGasInfoGasTolerance, err := arbGasInfo.GetGasBacklogTolerance(&bind.CallOpts{Context: ctx}) + Require(t, err) + if arbGasInfoGasTolerance != gasTolerance { + Fatal(t, "expected gas tolerance to be", gasTolerance, "got", arbGasInfoGasTolerance) + } + + // GetPerBatchGasCharge + perBatchGasCharge := int64(11) + tx, err = arbOwner.SetPerBatchGasCharge(&auth, perBatchGasCharge) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + arbGasInfoPerBatchGasCharge, err := arbGasInfo.GetPerBatchGasCharge(&bind.CallOpts{Context: ctx}) + Require(t, err) + if arbGasInfoPerBatchGasCharge != perBatchGasCharge { + Fatal(t, "expected per batch gas charge to be", perBatchGasCharge, "got", arbGasInfoPerBatchGasCharge) + } + + // GetL1PricingEquilibrationUnits + equilUnits := big.NewInt(11) + tx, err = arbOwner.SetL1PricingEquilibrationUnits(&auth, equilUnits) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + arbGasInfoEquilUnits, err := arbGasInfo.GetL1PricingEquilibrationUnits(&bind.CallOpts{Context: ctx}) + Require(t, err) + if arbGasInfoEquilUnits.Cmp(equilUnits) != 0 { + Fatal(t, "expected equilibration units to be", equilUnits, "got", arbGasInfoEquilUnits) + } +} + func TestGetBrotliCompressionLevel(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() From 7ce998e2cb1bdd3a1dd61cea15c2714c18bfda23 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 26 Sep 2024 10:09:59 -0300 Subject: [PATCH 405/489] Test for GetNetworkFeeAccount --- precompiles/ArbGasInfo.go | 1 - precompiles/ArbOwner.go | 1 - precompiles/ArbOwner_test.go | 8 ++++++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/precompiles/ArbGasInfo.go b/precompiles/ArbGasInfo.go index cd3ca6458d..133e1cb6e1 100644 --- a/precompiles/ArbGasInfo.go +++ b/precompiles/ArbGasInfo.go @@ -195,7 +195,6 @@ func (con ArbGasInfo) GetL1RewardRecipient(c ctx, evm mech) (common.Address, err return c.State.L1PricingState().PayRewardsTo() } -// TODO: add test // GetL1GasPriceEstimate gets the current estimate of the L1 basefee func (con ArbGasInfo) GetL1GasPriceEstimate(c ctx, evm mech) (huge, error) { return con.GetL1BaseFeeEstimate(c, evm) diff --git a/precompiles/ArbOwner.go b/precompiles/ArbOwner.go index 52096c1f26..98c874fc17 100644 --- a/precompiles/ArbOwner.go +++ b/precompiles/ArbOwner.go @@ -97,7 +97,6 @@ func (con ArbOwner) SetL2GasBacklogTolerance(c ctx, evm mech, sec uint64) error return c.State.L2PricingState().SetBacklogTolerance(sec) } -// TODO: add test // GetNetworkFeeAccount gets the network fee collector func (con ArbOwner) GetNetworkFeeAccount(c ctx, evm mech) (addr, error) { return c.State.NetworkFeeAccount() diff --git a/precompiles/ArbOwner_test.go b/precompiles/ArbOwner_test.go index 1f8c7ae4cd..b2ee1b5029 100644 --- a/precompiles/ArbOwner_test.go +++ b/precompiles/ArbOwner_test.go @@ -151,6 +151,14 @@ func TestArbOwner(t *testing.T) { if avail.Cmp(deposited) != 0 { Fail(t, avail, deposited) } + + err = prec.SetNetworkFeeAccount(callCtx, evm, addr1) + Require(t, err) + retrievedNetworkFeeAccount, err := prec.GetNetworkFeeAccount(callCtx, evm) + Require(t, err) + if retrievedNetworkFeeAccount != addr1 { + Fail(t, "Expected", addr1, "got", retrievedNetworkFeeAccount) + } } func TestArbOwnerSetChainConfig(t *testing.T) { From 136ad43d099a3559d7c962884ee037900af7abc7 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 26 Sep 2024 10:13:17 -0300 Subject: [PATCH 406/489] Test for SetL1PricingInertia --- precompiles/ArbOwner.go | 1 - system_tests/precompile_test.go | 12 ++++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/precompiles/ArbOwner.go b/precompiles/ArbOwner.go index 98c874fc17..b98a4e31e7 100644 --- a/precompiles/ArbOwner.go +++ b/precompiles/ArbOwner.go @@ -126,7 +126,6 @@ func (con ArbOwner) SetL1PricingEquilibrationUnits(c ctx, evm mech, equilibratio return c.State.L1PricingState().SetEquilibrationUnits(equilibrationUnits) } -// TODO: add test func (con ArbOwner) SetL1PricingInertia(c ctx, evm mech, inertia uint64) error { return c.State.L1PricingState().SetInertia(inertia) } diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index 9f47013133..ecd3125c6a 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -168,6 +168,18 @@ func TestArbGasInfoAndArbOwner(t *testing.T) { Fatal(t, "expected inertia to be", inertia, "got", arbGasInfoInertia) } + // GetL1BaseFeeEstimateInertia test, but using a different setter from ArbOwner + inertia = uint64(11) + tx, err = arbOwner.SetL1PricingInertia(&auth, inertia) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + arbGasInfoInertia, err = arbGasInfo.GetL1BaseFeeEstimateInertia(&bind.CallOpts{Context: ctx}) + Require(t, err) + if arbGasInfoInertia != inertia { + Fatal(t, "expected inertia to be", inertia, "got", arbGasInfoInertia) + } + // GetL1RewardRate test perUnitReward := uint64(11) tx, err = arbOwner.SetL1PricingRewardRate(&auth, perUnitReward) From fd75c88bf6e581198aca525fe2e7fc538a0af3b5 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 26 Sep 2024 10:40:45 -0300 Subject: [PATCH 407/489] Test for GetGasAccountingParams. Fixes GetGasBacklogTolerance test --- precompiles/ArbGasInfo.go | 1 - precompiles/ArbOwner.go | 1 - system_tests/precompile_test.go | 37 ++++++++++++++++++++++++++------- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/precompiles/ArbGasInfo.go b/precompiles/ArbGasInfo.go index 133e1cb6e1..48d95f0a91 100644 --- a/precompiles/ArbGasInfo.go +++ b/precompiles/ArbGasInfo.go @@ -161,7 +161,6 @@ func (con ArbGasInfo) GetPricesInArbGas(c ctx, evm mech) (huge, huge, huge, erro return con.GetPricesInArbGasWithAggregator(c, evm, addr{}) } -// TODO: add test // GetGasAccountingParams gets the rollup's speed limit, pool size, and tx gas limit func (con ArbGasInfo) GetGasAccountingParams(c ctx, evm mech) (huge, huge, huge, error) { l2pricing := c.State.L2PricingState() diff --git a/precompiles/ArbOwner.go b/precompiles/ArbOwner.go index b98a4e31e7..5310254ead 100644 --- a/precompiles/ArbOwner.go +++ b/precompiles/ArbOwner.go @@ -76,7 +76,6 @@ func (con ArbOwner) SetMinimumL2BaseFee(c ctx, evm mech, priceInWei huge) error return c.State.L2PricingState().SetMinBaseFeeWei(priceInWei) } -// TODO: add test // SetSpeedLimit sets the computational speed limit for the chain func (con ArbOwner) SetSpeedLimit(c ctx, evm mech, limit uint64) error { return c.State.L2PricingState().SetSpeedLimitPerSecond(limit) diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index ecd3125c6a..0d8a14f9b1 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -169,7 +169,7 @@ func TestArbGasInfoAndArbOwner(t *testing.T) { } // GetL1BaseFeeEstimateInertia test, but using a different setter from ArbOwner - inertia = uint64(11) + inertia = uint64(12) tx, err = arbOwner.SetL1PricingInertia(&auth, inertia) Require(t, err) _, err = builder.L2.EnsureTxSucceeded(tx) @@ -181,7 +181,7 @@ func TestArbGasInfoAndArbOwner(t *testing.T) { } // GetL1RewardRate test - perUnitReward := uint64(11) + perUnitReward := uint64(13) tx, err = arbOwner.SetL1PricingRewardRate(&auth, perUnitReward) Require(t, err) _, err = builder.L2.EnsureTxSucceeded(tx) @@ -205,7 +205,7 @@ func TestArbGasInfoAndArbOwner(t *testing.T) { } // GetPricingInertia - inertia = uint64(11) + inertia = uint64(14) tx, err = arbOwner.SetL2GasPricingInertia(&auth, inertia) Require(t, err) _, err = builder.L2.EnsureTxSucceeded(tx) @@ -217,8 +217,8 @@ func TestArbGasInfoAndArbOwner(t *testing.T) { } // GetGasBacklogTolerance - gasTolerance := uint64(11) - tx, err = arbOwner.SetL2GasBacklogTolerance(&auth, inertia) + gasTolerance := uint64(15) + tx, err = arbOwner.SetL2GasBacklogTolerance(&auth, gasTolerance) Require(t, err) _, err = builder.L2.EnsureTxSucceeded(tx) Require(t, err) @@ -229,7 +229,7 @@ func TestArbGasInfoAndArbOwner(t *testing.T) { } // GetPerBatchGasCharge - perBatchGasCharge := int64(11) + perBatchGasCharge := int64(16) tx, err = arbOwner.SetPerBatchGasCharge(&auth, perBatchGasCharge) Require(t, err) _, err = builder.L2.EnsureTxSucceeded(tx) @@ -241,7 +241,7 @@ func TestArbGasInfoAndArbOwner(t *testing.T) { } // GetL1PricingEquilibrationUnits - equilUnits := big.NewInt(11) + equilUnits := big.NewInt(17) tx, err = arbOwner.SetL1PricingEquilibrationUnits(&auth, equilUnits) Require(t, err) _, err = builder.L2.EnsureTxSucceeded(tx) @@ -251,6 +251,29 @@ func TestArbGasInfoAndArbOwner(t *testing.T) { if arbGasInfoEquilUnits.Cmp(equilUnits) != 0 { Fatal(t, "expected equilibration units to be", equilUnits, "got", arbGasInfoEquilUnits) } + + // GetGasAccountingParams + speedLimit := uint64(18) + txGasLimit := uint64(19) + tx, err = arbOwner.SetSpeedLimit(&auth, speedLimit) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + tx, err = arbOwner.SetMaxTxGasLimit(&auth, txGasLimit) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + arbGasInfoSpeedLimit, arbGasInfoPoolSize, arbGasInfoTxGasLimit, err := arbGasInfo.GetGasAccountingParams(&bind.CallOpts{Context: ctx}) + Require(t, err) + if arbGasInfoSpeedLimit.Cmp(big.NewInt(int64(speedLimit))) != 0 { + Fatal(t, "expected speed limit to be", speedLimit, "got", arbGasInfoSpeedLimit) + } + if arbGasInfoPoolSize.Cmp(big.NewInt(int64(txGasLimit))) != 0 { + Fatal(t, "expected pool size to be", txGasLimit, "got", arbGasInfoPoolSize) + } + if arbGasInfoTxGasLimit.Cmp(big.NewInt(int64(txGasLimit))) != 0 { + Fatal(t, "expected tx gas limit to be", txGasLimit, "got", arbGasInfoTxGasLimit) + } } func TestGetBrotliCompressionLevel(t *testing.T) { From 99e989bd2b118fa0ab9a028f824ac7d1f4189453 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 26 Sep 2024 10:51:43 -0300 Subject: [PATCH 408/489] Test for ArbOwner.SetL2BaseFee --- precompiles/ArbOwner.go | 1 - precompiles/ArbOwner_test.go | 10 ++++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/precompiles/ArbOwner.go b/precompiles/ArbOwner.go index 5310254ead..8b87445e0e 100644 --- a/precompiles/ArbOwner.go +++ b/precompiles/ArbOwner.go @@ -62,7 +62,6 @@ func (con ArbOwner) SetL1BaseFeeEstimateInertia(c ctx, evm mech, inertia uint64) return c.State.L1PricingState().SetInertia(inertia) } -// TODO: add test // SetL2BaseFee sets the L2 gas price directly, bypassing the pool calculus func (con ArbOwner) SetL2BaseFee(c ctx, evm mech, priceInWei huge) error { return c.State.L2PricingState().SetBaseFeeWei(priceInWei) diff --git a/precompiles/ArbOwner_test.go b/precompiles/ArbOwner_test.go index b2ee1b5029..2a2005b705 100644 --- a/precompiles/ArbOwner_test.go +++ b/precompiles/ArbOwner_test.go @@ -159,6 +159,16 @@ func TestArbOwner(t *testing.T) { if retrievedNetworkFeeAccount != addr1 { Fail(t, "Expected", addr1, "got", retrievedNetworkFeeAccount) } + + l2BaseFee := big.NewInt(123) + err = prec.SetL2BaseFee(callCtx, evm, l2BaseFee) + Require(t, err) + retrievedL2BaseFee, err := state.L2PricingState().BaseFeeWei() + Require(t, err) + if l2BaseFee.Cmp(retrievedL2BaseFee) != 0 { + Fail(t, "Expected", l2BaseFee, "got", retrievedL2BaseFee) + } + } func TestArbOwnerSetChainConfig(t *testing.T) { From 13092f7093a752d3b111d29fc3ee5609e9ca9c62 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 26 Sep 2024 11:10:17 -0300 Subject: [PATCH 409/489] More ArbGasInfo tests --- precompiles/ArbGasInfo.go | 5 --- precompiles/ArbGasInfo_test.go | 81 ++++++++++++++++++++++++++++++++++ precompiles/ArbOwner_test.go | 1 - 3 files changed, 81 insertions(+), 6 deletions(-) create mode 100644 precompiles/ArbGasInfo_test.go diff --git a/precompiles/ArbGasInfo.go b/precompiles/ArbGasInfo.go index 48d95f0a91..c476f9ec94 100644 --- a/precompiles/ArbGasInfo.go +++ b/precompiles/ArbGasInfo.go @@ -205,7 +205,6 @@ func (con ArbGasInfo) GetCurrentTxL1GasFees(c ctx, evm mech) (huge, error) { return c.txProcessor.PosterFee, nil } -// TODO: add test // GetGasBacklog gets the backlogged amount of gas burnt in excess of the speed limit func (con ArbGasInfo) GetGasBacklog(c ctx, evm mech) (uint64, error) { return c.State.L2PricingState().GasBacklog() @@ -265,25 +264,21 @@ func (con ArbGasInfo) GetL1PricingEquilibrationUnits(c ctx, evm mech) (*big.Int, return c.State.L1PricingState().EquilibrationUnits() } -// TODO: add test // GetLastL1PricingUpdateTime gets the last time the L1 calldata pricer was updated func (con ArbGasInfo) GetLastL1PricingUpdateTime(c ctx, evm mech) (uint64, error) { return c.State.L1PricingState().LastUpdateTime() } -// TODO: add test // GetL1PricingFundsDueForRewards gets the amount of L1 calldata payments due for rewards (per the L1 reward rate) func (con ArbGasInfo) GetL1PricingFundsDueForRewards(c ctx, evm mech) (*big.Int, error) { return c.State.L1PricingState().FundsDueForRewards() } -// TODO: add test // GetL1PricingUnitsSinceUpdate gets the amount of L1 calldata posted since the last update func (con ArbGasInfo) GetL1PricingUnitsSinceUpdate(c ctx, evm mech) (uint64, error) { return c.State.L1PricingState().UnitsSinceUpdate() } -// TODO: add test // GetLastL1PricingSurplus gets the L1 pricing surplus as of the last update (may be negative) func (con ArbGasInfo) GetLastL1PricingSurplus(c ctx, evm mech) (*big.Int, error) { return c.State.L1PricingState().LastSurplus() diff --git a/precompiles/ArbGasInfo_test.go b/precompiles/ArbGasInfo_test.go new file mode 100644 index 0000000000..dc893abe61 --- /dev/null +++ b/precompiles/ArbGasInfo_test.go @@ -0,0 +1,81 @@ +// Copyright 2021-2024, Offchain Labs, Inc. +// For license information, see https://github.com/nitro/blob/master/LICENSE + +package precompiles + +import ( + "math/big" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/params" + "github.com/offchainlabs/nitro/arbos/arbosState" + "github.com/offchainlabs/nitro/arbos/burn" + "github.com/offchainlabs/nitro/arbos/util" + "github.com/offchainlabs/nitro/util/testhelpers" +) + +func TestArbGasInfo(t *testing.T) { + t.Parallel() + + evm := newMockEVMForTesting() + caller := common.BytesToAddress(crypto.Keccak256([]byte{})[:20]) + tracer := util.NewTracingInfo(evm, testhelpers.RandomAddress(), types.ArbosAddress, util.TracingDuringEVM) + state, err := arbosState.OpenArbosState(evm.StateDB, burn.NewSystemBurner(tracer, false)) + Require(t, err) + + arbGasInfo := &ArbGasInfo{} + callCtx := testContext(caller, evm) + + // GetGasBacklog test + backlog := uint64(1000) + err = state.L2PricingState().SetGasBacklog(backlog) + Require(t, err) + retrievedBacklog, err := arbGasInfo.GetGasBacklog(callCtx, evm) + Require(t, err) + if retrievedBacklog != backlog { + t.Fatal("expected backlog to be", backlog, "but got", retrievedBacklog) + } + + // GetLastL1PricingUpdateTime test + lastUpdateTime := uint64(1001) + err = state.L1PricingState().SetLastUpdateTime(lastUpdateTime) + Require(t, err) + retrievedLastUpdateTime, err := arbGasInfo.GetLastL1PricingUpdateTime(callCtx, evm) + Require(t, err) + if retrievedLastUpdateTime != lastUpdateTime { + t.Fatal("expected last update time to be", lastUpdateTime, "but got", retrievedLastUpdateTime) + } + + // GetL1PricingFundsDueForRewards test + fundsDueForRewards := big.NewInt(1002) + err = state.L1PricingState().SetFundsDueForRewards(fundsDueForRewards) + Require(t, err) + retrievedFundsDueForRewards, err := arbGasInfo.GetL1PricingFundsDueForRewards(callCtx, evm) + Require(t, err) + if retrievedFundsDueForRewards.Cmp(fundsDueForRewards) != 0 { + t.Fatal("expected funds due for rewards to be", fundsDueForRewards, "but got", retrievedFundsDueForRewards) + } + + // GetL1PricingUnitsSinceUpdate test + pricingUnitsSinceUpdate := uint64(1003) + err = state.L1PricingState().SetUnitsSinceUpdate(pricingUnitsSinceUpdate) + Require(t, err) + retrievedPricingUnitsSinceUpdate, err := arbGasInfo.GetL1PricingUnitsSinceUpdate(callCtx, evm) + Require(t, err) + if retrievedPricingUnitsSinceUpdate != pricingUnitsSinceUpdate { + t.Fatal("expected pricing units since update to be", pricingUnitsSinceUpdate, "but got", retrievedPricingUnitsSinceUpdate) + } + + // GetLastL1PricingSurplus test + lastSurplus := big.NewInt(1004) + err = state.L1PricingState().SetLastSurplus(lastSurplus, params.ArbosVersion_Stylus) + Require(t, err) + retrievedLastSurplus, err := arbGasInfo.GetLastL1PricingSurplus(callCtx, evm) + Require(t, err) + if retrievedLastSurplus.Cmp(lastSurplus) != 0 { + t.Fatal("expected last surplus to be", lastSurplus, "but got", retrievedLastSurplus) + } +} diff --git a/precompiles/ArbOwner_test.go b/precompiles/ArbOwner_test.go index 2a2005b705..6d3256ca65 100644 --- a/precompiles/ArbOwner_test.go +++ b/precompiles/ArbOwner_test.go @@ -168,7 +168,6 @@ func TestArbOwner(t *testing.T) { if l2BaseFee.Cmp(retrievedL2BaseFee) != 0 { Fail(t, "Expected", l2BaseFee, "got", retrievedL2BaseFee) } - } func TestArbOwnerSetChainConfig(t *testing.T) { From 5fbfd672abf01825d56f5dd43f1309c90cfa542c Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 26 Sep 2024 11:39:15 -0300 Subject: [PATCH 410/489] ArbRetryableTx.GetBeneficiary test --- precompiles/ArbRetryableTx.go | 1 - precompiles/ArbRetryableTx_test.go | 19 ++++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/precompiles/ArbRetryableTx.go b/precompiles/ArbRetryableTx.go index 3410754b63..94b7172516 100644 --- a/precompiles/ArbRetryableTx.go +++ b/precompiles/ArbRetryableTx.go @@ -182,7 +182,6 @@ func (con ArbRetryableTx) Keepalive(c ctx, evm mech, ticketId bytes32) (huge, er return bigNewTimeout, err } -// TODO: add test // GetBeneficiary gets the beneficiary of the ticket func (con ArbRetryableTx) GetBeneficiary(c ctx, evm mech, ticketId bytes32) (addr, error) { retryableState := c.State.RetryableState() diff --git a/precompiles/ArbRetryableTx_test.go b/precompiles/ArbRetryableTx_test.go index af52926e75..03433dab15 100644 --- a/precompiles/ArbRetryableTx_test.go +++ b/precompiles/ArbRetryableTx_test.go @@ -26,7 +26,7 @@ func TestGetLifetime(t *testing.T) { } } -func TestRetryableRedeem(t *testing.T) { +func TestRetryableRedeemAndGetBeneficiary(t *testing.T) { evm := newMockEVMForTesting() precompileCtx := testContext(common.Address{}, evm) @@ -75,4 +75,21 @@ func TestRetryableRedeem(t *testing.T) { // to handle both cases, and some will be left over in this test's use case. Fail(t, "didn't consume all the expected gas") } + + getBeneficiaryCallData, err := retryABI.Pack("getBeneficiary", id) + Require(t, err) + retrievedBeneficiary, _, err := Precompiles()[retryAddress].Call( + getBeneficiaryCallData, + retryAddress, + retryAddress, + common.Address{}, + big.NewInt(0), + false, + 1000000, + evm, + ) + Require(t, err) + if common.BytesToAddress(retrievedBeneficiary).Cmp(beneficiary) != 0 { + Fail(t, "expected beneficiary to be ", beneficiary, " but got ", common.BytesToAddress(retrievedBeneficiary)) + } } From e8734333fe1284ab4975fe64d0f1a2537b4398c9 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 26 Sep 2024 13:31:11 -0300 Subject: [PATCH 411/489] GetPricesInArbGas test --- precompiles/ArbGasInfo.go | 2 -- precompiles/ArbGasInfo_test.go | 18 ++++++++++++++++++ precompiles/ArbRetryableTx.go | 1 - 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/precompiles/ArbGasInfo.go b/precompiles/ArbGasInfo.go index c476f9ec94..b18945447d 100644 --- a/precompiles/ArbGasInfo.go +++ b/precompiles/ArbGasInfo.go @@ -102,7 +102,6 @@ func (con ArbGasInfo) GetPricesInWei(c ctx, evm mech) (huge, huge, huge, huge, h return con.GetPricesInWeiWithAggregator(c, evm, addr{}) } -// TODO: add test // GetPricesInArbGasWithAggregator gets prices in ArbGas when using the provided aggregator func (con ArbGasInfo) GetPricesInArbGasWithAggregator(c ctx, evm mech, aggregator addr) (huge, huge, huge, error) { if c.State.ArbOSVersion() < 4 { @@ -155,7 +154,6 @@ func (con ArbGasInfo) _preVersion4_GetPricesInArbGasWithAggregator(c ctx, evm me return perL2Tx, gasForL1Calldata, storageArbGas, nil } -// TODO: add test // GetPricesInArbGas gets prices in ArbGas when using the caller's preferred aggregator func (con ArbGasInfo) GetPricesInArbGas(c ctx, evm mech) (huge, huge, huge, error) { return con.GetPricesInArbGasWithAggregator(c, evm, addr{}) diff --git a/precompiles/ArbGasInfo_test.go b/precompiles/ArbGasInfo_test.go index dc893abe61..72bc13d71e 100644 --- a/precompiles/ArbGasInfo_test.go +++ b/precompiles/ArbGasInfo_test.go @@ -13,6 +13,7 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/offchainlabs/nitro/arbos/arbosState" "github.com/offchainlabs/nitro/arbos/burn" + "github.com/offchainlabs/nitro/arbos/storage" "github.com/offchainlabs/nitro/arbos/util" "github.com/offchainlabs/nitro/util/testhelpers" ) @@ -78,4 +79,21 @@ func TestArbGasInfo(t *testing.T) { if retrievedLastSurplus.Cmp(lastSurplus) != 0 { t.Fatal("expected last surplus to be", lastSurplus, "but got", retrievedLastSurplus) } + + // GetPricesInArbGas test + evm.Context.BaseFee = big.NewInt(1005) + expectedGasPerL2Tx := big.NewInt(111442786069) + expectedGasForL1Calldata := big.NewInt(796019900) + expectedStorageArgGas := big.NewInt(int64(storage.StorageWriteCost)) + gasPerL2Tx, gasForL1Calldata, storageArgGas, err := arbGasInfo.GetPricesInArbGas(callCtx, evm) + Require(t, err) + if gasPerL2Tx.Cmp(expectedGasPerL2Tx) != 0 { + t.Fatal("expected gas per L2 tx to be", expectedGasPerL2Tx, "but got", gasPerL2Tx) + } + if gasForL1Calldata.Cmp(expectedGasForL1Calldata) != 0 { + t.Fatal("expected gas for L1 calldata to be", expectedGasForL1Calldata, "but got", gasForL1Calldata) + } + if storageArgGas.Cmp(expectedStorageArgGas) != 0 { + t.Fatal("expected storage arg gas to be", expectedStorageArgGas, "but got", storageArgGas) + } } diff --git a/precompiles/ArbRetryableTx.go b/precompiles/ArbRetryableTx.go index 94b7172516..ed7919c7dd 100644 --- a/precompiles/ArbRetryableTx.go +++ b/precompiles/ArbRetryableTx.go @@ -233,7 +233,6 @@ func (con ArbRetryableTx) GetCurrentRedeemer(c ctx, evm mech) (common.Address, e return common.Address{}, nil } -// TODO: add test func (con ArbRetryableTx) SubmitRetryable( c ctx, evm mech, requestId bytes32, l1BaseFee, deposit, callvalue, gasFeeCap huge, gasLimit uint64, maxSubmissionFee huge, From 4ce2fb88ef146c82275b3d53b3e411b2d7159220 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 27 Sep 2024 07:50:10 -0300 Subject: [PATCH 412/489] GetCurrentTxL1GasFees test --- precompiles/ArbGasInfo.go | 1 - system_tests/precompile_test.go | 9 +++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/precompiles/ArbGasInfo.go b/precompiles/ArbGasInfo.go index b18945447d..b41dfda8a2 100644 --- a/precompiles/ArbGasInfo.go +++ b/precompiles/ArbGasInfo.go @@ -197,7 +197,6 @@ func (con ArbGasInfo) GetL1GasPriceEstimate(c ctx, evm mech) (huge, error) { return con.GetL1BaseFeeEstimate(c, evm) } -// TODO: add test // GetCurrentTxL1GasFees gets the fee paid to the aggregator for posting this tx func (con ArbGasInfo) GetCurrentTxL1GasFees(c ctx, evm mech) (huge, error) { return c.txProcessor.PosterFee, nil diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index 0d8a14f9b1..780373a0cd 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -274,6 +274,15 @@ func TestArbGasInfoAndArbOwner(t *testing.T) { if arbGasInfoTxGasLimit.Cmp(big.NewInt(int64(txGasLimit))) != 0 { Fatal(t, "expected tx gas limit to be", txGasLimit, "got", arbGasInfoTxGasLimit) } + + currTxL1GasFees, err := arbGasInfo.GetCurrentTxL1GasFees(&bind.CallOpts{Context: ctx}) + Require(t, err) + if currTxL1GasFees == nil { + Fatal(t, "currTxL1GasFees is nil") + } + if currTxL1GasFees.Cmp(big.NewInt(0)) != 1 { + Fatal(t, "expected currTxL1GasFees to be greater than 0, got", currTxL1GasFees) + } } func TestGetBrotliCompressionLevel(t *testing.T) { From b2a42d75a5ead89ece84e71a745062a48369b54c Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 27 Sep 2024 11:51:37 -0300 Subject: [PATCH 413/489] GetCurrentRedeemer test --- precompiles/ArbRetryableTx.go | 1 - precompiles/ArbRetryableTx_test.go | 21 ++++++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/precompiles/ArbRetryableTx.go b/precompiles/ArbRetryableTx.go index ed7919c7dd..eab459af42 100644 --- a/precompiles/ArbRetryableTx.go +++ b/precompiles/ArbRetryableTx.go @@ -225,7 +225,6 @@ func (con ArbRetryableTx) Cancel(c ctx, evm mech, ticketId bytes32) error { return con.Canceled(c, evm, ticketId) } -// TODO: add test func (con ArbRetryableTx) GetCurrentRedeemer(c ctx, evm mech) (common.Address, error) { if c.txProcessor.CurrentRefundTo != nil { return *c.txProcessor.CurrentRefundTo, nil diff --git a/precompiles/ArbRetryableTx_test.go b/precompiles/ArbRetryableTx_test.go index 03433dab15..58eed5ad77 100644 --- a/precompiles/ArbRetryableTx_test.go +++ b/precompiles/ArbRetryableTx_test.go @@ -7,15 +7,28 @@ import ( "math/big" "testing" + "github.com/offchainlabs/nitro/arbos" "github.com/offchainlabs/nitro/arbos/retryables" "github.com/offchainlabs/nitro/arbos/storage" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/vm" templates "github.com/offchainlabs/nitro/solgen/go/precompilesgen" ) -func TestGetLifetime(t *testing.T) { +func newMockEVMForTestingWithCurrentRefundTo(currentRefundTo *common.Address) *vm.EVM { evm := newMockEVMForTesting() + txProcessor := arbos.NewTxProcessor(evm, &core.Message{}) + txProcessor.CurrentRefundTo = currentRefundTo + evm.ProcessingHook = txProcessor + return evm +} + +func TestGetLifetimeAndCurrentRedeemer(t *testing.T) { + currentRefundTo := common.HexToAddress("0x030405") + + evm := newMockEVMForTestingWithCurrentRefundTo(¤tRefundTo) retryableTx := ArbRetryableTx{} context := testContext(common.Address{}, evm) @@ -24,6 +37,12 @@ func TestGetLifetime(t *testing.T) { if lifetime.Cmp(big.NewInt(retryables.RetryableLifetimeSeconds)) != 0 { t.Fatal("Expected to be ", retryables.RetryableLifetimeSeconds, " but got ", lifetime) } + + currentRedeemer, err := retryableTx.GetCurrentRedeemer(context, evm) + Require(t, err) + if currentRefundTo.Cmp(currentRedeemer) != 0 { + t.Fatal("Expected to be ", currentRefundTo, " but got ", currentRedeemer) + } } func TestRetryableRedeemAndGetBeneficiary(t *testing.T) { From 98537b59cd7eeeb820127ba3851c2b0c0c05d104 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 27 Sep 2024 12:45:36 -0300 Subject: [PATCH 414/489] ArbRetryableTx.Cancel test --- precompiles/ArbRetryableTx.go | 1 - system_tests/retryable_test.go | 72 ++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/precompiles/ArbRetryableTx.go b/precompiles/ArbRetryableTx.go index eab459af42..b4696d5b78 100644 --- a/precompiles/ArbRetryableTx.go +++ b/precompiles/ArbRetryableTx.go @@ -195,7 +195,6 @@ func (con ArbRetryableTx) GetBeneficiary(c ctx, evm mech, ticketId bytes32) (add return retryable.Beneficiary() } -// TODO: add test // Cancel the ticket and refund its callvalue to its beneficiary func (con ArbRetryableTx) Cancel(c ctx, evm mech, ticketId bytes32) error { if c.txProcessor.CurrentRetryable != nil && ticketId == *c.txProcessor.CurrentRetryable { diff --git a/system_tests/retryable_test.go b/system_tests/retryable_test.go index aa9fbfd72e..b3038716d7 100644 --- a/system_tests/retryable_test.go +++ b/system_tests/retryable_test.go @@ -423,6 +423,78 @@ func TestSubmitRetryableFailThenRetry(t *testing.T) { } } +func TestCancelRetryable(t *testing.T) { + t.Parallel() + builder, delayedInbox, lookupL2Tx, ctx, teardown := retryableSetup(t) + defer teardown() + + ownerTxOpts := builder.L2Info.GetDefaultTransactOpts("Owner", ctx) + usertxopts := builder.L1Info.GetDefaultTransactOpts("Faucet", ctx) + usertxopts.Value = arbmath.BigMul(big.NewInt(1e12), big.NewInt(1e12)) + + simpleAddr, _ := builder.L2.DeploySimple(t, ownerTxOpts) + simpleABI, err := mocksgen.SimpleMetaData.GetAbi() + Require(t, err) + + beneficiaryAddress := builder.L2Info.GetAddress("Beneficiary") + l1tx, err := delayedInbox.CreateRetryableTicket( + &usertxopts, + simpleAddr, + common.Big0, + big.NewInt(1e16), + beneficiaryAddress, + beneficiaryAddress, + // send enough L2 gas for intrinsic but not compute + big.NewInt(int64(params.TxGas+params.TxDataNonZeroGasEIP2028*4)), + big.NewInt(l2pricing.InitialBaseFeeWei*2), + simpleABI.Methods["incrementRedeem"].ID, + ) + Require(t, err) + + l1Receipt, err := builder.L1.EnsureTxSucceeded(l1tx) + Require(t, err) + if l1Receipt.Status != types.ReceiptStatusSuccessful { + Fatal(t, "l1Receipt indicated failure") + } + + waitForL1DelayBlocks(t, builder) + + receipt, err := builder.L2.EnsureTxSucceeded(lookupL2Tx(l1Receipt)) + Require(t, err) + if len(receipt.Logs) != 2 { + Fatal(t, len(receipt.Logs)) + } + ticketId := receipt.Logs[0].Topics[1] + firstRetryTxId := receipt.Logs[1].Topics[2] + + // make sure it failed + receipt, err = WaitForTx(ctx, builder.L2.Client, firstRetryTxId, time.Second*5) + Require(t, err) + if receipt.Status != types.ReceiptStatusFailed { + Fatal(t, receipt.GasUsed) + } + + arbRetryableTx, err := precompilesgen.NewArbRetryableTx(common.HexToAddress("6e"), builder.L2.Client) + Require(t, err) + + // checks that the ticket exists + _, err = arbRetryableTx.GetTimeout(&bind.CallOpts{}, ticketId) + Require(t, err) + + // cancel the ticket + beneficiaryTxOpts := builder.L2Info.GetDefaultTransactOpts("Beneficiary", ctx) + tx, err := arbRetryableTx.Cancel(&beneficiaryTxOpts, ticketId) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + + // checks that the ticket no longer exists + _, err = arbRetryableTx.GetTimeout(&bind.CallOpts{}, ticketId) + if (err == nil) || (err.Error() != "execution reverted: error NoTicketWithID(): NoTicketWithID()") { + Fatal(t, "didn't get expected NoTicketWithID error") + } +} + func TestSubmissionGasCosts(t *testing.T) { t.Parallel() builder, delayedInbox, lookupL2Tx, ctx, teardown := retryableSetup(t) From 96bd892a1041c11676dd1bb3c07b9dbe4be41e88 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 27 Sep 2024 13:19:03 -0300 Subject: [PATCH 415/489] ArbRetryableTx.Keepalive test --- precompiles/ArbRetryableTx.go | 2 -- system_tests/retryable_test.go | 16 +++++++++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/precompiles/ArbRetryableTx.go b/precompiles/ArbRetryableTx.go index b4696d5b78..d925499180 100644 --- a/precompiles/ArbRetryableTx.go +++ b/precompiles/ArbRetryableTx.go @@ -152,10 +152,8 @@ func (con ArbRetryableTx) GetTimeout(c ctx, evm mech, ticketId bytes32) (huge, e return new(big.Int).SetUint64(timeout), nil } -// TODO: add test // Keepalive adds one lifetime period to the ticket's expiry func (con ArbRetryableTx) Keepalive(c ctx, evm mech, ticketId bytes32) (huge, error) { - // charge for the expiry update retryableState := c.State.RetryableState() nbytes, err := retryableState.RetryableSizeBytes(ticketId, evm.Context.Time) diff --git a/system_tests/retryable_test.go b/system_tests/retryable_test.go index b3038716d7..6651c8ca74 100644 --- a/system_tests/retryable_test.go +++ b/system_tests/retryable_test.go @@ -423,7 +423,7 @@ func TestSubmitRetryableFailThenRetry(t *testing.T) { } } -func TestCancelRetryable(t *testing.T) { +func TestKeepaliveAndCancelRetryable(t *testing.T) { t.Parallel() builder, delayedInbox, lookupL2Tx, ctx, teardown := retryableSetup(t) defer teardown() @@ -477,9 +477,19 @@ func TestCancelRetryable(t *testing.T) { arbRetryableTx, err := precompilesgen.NewArbRetryableTx(common.HexToAddress("6e"), builder.L2.Client) Require(t, err) - // checks that the ticket exists - _, err = arbRetryableTx.GetTimeout(&bind.CallOpts{}, ticketId) + // checks that the ticket exists and gets current timeout + timeoutBeforeKeepalive, err := arbRetryableTx.GetTimeout(&bind.CallOpts{}, ticketId) + Require(t, err) + // checks that keepalive increases the timeout as expected + _, err = arbRetryableTx.Keepalive(&ownerTxOpts, ticketId) Require(t, err) + timeoutAfterKeepalive, err := arbRetryableTx.GetTimeout(&bind.CallOpts{}, ticketId) + Require(t, err) + expectedTimeoutAfterKeepAlive := timeoutBeforeKeepalive + expectedTimeoutAfterKeepAlive.Add(expectedTimeoutAfterKeepAlive, big.NewInt(retryables.RetryableLifetimeSeconds)) + if timeoutAfterKeepalive.Cmp(expectedTimeoutAfterKeepAlive) != 0 { + Fatal(t, "expected timeout after keepalive to be", expectedTimeoutAfterKeepAlive, "but got", timeoutAfterKeepalive) + } // cancel the ticket beneficiaryTxOpts := builder.L2Info.GetDefaultTransactOpts("Beneficiary", ctx) From 7993502d0902bd58b2c975f95c20a0b20398b3c8 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 27 Sep 2024 14:46:58 -0300 Subject: [PATCH 416/489] Fixes address comparison --- precompiles/ArbOwner_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/precompiles/ArbOwner_test.go b/precompiles/ArbOwner_test.go index 6d3256ca65..1fc6e679cb 100644 --- a/precompiles/ArbOwner_test.go +++ b/precompiles/ArbOwner_test.go @@ -156,7 +156,7 @@ func TestArbOwner(t *testing.T) { Require(t, err) retrievedNetworkFeeAccount, err := prec.GetNetworkFeeAccount(callCtx, evm) Require(t, err) - if retrievedNetworkFeeAccount != addr1 { + if retrievedNetworkFeeAccount.Cmp(addr1) != 0 { Fail(t, "Expected", addr1, "got", retrievedNetworkFeeAccount) } From bdb8010611f0cba91080f52a36a9c1e18c24f10e Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 27 Sep 2024 14:49:13 -0300 Subject: [PATCH 417/489] Adds missing t.Parallel() --- precompiles/ArbFunctionTable_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/precompiles/ArbFunctionTable_test.go b/precompiles/ArbFunctionTable_test.go index 6a68bffd22..f3c6d97bbf 100644 --- a/precompiles/ArbFunctionTable_test.go +++ b/precompiles/ArbFunctionTable_test.go @@ -12,6 +12,8 @@ import ( ) func TestArbFunctionTable(t *testing.T) { + t.Parallel() + evm := newMockEVMForTesting() ftab := ArbFunctionTable{} context := testContext(common.Address{}, evm) From 9b78b42447ecefc7964b6bcdda926e0e77fa478b Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 27 Sep 2024 14:53:15 -0300 Subject: [PATCH 418/489] Fixes comments --- system_tests/program_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/system_tests/program_test.go b/system_tests/program_test.go index cda592ee06..dcdd7da60e 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -535,7 +535,7 @@ func testCalls(t *testing.T, jit bool) { defer cleanup() callsAddr := deployWasm(t, ctx, auth, l2client, rustFile("multicall")) - // checks ArbInfo.GetCode works properly + // checks that ArbInfo.GetCode works properly codeFromFile, _ := readWasmFile(t, rustFile("multicall")) arbInfo, err := pgen.NewArbInfo(types.ArbInfoAddress, l2client) Require(t, err) @@ -726,7 +726,7 @@ func testCalls(t *testing.T, jit bool) { Fatal(t, balance, value) } - // checks ArbInfo.GetBalance works properly + // checks that ArbInfo.GetBalance works properly balance, err = arbInfo.GetBalance(nil, eoa) Require(t, err) if !arbmath.BigEquals(balance, value) { From 2a9fe74f415e01a5d3056f08e846d69c58faf8b7 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 27 Sep 2024 15:02:35 -0300 Subject: [PATCH 419/489] Improves arbSys.ArbOSVersion test --- system_tests/precompile_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index 780373a0cd..d6382b89cd 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -38,10 +38,11 @@ func TestPurePrecompileMethodCalls(t *testing.T) { Fatal(t, "Wrong ChainID", chainId.Uint64()) } + expectedArbosVersion := 55 + arbosVersion // Nitro versions start at 56 arbSysArbosVersion, err := arbSys.ArbOSVersion(&bind.CallOpts{}) Require(t, err) - if arbSysArbosVersion.Uint64() != 55+arbosVersion { // Nitro versios start at 56 - Fatal(t, "Expected ArbOSVersion 86, got", arbosVersion) + if arbSysArbosVersion.Uint64() != expectedArbosVersion { + Fatal(t, "Expected ArbOS version", expectedArbosVersion, "got", arbSysArbosVersion) } storageGasAvailable, err := arbSys.GetStorageGasAvailable(&bind.CallOpts{}) From 8e8bece8bf5a9e994c397ae3ad780643dae32b8e Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 27 Sep 2024 15:04:10 -0300 Subject: [PATCH 420/489] Fixes comment --- system_tests/precompile_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index d6382b89cd..a8ec863ce0 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -169,7 +169,7 @@ func TestArbGasInfoAndArbOwner(t *testing.T) { Fatal(t, "expected inertia to be", inertia, "got", arbGasInfoInertia) } - // GetL1BaseFeeEstimateInertia test, but using a different setter from ArbOwner + // GetL1BaseFeeEstimateInertia test, but now using a different setter from ArbOwner inertia = uint64(12) tx, err = arbOwner.SetL1PricingInertia(&auth, inertia) Require(t, err) From cbf2dc046d5b3b503558534ec67cc33c04a4c74b Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 27 Sep 2024 15:06:21 -0300 Subject: [PATCH 421/489] Fixes comment --- system_tests/precompile_test.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index a8ec863ce0..88df3e0beb 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -205,7 +205,7 @@ func TestArbGasInfoAndArbOwner(t *testing.T) { Fatal(t, "expected reward recipient to be", rewardRecipient, "got", arbGasInfoRewardRecipient) } - // GetPricingInertia + // GetPricingInertia test inertia = uint64(14) tx, err = arbOwner.SetL2GasPricingInertia(&auth, inertia) Require(t, err) @@ -217,7 +217,7 @@ func TestArbGasInfoAndArbOwner(t *testing.T) { Fatal(t, "expected inertia to be", inertia, "got", arbGasInfoInertia) } - // GetGasBacklogTolerance + // GetGasBacklogTolerance test gasTolerance := uint64(15) tx, err = arbOwner.SetL2GasBacklogTolerance(&auth, gasTolerance) Require(t, err) @@ -229,7 +229,7 @@ func TestArbGasInfoAndArbOwner(t *testing.T) { Fatal(t, "expected gas tolerance to be", gasTolerance, "got", arbGasInfoGasTolerance) } - // GetPerBatchGasCharge + // GetPerBatchGasCharge test perBatchGasCharge := int64(16) tx, err = arbOwner.SetPerBatchGasCharge(&auth, perBatchGasCharge) Require(t, err) @@ -241,7 +241,7 @@ func TestArbGasInfoAndArbOwner(t *testing.T) { Fatal(t, "expected per batch gas charge to be", perBatchGasCharge, "got", arbGasInfoPerBatchGasCharge) } - // GetL1PricingEquilibrationUnits + // GetL1PricingEquilibrationUnits test equilUnits := big.NewInt(17) tx, err = arbOwner.SetL1PricingEquilibrationUnits(&auth, equilUnits) Require(t, err) @@ -253,7 +253,7 @@ func TestArbGasInfoAndArbOwner(t *testing.T) { Fatal(t, "expected equilibration units to be", equilUnits, "got", arbGasInfoEquilUnits) } - // GetGasAccountingParams + // GetGasAccountingParams test speedLimit := uint64(18) txGasLimit := uint64(19) tx, err = arbOwner.SetSpeedLimit(&auth, speedLimit) @@ -276,6 +276,7 @@ func TestArbGasInfoAndArbOwner(t *testing.T) { Fatal(t, "expected tx gas limit to be", txGasLimit, "got", arbGasInfoTxGasLimit) } + // GetCurrentTxL1GasFees test currTxL1GasFees, err := arbGasInfo.GetCurrentTxL1GasFees(&bind.CallOpts{Context: ctx}) Require(t, err) if currTxL1GasFees == nil { From e891a5444fb6f858151005bd514aa9454e06e950 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 1 Oct 2024 09:18:14 -0300 Subject: [PATCH 422/489] Separate long setter/getter test in multiple test functions --- precompiles/ArbGasInfo_test.go | 77 +++++++++++++---- system_tests/precompile_test.go | 142 ++++++++++++++++++++++++++------ 2 files changed, 175 insertions(+), 44 deletions(-) diff --git a/precompiles/ArbGasInfo_test.go b/precompiles/ArbGasInfo_test.go index 72bc13d71e..260d7b3cef 100644 --- a/precompiles/ArbGasInfo_test.go +++ b/precompiles/ArbGasInfo_test.go @@ -9,6 +9,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" "github.com/offchainlabs/nitro/arbos/arbosState" @@ -18,9 +19,14 @@ import ( "github.com/offchainlabs/nitro/util/testhelpers" ) -func TestArbGasInfo(t *testing.T) { - t.Parallel() - +func setupArbGasInfo( + t *testing.T, +) ( + *vm.EVM, + *arbosState.ArbosState, + *Context, + *ArbGasInfo, +) { evm := newMockEVMForTesting() caller := common.BytesToAddress(crypto.Keccak256([]byte{})[:20]) tracer := util.NewTracingInfo(evm, testhelpers.RandomAddress(), types.ArbosAddress, util.TracingDuringEVM) @@ -30,62 +36,97 @@ func TestArbGasInfo(t *testing.T) { arbGasInfo := &ArbGasInfo{} callCtx := testContext(caller, evm) - // GetGasBacklog test + return evm, state, callCtx, arbGasInfo +} + +func TestGetGasBacklog(t *testing.T) { + t.Parallel() + + evm, state, callCtx, arbGasInfo := setupArbGasInfo(t) + backlog := uint64(1000) - err = state.L2PricingState().SetGasBacklog(backlog) + err := state.L2PricingState().SetGasBacklog(backlog) Require(t, err) retrievedBacklog, err := arbGasInfo.GetGasBacklog(callCtx, evm) Require(t, err) if retrievedBacklog != backlog { t.Fatal("expected backlog to be", backlog, "but got", retrievedBacklog) } +} + +func TestGetL1PricingUpdateTime(t *testing.T) { + t.Parallel() + + evm, state, callCtx, arbGasInfo := setupArbGasInfo(t) - // GetLastL1PricingUpdateTime test lastUpdateTime := uint64(1001) - err = state.L1PricingState().SetLastUpdateTime(lastUpdateTime) + err := state.L1PricingState().SetLastUpdateTime(lastUpdateTime) Require(t, err) retrievedLastUpdateTime, err := arbGasInfo.GetLastL1PricingUpdateTime(callCtx, evm) Require(t, err) if retrievedLastUpdateTime != lastUpdateTime { t.Fatal("expected last update time to be", lastUpdateTime, "but got", retrievedLastUpdateTime) } +} + +func TestGetL1PricingFundsDueForRewards(t *testing.T) { + t.Parallel() + + evm, state, callCtx, arbGasInfo := setupArbGasInfo(t) - // GetL1PricingFundsDueForRewards test fundsDueForRewards := big.NewInt(1002) - err = state.L1PricingState().SetFundsDueForRewards(fundsDueForRewards) + err := state.L1PricingState().SetFundsDueForRewards(fundsDueForRewards) Require(t, err) retrievedFundsDueForRewards, err := arbGasInfo.GetL1PricingFundsDueForRewards(callCtx, evm) Require(t, err) if retrievedFundsDueForRewards.Cmp(fundsDueForRewards) != 0 { t.Fatal("expected funds due for rewards to be", fundsDueForRewards, "but got", retrievedFundsDueForRewards) } +} + +func TestGetL1PricingUnitsSinceUpdate(t *testing.T) { + t.Parallel() + + evm, state, callCtx, arbGasInfo := setupArbGasInfo(t) - // GetL1PricingUnitsSinceUpdate test pricingUnitsSinceUpdate := uint64(1003) - err = state.L1PricingState().SetUnitsSinceUpdate(pricingUnitsSinceUpdate) + err := state.L1PricingState().SetUnitsSinceUpdate(pricingUnitsSinceUpdate) Require(t, err) retrievedPricingUnitsSinceUpdate, err := arbGasInfo.GetL1PricingUnitsSinceUpdate(callCtx, evm) Require(t, err) if retrievedPricingUnitsSinceUpdate != pricingUnitsSinceUpdate { t.Fatal("expected pricing units since update to be", pricingUnitsSinceUpdate, "but got", retrievedPricingUnitsSinceUpdate) } +} + +func TestGetLastL1PricingSurplus(t *testing.T) { + t.Parallel() + + evm, state, callCtx, arbGasInfo := setupArbGasInfo(t) - // GetLastL1PricingSurplus test lastSurplus := big.NewInt(1004) - err = state.L1PricingState().SetLastSurplus(lastSurplus, params.ArbosVersion_Stylus) + err := state.L1PricingState().SetLastSurplus(lastSurplus, params.ArbosVersion_Stylus) Require(t, err) retrievedLastSurplus, err := arbGasInfo.GetLastL1PricingSurplus(callCtx, evm) Require(t, err) if retrievedLastSurplus.Cmp(lastSurplus) != 0 { t.Fatal("expected last surplus to be", lastSurplus, "but got", retrievedLastSurplus) } +} + +func TestGetPricesInArbGas(t *testing.T) { + t.Parallel() + + evm := newMockEVMForTesting() + caller := common.BytesToAddress(crypto.Keccak256([]byte{})[:20]) + arbGasInfo := &ArbGasInfo{} + callCtx := testContext(caller, evm) - // GetPricesInArbGas test evm.Context.BaseFee = big.NewInt(1005) expectedGasPerL2Tx := big.NewInt(111442786069) expectedGasForL1Calldata := big.NewInt(796019900) - expectedStorageArgGas := big.NewInt(int64(storage.StorageWriteCost)) - gasPerL2Tx, gasForL1Calldata, storageArgGas, err := arbGasInfo.GetPricesInArbGas(callCtx, evm) + expectedStorageArbGas := big.NewInt(int64(storage.StorageWriteCost)) + gasPerL2Tx, gasForL1Calldata, storageArbGas, err := arbGasInfo.GetPricesInArbGas(callCtx, evm) Require(t, err) if gasPerL2Tx.Cmp(expectedGasPerL2Tx) != 0 { t.Fatal("expected gas per L2 tx to be", expectedGasPerL2Tx, "but got", gasPerL2Tx) @@ -93,7 +134,7 @@ func TestArbGasInfo(t *testing.T) { if gasForL1Calldata.Cmp(expectedGasForL1Calldata) != 0 { t.Fatal("expected gas for L1 calldata to be", expectedGasForL1Calldata, "but got", gasForL1Calldata) } - if storageArgGas.Cmp(expectedStorageArgGas) != 0 { - t.Fatal("expected storage arg gas to be", expectedStorageArgGas, "but got", storageArgGas) + if storageArbGas.Cmp(expectedStorageArbGas) != 0 { + t.Fatal("expected storage arb gas to be", expectedStorageArbGas, "but got", storageArbGas) } } diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index 88df3e0beb..2b979f0dfd 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -142,13 +142,24 @@ func TestPrecompileErrorGasLeft(t *testing.T) { assertNotAllGasConsumed(common.HexToAddress("0xff"), arbDebug.Methods["legacyError"].ID) } -func TestArbGasInfoAndArbOwner(t *testing.T) { +func setupArbOwnerAndArbGasInfo( + t *testing.T, +) ( + *NodeBuilder, + func(), + bind.TransactOpts, + *precompilesgen.ArbOwner, + *precompilesgen.ArbGasInfo, +) { ctx, cancel := context.WithCancel(context.Background()) - defer cancel() builder := NewNodeBuilder(ctx).DefaultConfig(t, false) - cleanup := builder.Build(t) - defer cleanup() + builderCleanup := builder.Build(t) + + cleanup := func() { + builderCleanup() + cancel() + } auth := builder.L2Info.GetDefaultTransactOpts("Owner", ctx) @@ -157,7 +168,16 @@ func TestArbGasInfoAndArbOwner(t *testing.T) { arbGasInfo, err := precompilesgen.NewArbGasInfo(common.HexToAddress("0x6c"), builder.L2.Client) Require(t, err) - // GetL1BaseFeeEstimateInertia test + return builder, cleanup, auth, arbOwner, arbGasInfo +} + +func TestL1BaseFeeEstimateInertia(t *testing.T) { + t.Parallel() + + builder, cleanup, auth, arbOwner, arbGasInfo := setupArbOwnerAndArbGasInfo(t) + defer cleanup() + ctx := builder.ctx + inertia := uint64(11) tx, err := arbOwner.SetL1BaseFeeEstimateInertia(&auth, inertia) Require(t, err) @@ -168,22 +188,37 @@ func TestArbGasInfoAndArbOwner(t *testing.T) { if arbGasInfoInertia != inertia { Fatal(t, "expected inertia to be", inertia, "got", arbGasInfoInertia) } +} + +// Similar to TestL1BaseFeeEstimateInertia, but now using a different setter from ArbOwner +func TestL1PricingInertia(t *testing.T) { + t.Parallel() + + builder, cleanup, auth, arbOwner, arbGasInfo := setupArbOwnerAndArbGasInfo(t) + defer cleanup() + ctx := builder.ctx - // GetL1BaseFeeEstimateInertia test, but now using a different setter from ArbOwner - inertia = uint64(12) - tx, err = arbOwner.SetL1PricingInertia(&auth, inertia) + inertia := uint64(12) + tx, err := arbOwner.SetL1PricingInertia(&auth, inertia) Require(t, err) _, err = builder.L2.EnsureTxSucceeded(tx) Require(t, err) - arbGasInfoInertia, err = arbGasInfo.GetL1BaseFeeEstimateInertia(&bind.CallOpts{Context: ctx}) + arbGasInfoInertia, err := arbGasInfo.GetL1BaseFeeEstimateInertia(&bind.CallOpts{Context: ctx}) Require(t, err) if arbGasInfoInertia != inertia { Fatal(t, "expected inertia to be", inertia, "got", arbGasInfoInertia) } +} + +func TestL1PricingRewardRate(t *testing.T) { + t.Parallel() + + builder, cleanup, auth, arbOwner, arbGasInfo := setupArbOwnerAndArbGasInfo(t) + defer cleanup() + ctx := builder.ctx - // GetL1RewardRate test perUnitReward := uint64(13) - tx, err = arbOwner.SetL1PricingRewardRate(&auth, perUnitReward) + tx, err := arbOwner.SetL1PricingRewardRate(&auth, perUnitReward) Require(t, err) _, err = builder.L2.EnsureTxSucceeded(tx) Require(t, err) @@ -192,10 +227,17 @@ func TestArbGasInfoAndArbOwner(t *testing.T) { if arbGasInfoPerUnitReward != perUnitReward { Fatal(t, "expected per unit reward to be", perUnitReward, "got", arbGasInfoPerUnitReward) } +} + +func TestL1PricingRewardRecipient(t *testing.T) { + t.Parallel() + + builder, cleanup, auth, arbOwner, arbGasInfo := setupArbOwnerAndArbGasInfo(t) + defer cleanup() + ctx := builder.ctx - // GetL1RewardRecipient test rewardRecipient := common.BytesToAddress(crypto.Keccak256([]byte{})[:20]) - tx, err = arbOwner.SetL1PricingRewardRecipient(&auth, rewardRecipient) + tx, err := arbOwner.SetL1PricingRewardRecipient(&auth, rewardRecipient) Require(t, err) _, err = builder.L2.EnsureTxSucceeded(tx) Require(t, err) @@ -204,22 +246,36 @@ func TestArbGasInfoAndArbOwner(t *testing.T) { if arbGasInfoRewardRecipient.Cmp(rewardRecipient) != 0 { Fatal(t, "expected reward recipient to be", rewardRecipient, "got", arbGasInfoRewardRecipient) } +} + +func TestL2GasPricingInertia(t *testing.T) { + t.Parallel() - // GetPricingInertia test - inertia = uint64(14) - tx, err = arbOwner.SetL2GasPricingInertia(&auth, inertia) + builder, cleanup, auth, arbOwner, arbGasInfo := setupArbOwnerAndArbGasInfo(t) + defer cleanup() + ctx := builder.ctx + + inertia := uint64(14) + tx, err := arbOwner.SetL2GasPricingInertia(&auth, inertia) Require(t, err) _, err = builder.L2.EnsureTxSucceeded(tx) Require(t, err) - arbGasInfoInertia, err = arbGasInfo.GetPricingInertia(&bind.CallOpts{Context: ctx}) + arbGasInfoInertia, err := arbGasInfo.GetPricingInertia(&bind.CallOpts{Context: ctx}) Require(t, err) if arbGasInfoInertia != inertia { Fatal(t, "expected inertia to be", inertia, "got", arbGasInfoInertia) } +} + +func TestL2GasBacklogTolerance(t *testing.T) { + t.Parallel() + + builder, cleanup, auth, arbOwner, arbGasInfo := setupArbOwnerAndArbGasInfo(t) + defer cleanup() + ctx := builder.ctx - // GetGasBacklogTolerance test gasTolerance := uint64(15) - tx, err = arbOwner.SetL2GasBacklogTolerance(&auth, gasTolerance) + tx, err := arbOwner.SetL2GasBacklogTolerance(&auth, gasTolerance) Require(t, err) _, err = builder.L2.EnsureTxSucceeded(tx) Require(t, err) @@ -228,10 +284,17 @@ func TestArbGasInfoAndArbOwner(t *testing.T) { if arbGasInfoGasTolerance != gasTolerance { Fatal(t, "expected gas tolerance to be", gasTolerance, "got", arbGasInfoGasTolerance) } +} + +func TestPerBatchGasCharge(t *testing.T) { + t.Parallel() + + builder, cleanup, auth, arbOwner, arbGasInfo := setupArbOwnerAndArbGasInfo(t) + defer cleanup() + ctx := builder.ctx - // GetPerBatchGasCharge test perBatchGasCharge := int64(16) - tx, err = arbOwner.SetPerBatchGasCharge(&auth, perBatchGasCharge) + tx, err := arbOwner.SetPerBatchGasCharge(&auth, perBatchGasCharge) Require(t, err) _, err = builder.L2.EnsureTxSucceeded(tx) Require(t, err) @@ -240,10 +303,17 @@ func TestArbGasInfoAndArbOwner(t *testing.T) { if arbGasInfoPerBatchGasCharge != perBatchGasCharge { Fatal(t, "expected per batch gas charge to be", perBatchGasCharge, "got", arbGasInfoPerBatchGasCharge) } +} + +func TestL1PricingEquilibrationUnits(t *testing.T) { + t.Parallel() + + builder, cleanup, auth, arbOwner, arbGasInfo := setupArbOwnerAndArbGasInfo(t) + defer cleanup() + ctx := builder.ctx - // GetL1PricingEquilibrationUnits test equilUnits := big.NewInt(17) - tx, err = arbOwner.SetL1PricingEquilibrationUnits(&auth, equilUnits) + tx, err := arbOwner.SetL1PricingEquilibrationUnits(&auth, equilUnits) Require(t, err) _, err = builder.L2.EnsureTxSucceeded(tx) Require(t, err) @@ -252,11 +322,18 @@ func TestArbGasInfoAndArbOwner(t *testing.T) { if arbGasInfoEquilUnits.Cmp(equilUnits) != 0 { Fatal(t, "expected equilibration units to be", equilUnits, "got", arbGasInfoEquilUnits) } +} + +func TestGasAccountingParams(t *testing.T) { + t.Parallel() + + builder, cleanup, auth, arbOwner, arbGasInfo := setupArbOwnerAndArbGasInfo(t) + defer cleanup() + ctx := builder.ctx - // GetGasAccountingParams test speedLimit := uint64(18) txGasLimit := uint64(19) - tx, err = arbOwner.SetSpeedLimit(&auth, speedLimit) + tx, err := arbOwner.SetSpeedLimit(&auth, speedLimit) Require(t, err) _, err = builder.L2.EnsureTxSucceeded(tx) Require(t, err) @@ -275,8 +352,21 @@ func TestArbGasInfoAndArbOwner(t *testing.T) { if arbGasInfoTxGasLimit.Cmp(big.NewInt(int64(txGasLimit))) != 0 { Fatal(t, "expected tx gas limit to be", txGasLimit, "got", arbGasInfoTxGasLimit) } +} + +func TestCurrentTxL1GasFees(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, false) + cleanup := builder.Build(t) + defer cleanup() + + arbGasInfo, err := precompilesgen.NewArbGasInfo(common.HexToAddress("0x6c"), builder.L2.Client) + Require(t, err) - // GetCurrentTxL1GasFees test currTxL1GasFees, err := arbGasInfo.GetCurrentTxL1GasFees(&bind.CallOpts{Context: ctx}) Require(t, err) if currTxL1GasFees == nil { From 38e2180c58e66b32a2d96abc79feb750d1dae993 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 11 Oct 2024 10:24:49 -0300 Subject: [PATCH 423/489] Comments to precompile functions that needs to be tested as system tests --- precompiles/ArbAddressTable.go | 7 +++++++ precompiles/ArbAggregator.go | 7 +++++++ precompiles/ArbDebug.go | 2 ++ precompiles/ArbFunctionTable.go | 3 +++ precompiles/ArbGasInfo.go | 10 ++++++++++ precompiles/ArbOwner.go | 9 +++++++++ precompiles/ArbOwnerPublic.go | 2 ++ precompiles/ArbRetryableTx.go | 4 ++++ precompiles/ArbStatistics.go | 1 + precompiles/ArbSys.go | 3 ++- precompiles/ArbosActs.go | 2 ++ precompiles/ArbosTest.go | 1 + 12 files changed, 50 insertions(+), 1 deletion(-) diff --git a/precompiles/ArbAddressTable.go b/precompiles/ArbAddressTable.go index 102fd55c3b..45b708bd7d 100644 --- a/precompiles/ArbAddressTable.go +++ b/precompiles/ArbAddressTable.go @@ -13,16 +13,19 @@ type ArbAddressTable struct { Address addr // 0x66 } +// TODO: add system test // AddressExists checks if an address exists in the table func (con ArbAddressTable) AddressExists(c ctx, evm mech, addr addr) (bool, error) { return c.State.AddressTable().AddressExists(addr) } +// TODO: add system test // Compress and returns the bytes that represent the address func (con ArbAddressTable) Compress(c ctx, evm mech, addr addr) ([]uint8, error) { return c.State.AddressTable().Compress(addr) } +// TODO: add system test // Decompress the compressed bytes at the given offset with those of the corresponding account func (con ArbAddressTable) Decompress(c ctx, evm mech, buf []uint8, offset huge) (addr, huge, error) { if !offset.IsInt64() { @@ -36,6 +39,7 @@ func (con ArbAddressTable) Decompress(c ctx, evm mech, buf []uint8, offset huge) return result, new(big.Int).SetUint64(nbytes), err } +// TODO: add system test // Lookup the index of an address in the table func (con ArbAddressTable) Lookup(c ctx, evm mech, addr addr) (huge, error) { result, exists, err := c.State.AddressTable().Lookup(addr) @@ -48,6 +52,7 @@ func (con ArbAddressTable) Lookup(c ctx, evm mech, addr addr) (huge, error) { return new(big.Int).SetUint64(result), nil } +// TODO: add system test // LookupIndex for an address in the table by index func (con ArbAddressTable) LookupIndex(c ctx, evm mech, index huge) (addr, error) { if !index.IsUint64() { @@ -63,12 +68,14 @@ func (con ArbAddressTable) LookupIndex(c ctx, evm mech, index huge) (addr, error return result, nil } +// TODO: add system test // Register adds an account to the table, shrinking its compressed representation func (con ArbAddressTable) Register(c ctx, evm mech, addr addr) (huge, error) { slot, err := c.State.AddressTable().Register(addr) return new(big.Int).SetUint64(slot), err } +// TODO: add system test // Size gets the number of addresses in the table func (con ArbAddressTable) Size(c ctx, evm mech) (huge, error) { size, err := c.State.AddressTable().Size() diff --git a/precompiles/ArbAggregator.go b/precompiles/ArbAggregator.go index b74e280fe8..c3ee19c12c 100644 --- a/precompiles/ArbAggregator.go +++ b/precompiles/ArbAggregator.go @@ -19,12 +19,14 @@ type ArbAggregator struct { var ErrNotOwner = errors.New("must be called by chain owner") +// TODO: add system test // GetPreferredAggregator returns the preferred aggregator address. // Deprecated: Do not use this method. func (con ArbAggregator) GetPreferredAggregator(c ctx, evm mech, address addr) (prefAgg addr, isDefault bool, err error) { return l1pricing.BatchPosterAddress, true, err } +// TODO: add system test // GetDefaultAggregator returns the default aggregator address. // Deprecated: Do not use this method. func (con ArbAggregator) GetDefaultAggregator(c ctx, evm mech) (addr, error) { @@ -36,6 +38,7 @@ func (con ArbAggregator) GetBatchPosters(c ctx, evm mech) ([]addr, error) { return c.State.L1PricingState().BatchPosterTable().AllPosters(65536) } +// TODO: add system test func (con ArbAggregator) AddBatchPoster(c ctx, evm mech, newBatchPoster addr) error { isOwner, err := c.State.ChainOwners().IsMember(c.caller) if err != nil { @@ -58,6 +61,7 @@ func (con ArbAggregator) AddBatchPoster(c ctx, evm mech, newBatchPoster addr) er return nil } +// TODO: add system test // GetFeeCollector gets a batch poster's fee collector func (con ArbAggregator) GetFeeCollector(c ctx, evm mech, batchPoster addr) (addr, error) { posterInfo, err := c.State.L1PricingState().BatchPosterTable().OpenPoster(batchPoster, false) @@ -67,6 +71,7 @@ func (con ArbAggregator) GetFeeCollector(c ctx, evm mech, batchPoster addr) (add return posterInfo.PayTo() } +// TODO: add system test // SetFeeCollector sets a batch poster's fee collector (caller must be the batch poster, its fee collector, or an owner) func (con ArbAggregator) SetFeeCollector(c ctx, evm mech, batchPoster addr, newFeeCollector addr) error { posterInfo, err := c.State.L1PricingState().BatchPosterTable().OpenPoster(batchPoster, false) @@ -89,12 +94,14 @@ func (con ArbAggregator) SetFeeCollector(c ctx, evm mech, batchPoster addr, newF return posterInfo.SetPayTo(newFeeCollector) } +// TODO: add system test // GetTxBaseFee gets an aggregator's current fixed fee to submit a tx func (con ArbAggregator) GetTxBaseFee(c ctx, evm mech, aggregator addr) (huge, error) { // This is deprecated and now always returns zero. return big.NewInt(0), nil } +// TODO: add system test // SetTxBaseFee sets an aggregator's fixed fee (caller must be the aggregator, its fee collector, or an owner) func (con ArbAggregator) SetTxBaseFee(c ctx, evm mech, aggregator addr, feeInL1Gas huge) error { // This is deprecated and is now a no-op. diff --git a/precompiles/ArbDebug.go b/precompiles/ArbDebug.go index bf85d5e18f..9bfc8c9815 100644 --- a/precompiles/ArbDebug.go +++ b/precompiles/ArbDebug.go @@ -56,11 +56,13 @@ func (con ArbDebug) BecomeChainOwner(c ctx, evm mech) error { return c.State.ChainOwners().Add(c.caller) } +// TODO: add system test // Halts the chain by panicking in the STF func (con ArbDebug) Panic(c ctx, evm mech) error { panic("called ArbDebug's debug-only Panic method") } +// TODO: add system test func (con ArbDebug) LegacyError(c ctx) error { return errors.New("example legacy error") } diff --git a/precompiles/ArbFunctionTable.go b/precompiles/ArbFunctionTable.go index a870995e0f..e7091813db 100644 --- a/precompiles/ArbFunctionTable.go +++ b/precompiles/ArbFunctionTable.go @@ -15,16 +15,19 @@ type ArbFunctionTable struct { Address addr // 0x68 } +// TODO: add system test // Upload does nothing func (con ArbFunctionTable) Upload(c ctx, evm mech, buf []byte) error { return nil } +// TODO: add system test // Size returns the empty table's size, which is 0 func (con ArbFunctionTable) Size(c ctx, evm mech, addr addr) (huge, error) { return big.NewInt(0), nil } +// TODO: add system test // Get reverts since the table is empty func (con ArbFunctionTable) Get(c ctx, evm mech, addr addr, index huge) (huge, bool, huge, error) { return nil, false, nil, errors.New("table is empty") diff --git a/precompiles/ArbGasInfo.go b/precompiles/ArbGasInfo.go index b41dfda8a2..c738f0fa37 100644 --- a/precompiles/ArbGasInfo.go +++ b/precompiles/ArbGasInfo.go @@ -22,6 +22,7 @@ var storageArbGas = big.NewInt(int64(storage.StorageWriteCost)) const AssumedSimpleTxSize = 140 +// TODO: add system test // GetPricesInWeiWithAggregator gets prices in wei when using the provided aggregator func (con ArbGasInfo) GetPricesInWeiWithAggregator( c ctx, @@ -102,6 +103,7 @@ func (con ArbGasInfo) GetPricesInWei(c ctx, evm mech) (huge, huge, huge, huge, h return con.GetPricesInWeiWithAggregator(c, evm, addr{}) } +// TODO: add system test // GetPricesInArbGasWithAggregator gets prices in ArbGas when using the provided aggregator func (con ArbGasInfo) GetPricesInArbGasWithAggregator(c ctx, evm mech, aggregator addr) (huge, huge, huge, error) { if c.State.ArbOSVersion() < 4 { @@ -154,6 +156,7 @@ func (con ArbGasInfo) _preVersion4_GetPricesInArbGasWithAggregator(c ctx, evm me return perL2Tx, gasForL1Calldata, storageArbGas, nil } +// TODO: add system test // GetPricesInArbGas gets prices in ArbGas when using the caller's preferred aggregator func (con ArbGasInfo) GetPricesInArbGas(c ctx, evm mech) (huge, huge, huge, error) { return con.GetPricesInArbGasWithAggregator(c, evm, addr{}) @@ -192,6 +195,7 @@ func (con ArbGasInfo) GetL1RewardRecipient(c ctx, evm mech) (common.Address, err return c.State.L1PricingState().PayRewardsTo() } +// TODO: add system test // GetL1GasPriceEstimate gets the current estimate of the L1 basefee func (con ArbGasInfo) GetL1GasPriceEstimate(c ctx, evm mech) (huge, error) { return con.GetL1BaseFeeEstimate(c, evm) @@ -246,11 +250,13 @@ func (con ArbGasInfo) GetPerBatchGasCharge(c ctx, evm mech) (int64, error) { return c.State.L1PricingState().PerBatchGasCost() } +// TODO: add system test // GetAmortizedCostCapBips gets the cost amortization cap in basis points func (con ArbGasInfo) GetAmortizedCostCapBips(c ctx, evm mech) (uint64, error) { return c.State.L1PricingState().AmortizedCostCapBips() } +// TODO: add system test // GetL1FeesAvailable gets the available funds from L1 fees func (con ArbGasInfo) GetL1FeesAvailable(c ctx, evm mech) (huge, error) { return c.State.L1PricingState().L1FeesAvailable() @@ -261,21 +267,25 @@ func (con ArbGasInfo) GetL1PricingEquilibrationUnits(c ctx, evm mech) (*big.Int, return c.State.L1PricingState().EquilibrationUnits() } +// TODO: add system test // GetLastL1PricingUpdateTime gets the last time the L1 calldata pricer was updated func (con ArbGasInfo) GetLastL1PricingUpdateTime(c ctx, evm mech) (uint64, error) { return c.State.L1PricingState().LastUpdateTime() } +// TODO: add system test // GetL1PricingFundsDueForRewards gets the amount of L1 calldata payments due for rewards (per the L1 reward rate) func (con ArbGasInfo) GetL1PricingFundsDueForRewards(c ctx, evm mech) (*big.Int, error) { return c.State.L1PricingState().FundsDueForRewards() } +// TODO: add system test // GetL1PricingUnitsSinceUpdate gets the amount of L1 calldata posted since the last update func (con ArbGasInfo) GetL1PricingUnitsSinceUpdate(c ctx, evm mech) (uint64, error) { return c.State.L1PricingState().UnitsSinceUpdate() } +// TODO: add system test // GetLastL1PricingSurplus gets the L1 pricing surplus as of the last update (may be negative) func (con ArbGasInfo) GetLastL1PricingSurplus(c ctx, evm mech) (*big.Int, error) { return c.State.L1PricingState().LastSurplus() diff --git a/precompiles/ArbOwner.go b/precompiles/ArbOwner.go index 8b87445e0e..7387b62a62 100644 --- a/precompiles/ArbOwner.go +++ b/precompiles/ArbOwner.go @@ -33,11 +33,13 @@ var ( ErrOutOfBounds = errors.New("value out of bounds") ) +// TODO: add system test // AddChainOwner adds account as a chain owner func (con ArbOwner) AddChainOwner(c ctx, evm mech, newOwner addr) error { return c.State.ChainOwners().Add(newOwner) } +// TODO: add system test // RemoveChainOwner removes account from the list of chain owners func (con ArbOwner) RemoveChainOwner(c ctx, evm mech, addr addr) error { member, _ := con.IsChainOwner(c, evm, addr) @@ -52,6 +54,7 @@ func (con ArbOwner) IsChainOwner(c ctx, evm mech, addr addr) (bool, error) { return c.State.ChainOwners().IsMember(addr) } +// TODO: add system test // GetAllChainOwners retrieves the list of chain owners func (con ArbOwner) GetAllChainOwners(c ctx, evm mech) ([]common.Address, error) { return c.State.ChainOwners().AllMembers(65536) @@ -62,6 +65,7 @@ func (con ArbOwner) SetL1BaseFeeEstimateInertia(c ctx, evm mech, inertia uint64) return c.State.L1PricingState().SetInertia(inertia) } +// TODO: add system test // SetL2BaseFee sets the L2 gas price directly, bypassing the pool calculus func (con ArbOwner) SetL2BaseFee(c ctx, evm mech, priceInWei huge) error { return c.State.L2PricingState().SetBaseFeeWei(priceInWei) @@ -95,11 +99,13 @@ func (con ArbOwner) SetL2GasBacklogTolerance(c ctx, evm mech, sec uint64) error return c.State.L2PricingState().SetBacklogTolerance(sec) } +// TODO: add system test // GetNetworkFeeAccount gets the network fee collector func (con ArbOwner) GetNetworkFeeAccount(c ctx, evm mech) (addr, error) { return c.State.NetworkFeeAccount() } +// TODO: add system test // GetInfraFeeAccount gets the infrastructure fee collector func (con ArbOwner) GetInfraFeeAccount(c ctx, evm mech) (addr, error) { return c.State.InfraFeeAccount() @@ -144,6 +150,7 @@ func (con ArbOwner) SetPerBatchGasCharge(c ctx, evm mech, cost int64) error { return c.State.L1PricingState().SetPerBatchGasCost(cost) } +// TODO: add system test func (con ArbOwner) SetAmortizedCostCapBips(c ctx, evm mech, cap uint64) error { return c.State.L1PricingState().SetAmortizedCostCapBips(cap) } @@ -152,6 +159,7 @@ func (con ArbOwner) SetBrotliCompressionLevel(c ctx, evm mech, level uint64) err return c.State.SetBrotliCompressionLevel(level) } +// TODO: add system test func (con ArbOwner) ReleaseL1PricerSurplusFunds(c ctx, evm mech, maxWeiToRelease huge) (huge, error) { balance := evm.StateDB.GetBalance(l1pricing.L1PricerFundsPoolAddress) l1p := c.State.L1PricingState() @@ -295,6 +303,7 @@ func (con ArbOwner) RemoveWasmCacheManager(c ctx, _ mech, manager addr) error { return managers.Remove(manager, c.State.ArbOSVersion()) } +// TODO: add system test func (con ArbOwner) SetChainConfig(c ctx, evm mech, serializedChainConfig []byte) error { if c == nil { return errors.New("nil context") diff --git a/precompiles/ArbOwnerPublic.go b/precompiles/ArbOwnerPublic.go index 451e18e1cc..d901c77f12 100644 --- a/precompiles/ArbOwnerPublic.go +++ b/precompiles/ArbOwnerPublic.go @@ -16,11 +16,13 @@ type ArbOwnerPublic struct { ChainOwnerRectifiedGasCost func(addr) (uint64, error) } +// TODO: add system test // GetAllChainOwners retrieves the list of chain owners func (con ArbOwnerPublic) GetAllChainOwners(c ctx, evm mech) ([]common.Address, error) { return c.State.ChainOwners().AllMembers(65536) } +// TODO: add system test // RectifyChainOwner checks if the account is a chain owner func (con ArbOwnerPublic) RectifyChainOwner(c ctx, evm mech, addr addr) error { err := c.State.ChainOwners().RectifyMapping(addr) diff --git a/precompiles/ArbRetryableTx.go b/precompiles/ArbRetryableTx.go index d925499180..dac112e1bd 100644 --- a/precompiles/ArbRetryableTx.go +++ b/precompiles/ArbRetryableTx.go @@ -130,6 +130,7 @@ func (con ArbRetryableTx) Redeem(c ctx, evm mech, ticketId bytes32) (bytes32, er return retryTxHash, c.State.L2PricingState().AddToGasPool(arbmath.SaturatingCast[int64](gasToDonate)) } +// TODO: add system test // GetLifetime gets the default lifetime period a retryable has at creation func (con ArbRetryableTx) GetLifetime(c ctx, evm mech) (huge, error) { return big.NewInt(retryables.RetryableLifetimeSeconds), nil @@ -180,6 +181,7 @@ func (con ArbRetryableTx) Keepalive(c ctx, evm mech, ticketId bytes32) (huge, er return bigNewTimeout, err } +// TODO: add system test // GetBeneficiary gets the beneficiary of the ticket func (con ArbRetryableTx) GetBeneficiary(c ctx, evm mech, ticketId bytes32) (addr, error) { retryableState := c.State.RetryableState() @@ -222,6 +224,7 @@ func (con ArbRetryableTx) Cancel(c ctx, evm mech, ticketId bytes32) error { return con.Canceled(c, evm, ticketId) } +// TODO: add system test func (con ArbRetryableTx) GetCurrentRedeemer(c ctx, evm mech) (common.Address, error) { if c.txProcessor.CurrentRefundTo != nil { return *c.txProcessor.CurrentRefundTo, nil @@ -229,6 +232,7 @@ func (con ArbRetryableTx) GetCurrentRedeemer(c ctx, evm mech) (common.Address, e return common.Address{}, nil } +// TODO: add system test func (con ArbRetryableTx) SubmitRetryable( c ctx, evm mech, requestId bytes32, l1BaseFee, deposit, callvalue, gasFeeCap huge, gasLimit uint64, maxSubmissionFee huge, diff --git a/precompiles/ArbStatistics.go b/precompiles/ArbStatistics.go index 83f867d09d..c4b6a39344 100644 --- a/precompiles/ArbStatistics.go +++ b/precompiles/ArbStatistics.go @@ -14,6 +14,7 @@ type ArbStatistics struct { Address addr // 0x6e } +// TODO: add system test // GetStats returns the current block number and some statistics about the rollup's pre-Nitro state func (con ArbStatistics) GetStats(c ctx, evm mech) (huge, huge, huge, huge, huge, huge, error) { blockNum := evm.Context.BlockNumber diff --git a/precompiles/ArbSys.go b/precompiles/ArbSys.go index d55067a09c..f48573fafe 100644 --- a/precompiles/ArbSys.go +++ b/precompiles/ArbSys.go @@ -75,6 +75,7 @@ func (con *ArbSys) IsTopLevelCall(c ctx, evm mech) (bool, error) { return evm.Depth() <= 2, nil } +// TODO: add system test // MapL1SenderContractAddressToL2Alias gets the contract's L2 alias func (con *ArbSys) MapL1SenderContractAddressToL2Alias(c ctx, sender addr, dest addr) (addr, error) { return util.RemapL1Address(sender), nil @@ -92,7 +93,6 @@ func (con *ArbSys) WasMyCallersAddressAliased(c ctx, evm mech) (bool, error) { // MyCallersAddressWithoutAliasing gets the caller's caller without any potential aliasing func (con *ArbSys) MyCallersAddressWithoutAliasing(c ctx, evm mech) (addr, error) { - address := addr{} if evm.Depth() > 1 { @@ -106,6 +106,7 @@ func (con *ArbSys) MyCallersAddressWithoutAliasing(c ctx, evm mech) (addr, error return address, err } +// TODO: add system test // SendTxToL1 sends a transaction to L1, adding it to the outbox func (con *ArbSys) SendTxToL1(c ctx, evm mech, value huge, destination addr, calldataForL1 []byte) (huge, error) { l1BlockNum, err := c.txProcessor.L1BlockNumber(vm.BlockContext{}) diff --git a/precompiles/ArbosActs.go b/precompiles/ArbosActs.go index e18aa43efb..45808f35a8 100644 --- a/precompiles/ArbosActs.go +++ b/precompiles/ArbosActs.go @@ -11,10 +11,12 @@ type ArbosActs struct { CallerNotArbOSError func() error } +// TODO: add system test func (con ArbosActs) StartBlock(c ctx, evm mech, l1BaseFee huge, l1BlockNumber, l2BlockNumber, timeLastBlock uint64) error { return con.CallerNotArbOSError() } +// TODO: add system test func (con ArbosActs) BatchPostingReport(c ctx, evm mech, batchTimestamp huge, batchPosterAddress addr, batchNumber uint64, batchDataGas uint64, l1BaseFeeWei huge) error { return con.CallerNotArbOSError() } diff --git a/precompiles/ArbosTest.go b/precompiles/ArbosTest.go index 6e988f533a..8a770ea339 100644 --- a/precompiles/ArbosTest.go +++ b/precompiles/ArbosTest.go @@ -12,6 +12,7 @@ type ArbosTest struct { Address addr // 0x69 } +// TODO: add system test // BurnArbGas unproductively burns the amount of L2 ArbGas func (con ArbosTest) BurnArbGas(c ctx, gasAmount huge) error { if !gasAmount.IsUint64() { From f9d877df0ac2397b878397aec4e559f9cbe89b30 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 11 Oct 2024 11:31:16 -0300 Subject: [PATCH 424/489] Moves TestArbStatistics to system_tests --- precompiles/ArbStatistics.go | 1 - precompiles/ArbStatistics_test.go | 22 ---------------------- system_tests/precompile_test.go | 24 ++++++++++++++++++++++++ 3 files changed, 24 insertions(+), 23 deletions(-) delete mode 100644 precompiles/ArbStatistics_test.go diff --git a/precompiles/ArbStatistics.go b/precompiles/ArbStatistics.go index c4b6a39344..83f867d09d 100644 --- a/precompiles/ArbStatistics.go +++ b/precompiles/ArbStatistics.go @@ -14,7 +14,6 @@ type ArbStatistics struct { Address addr // 0x6e } -// TODO: add system test // GetStats returns the current block number and some statistics about the rollup's pre-Nitro state func (con ArbStatistics) GetStats(c ctx, evm mech) (huge, huge, huge, huge, huge, huge, error) { blockNum := evm.Context.BlockNumber diff --git a/precompiles/ArbStatistics_test.go b/precompiles/ArbStatistics_test.go deleted file mode 100644 index 9787752906..0000000000 --- a/precompiles/ArbStatistics_test.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2021-2024, Offchain Labs, Inc. -// For license information, see https://github.com/nitro/blob/master/LICENSE - -package precompiles - -import ( - "testing" - - "github.com/ethereum/go-ethereum/common" -) - -func TestArbStatistics(t *testing.T) { - evm := newMockEVMForTesting() - stats := ArbStatistics{} - context := testContext(common.Address{}, evm) - - blockNum, _, _, _, _, _, err := stats.GetStats(context, evm) - Require(t, err) - if blockNum.Cmp(evm.Context.BlockNumber) != 0 { - t.Error("Unexpected block number") - } -} diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index 2b979f0dfd..5b9b89a91b 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -11,6 +11,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" "github.com/offchainlabs/nitro/arbos" @@ -460,3 +461,26 @@ func TestScheduleArbosUpgrade(t *testing.T) { t.Errorf("expected upgrade to be scheduled for version %v timestamp %v, got version %v timestamp %v", testVersion, testTimestamp, scheduled.ArbosVersion, scheduled.ScheduledForTimestamp) } } + +func TestArbStatistics(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, false) + cleanup := builder.Build(t) + defer cleanup() + + arbStatistics, err := precompilesgen.NewArbStatistics(types.ArbStatisticsAddress, builder.L2.Client) + Require(t, err) + + callOpts := &bind.CallOpts{Context: ctx} + blockNum, _, _, _, _, _, err := arbStatistics.GetStats(callOpts) + Require(t, err) + + expectedBlockNum, err := builder.L2.Client.BlockNumber(ctx) + Require(t, err) + + if blockNum.Uint64() != expectedBlockNum { + Fatal(t, "expected block number to be", expectedBlockNum, "got", blockNum) + } +} From 0b524822c93f577dff5fef270ea1f460ae478e2a Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 11 Oct 2024 13:10:43 -0300 Subject: [PATCH 425/489] Moves TestArbFunctionTable to system_tests --- precompiles/ArbFunctionTable.go | 3 --- precompiles/ArbFunctionTable_test.go | 37 --------------------------- system_tests/precompile_test.go | 38 ++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 40 deletions(-) delete mode 100644 precompiles/ArbFunctionTable_test.go diff --git a/precompiles/ArbFunctionTable.go b/precompiles/ArbFunctionTable.go index e7091813db..a870995e0f 100644 --- a/precompiles/ArbFunctionTable.go +++ b/precompiles/ArbFunctionTable.go @@ -15,19 +15,16 @@ type ArbFunctionTable struct { Address addr // 0x68 } -// TODO: add system test // Upload does nothing func (con ArbFunctionTable) Upload(c ctx, evm mech, buf []byte) error { return nil } -// TODO: add system test // Size returns the empty table's size, which is 0 func (con ArbFunctionTable) Size(c ctx, evm mech, addr addr) (huge, error) { return big.NewInt(0), nil } -// TODO: add system test // Get reverts since the table is empty func (con ArbFunctionTable) Get(c ctx, evm mech, addr addr, index huge) (huge, bool, huge, error) { return nil, false, nil, errors.New("table is empty") diff --git a/precompiles/ArbFunctionTable_test.go b/precompiles/ArbFunctionTable_test.go deleted file mode 100644 index f3c6d97bbf..0000000000 --- a/precompiles/ArbFunctionTable_test.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2021-2024, Offchain Labs, Inc. -// For license information, see https://github.com/nitro/blob/master/LICENSE - -package precompiles - -import ( - "math/big" - "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" -) - -func TestArbFunctionTable(t *testing.T) { - t.Parallel() - - evm := newMockEVMForTesting() - ftab := ArbFunctionTable{} - context := testContext(common.Address{}, evm) - - addr := common.BytesToAddress(crypto.Keccak256([]byte{})[:20]) - - // should be a noop - err := ftab.Upload(context, evm, []byte{0, 0, 0, 0}) - Require(t, err) - - size, err := ftab.Size(context, evm, addr) - Require(t, err) - if size.Cmp(big.NewInt(0)) != 0 { - t.Fatal("Size should be 0") - } - - _, _, _, err = ftab.Get(context, evm, addr, big.NewInt(10)) - if err == nil { - t.Fatal("Should error") - } -} diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index 5b9b89a91b..ea07427221 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -463,6 +463,8 @@ func TestScheduleArbosUpgrade(t *testing.T) { } func TestArbStatistics(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -484,3 +486,39 @@ func TestArbStatistics(t *testing.T) { Fatal(t, "expected block number to be", expectedBlockNum, "got", blockNum) } } + +func TestArbFunctionTable(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, false) + cleanup := builder.Build(t) + defer cleanup() + + auth := builder.L2Info.GetDefaultTransactOpts("Owner", ctx) + callOpts := &bind.CallOpts{Context: ctx} + + arbFunctionTable, err := precompilesgen.NewArbFunctionTable(types.ArbFunctionTableAddress, builder.L2.Client) + Require(t, err) + + addr := common.BytesToAddress(crypto.Keccak256([]byte{})[:20]) + + // should be a noop + tx, err := arbFunctionTable.Upload(&auth, []byte{0, 0, 0, 0}) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + + size, err := arbFunctionTable.Size(callOpts, addr) + Require(t, err) + if size.Cmp(big.NewInt(0)) != 0 { + t.Fatal("Size should be 0") + } + + _, _, _, err = arbFunctionTable.Get(callOpts, addr, big.NewInt(10)) + if err == nil { + t.Fatal("Should error") + } +} From da7958e08b7d346a93925bee8c5120bd60bbfef1 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 14 Oct 2024 10:05:30 -0300 Subject: [PATCH 426/489] ArbGasInfo.GetL1GasPriceEstimate test --- precompiles/ArbGasInfo.go | 1 - system_tests/fees_test.go | 6 ++++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/precompiles/ArbGasInfo.go b/precompiles/ArbGasInfo.go index c738f0fa37..abe0f53588 100644 --- a/precompiles/ArbGasInfo.go +++ b/precompiles/ArbGasInfo.go @@ -195,7 +195,6 @@ func (con ArbGasInfo) GetL1RewardRecipient(c ctx, evm mech) (common.Address, err return c.State.L1PricingState().PayRewardsTo() } -// TODO: add system test // GetL1GasPriceEstimate gets the current estimate of the L1 basefee func (con ArbGasInfo) GetL1GasPriceEstimate(c ctx, evm mech) (huge, error) { return con.GetL1BaseFeeEstimate(c, evm) diff --git a/system_tests/fees_test.go b/system_tests/fees_test.go index ccca82e009..76de23e2cb 100644 --- a/system_tests/fees_test.go +++ b/system_tests/fees_test.go @@ -55,6 +55,12 @@ func TestSequencerFeePaid(t *testing.T) { l1Estimate, err := arbGasInfo.GetL1BaseFeeEstimate(callOpts) Require(t, err) + l1EstimateThroughGetL1GasPriceEstimate, err := arbGasInfo.GetL1GasPriceEstimate(callOpts) + Require(t, err) + if !arbmath.BigEquals(l1Estimate, l1EstimateThroughGetL1GasPriceEstimate) { + Fatal(t, "GetL1BaseFeeEstimate and GetL1GasPriceEstimate should return the same value") + } + baseFee := builder.L2.GetBaseFee(t) builder.L2Info.GasPrice = baseFee From 4e109349a5aee4b1568889e26d6d183d7ccdf6f1 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 14 Oct 2024 10:13:53 -0300 Subject: [PATCH 427/489] TestArbGasInfoDoesntRevert --- precompiles/ArbGasInfo.go | 9 ------- system_tests/precompile_test.go | 45 +++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/precompiles/ArbGasInfo.go b/precompiles/ArbGasInfo.go index abe0f53588..b41dfda8a2 100644 --- a/precompiles/ArbGasInfo.go +++ b/precompiles/ArbGasInfo.go @@ -22,7 +22,6 @@ var storageArbGas = big.NewInt(int64(storage.StorageWriteCost)) const AssumedSimpleTxSize = 140 -// TODO: add system test // GetPricesInWeiWithAggregator gets prices in wei when using the provided aggregator func (con ArbGasInfo) GetPricesInWeiWithAggregator( c ctx, @@ -103,7 +102,6 @@ func (con ArbGasInfo) GetPricesInWei(c ctx, evm mech) (huge, huge, huge, huge, h return con.GetPricesInWeiWithAggregator(c, evm, addr{}) } -// TODO: add system test // GetPricesInArbGasWithAggregator gets prices in ArbGas when using the provided aggregator func (con ArbGasInfo) GetPricesInArbGasWithAggregator(c ctx, evm mech, aggregator addr) (huge, huge, huge, error) { if c.State.ArbOSVersion() < 4 { @@ -156,7 +154,6 @@ func (con ArbGasInfo) _preVersion4_GetPricesInArbGasWithAggregator(c ctx, evm me return perL2Tx, gasForL1Calldata, storageArbGas, nil } -// TODO: add system test // GetPricesInArbGas gets prices in ArbGas when using the caller's preferred aggregator func (con ArbGasInfo) GetPricesInArbGas(c ctx, evm mech) (huge, huge, huge, error) { return con.GetPricesInArbGasWithAggregator(c, evm, addr{}) @@ -249,13 +246,11 @@ func (con ArbGasInfo) GetPerBatchGasCharge(c ctx, evm mech) (int64, error) { return c.State.L1PricingState().PerBatchGasCost() } -// TODO: add system test // GetAmortizedCostCapBips gets the cost amortization cap in basis points func (con ArbGasInfo) GetAmortizedCostCapBips(c ctx, evm mech) (uint64, error) { return c.State.L1PricingState().AmortizedCostCapBips() } -// TODO: add system test // GetL1FeesAvailable gets the available funds from L1 fees func (con ArbGasInfo) GetL1FeesAvailable(c ctx, evm mech) (huge, error) { return c.State.L1PricingState().L1FeesAvailable() @@ -266,25 +261,21 @@ func (con ArbGasInfo) GetL1PricingEquilibrationUnits(c ctx, evm mech) (*big.Int, return c.State.L1PricingState().EquilibrationUnits() } -// TODO: add system test // GetLastL1PricingUpdateTime gets the last time the L1 calldata pricer was updated func (con ArbGasInfo) GetLastL1PricingUpdateTime(c ctx, evm mech) (uint64, error) { return c.State.L1PricingState().LastUpdateTime() } -// TODO: add system test // GetL1PricingFundsDueForRewards gets the amount of L1 calldata payments due for rewards (per the L1 reward rate) func (con ArbGasInfo) GetL1PricingFundsDueForRewards(c ctx, evm mech) (*big.Int, error) { return c.State.L1PricingState().FundsDueForRewards() } -// TODO: add system test // GetL1PricingUnitsSinceUpdate gets the amount of L1 calldata posted since the last update func (con ArbGasInfo) GetL1PricingUnitsSinceUpdate(c ctx, evm mech) (uint64, error) { return c.State.L1PricingState().UnitsSinceUpdate() } -// TODO: add system test // GetLastL1PricingSurplus gets the L1 pricing surplus as of the last update (may be negative) func (con ArbGasInfo) GetLastL1PricingSurplus(c ctx, evm mech) (*big.Int, error) { return c.State.L1PricingState().LastSurplus() diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index ea07427221..f354fdd8c3 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -522,3 +522,48 @@ func TestArbFunctionTable(t *testing.T) { t.Fatal("Should error") } } + +func TestArbGasInfoDoesntRevert(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, false) + cleanup := builder.Build(t) + defer cleanup() + + callOpts := &bind.CallOpts{Context: ctx} + addr := common.BytesToAddress(crypto.Keccak256([]byte{})[:20]) + + arbGasInfo, err := precompilesgen.NewArbGasInfo(types.ArbGasInfoAddress, builder.L2.Client) + Require(t, err) + + _, err = arbGasInfo.GetGasBacklog(callOpts) + Require(t, err) + + _, err = arbGasInfo.GetLastL1PricingUpdateTime(callOpts) + Require(t, err) + + _, err = arbGasInfo.GetL1PricingFundsDueForRewards(callOpts) + Require(t, err) + + _, err = arbGasInfo.GetL1PricingUnitsSinceUpdate(callOpts) + Require(t, err) + + _, err = arbGasInfo.GetLastL1PricingSurplus(callOpts) + Require(t, err) + + _, _, _, err = arbGasInfo.GetPricesInArbGas(callOpts) + Require(t, err) + + _, _, _, err = arbGasInfo.GetPricesInArbGasWithAggregator(callOpts, addr) + Require(t, err) + + _, err = arbGasInfo.GetAmortizedCostCapBips(callOpts) + Require(t, err) + + _, err = arbGasInfo.GetL1FeesAvailable(callOpts) + Require(t, err) + + _, _, _, _, _, _, err = arbGasInfo.GetPricesInWeiWithAggregator(callOpts, addr) + Require(t, err) +} From 1ac9018e03e88159059242f2c56f325dddd23daa Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 14 Oct 2024 10:50:29 -0300 Subject: [PATCH 428/489] System test for ArbRetryableTx.GetLifetime --- precompiles/ArbRetryableTx.go | 1 - precompiles/ArbRetryableTx_test.go | 9 +-------- system_tests/precompile_test.go | 2 ++ system_tests/retryable_test.go | 22 ++++++++++++++++++++++ 4 files changed, 25 insertions(+), 9 deletions(-) diff --git a/precompiles/ArbRetryableTx.go b/precompiles/ArbRetryableTx.go index dac112e1bd..b2d58b9dfd 100644 --- a/precompiles/ArbRetryableTx.go +++ b/precompiles/ArbRetryableTx.go @@ -130,7 +130,6 @@ func (con ArbRetryableTx) Redeem(c ctx, evm mech, ticketId bytes32) (bytes32, er return retryTxHash, c.State.L2PricingState().AddToGasPool(arbmath.SaturatingCast[int64](gasToDonate)) } -// TODO: add system test // GetLifetime gets the default lifetime period a retryable has at creation func (con ArbRetryableTx) GetLifetime(c ctx, evm mech) (huge, error) { return big.NewInt(retryables.RetryableLifetimeSeconds), nil diff --git a/precompiles/ArbRetryableTx_test.go b/precompiles/ArbRetryableTx_test.go index 58eed5ad77..ced3208d27 100644 --- a/precompiles/ArbRetryableTx_test.go +++ b/precompiles/ArbRetryableTx_test.go @@ -8,7 +8,6 @@ import ( "testing" "github.com/offchainlabs/nitro/arbos" - "github.com/offchainlabs/nitro/arbos/retryables" "github.com/offchainlabs/nitro/arbos/storage" "github.com/ethereum/go-ethereum/common" @@ -25,19 +24,13 @@ func newMockEVMForTestingWithCurrentRefundTo(currentRefundTo *common.Address) *v return evm } -func TestGetLifetimeAndCurrentRedeemer(t *testing.T) { +func TestGetCurrentRedeemer(t *testing.T) { currentRefundTo := common.HexToAddress("0x030405") evm := newMockEVMForTestingWithCurrentRefundTo(¤tRefundTo) retryableTx := ArbRetryableTx{} context := testContext(common.Address{}, evm) - lifetime, err := retryableTx.GetLifetime(context, evm) - Require(t, err) - if lifetime.Cmp(big.NewInt(retryables.RetryableLifetimeSeconds)) != 0 { - t.Fatal("Expected to be ", retryables.RetryableLifetimeSeconds, " but got ", lifetime) - } - currentRedeemer, err := retryableTx.GetCurrentRedeemer(context, evm) Require(t, err) if currentRefundTo.Cmp(currentRedeemer) != 0 { diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index f354fdd8c3..69ed699dc8 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -524,6 +524,8 @@ func TestArbFunctionTable(t *testing.T) { } func TestArbGasInfoDoesntRevert(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/system_tests/retryable_test.go b/system_tests/retryable_test.go index 6651c8ca74..01519cc2c0 100644 --- a/system_tests/retryable_test.go +++ b/system_tests/retryable_test.go @@ -423,6 +423,28 @@ func TestSubmitRetryableFailThenRetry(t *testing.T) { } } +func TestGetLifetime(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, false) + cleanup := builder.Build(t) + defer cleanup() + + callOpts := &bind.CallOpts{Context: ctx} + + arbRetryableTx, err := precompilesgen.NewArbRetryableTx(common.HexToAddress("6e"), builder.L2.Client) + Require(t, err) + + lifetime, err := arbRetryableTx.GetLifetime(callOpts) + Require(t, err) + if lifetime.Cmp(big.NewInt(retryables.RetryableLifetimeSeconds)) != 0 { + t.Fatal("Expected to be ", retryables.RetryableLifetimeSeconds, " but got ", lifetime) + } +} + func TestKeepaliveAndCancelRetryable(t *testing.T) { t.Parallel() builder, delayedInbox, lookupL2Tx, ctx, teardown := retryableSetup(t) From 01d154b5fbb579f55d05f517f7c27c611b9835a1 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 14 Oct 2024 10:59:35 -0300 Subject: [PATCH 429/489] Moves ArbRetryableTx.GetBeneficiary to system test --- precompiles/ArbRetryableTx.go | 1 - precompiles/ArbRetryableTx_test.go | 19 +------------------ system_tests/retryable_test.go | 8 ++++++++ 3 files changed, 9 insertions(+), 19 deletions(-) diff --git a/precompiles/ArbRetryableTx.go b/precompiles/ArbRetryableTx.go index b2d58b9dfd..58342fd825 100644 --- a/precompiles/ArbRetryableTx.go +++ b/precompiles/ArbRetryableTx.go @@ -180,7 +180,6 @@ func (con ArbRetryableTx) Keepalive(c ctx, evm mech, ticketId bytes32) (huge, er return bigNewTimeout, err } -// TODO: add system test // GetBeneficiary gets the beneficiary of the ticket func (con ArbRetryableTx) GetBeneficiary(c ctx, evm mech, ticketId bytes32) (addr, error) { retryableState := c.State.RetryableState() diff --git a/precompiles/ArbRetryableTx_test.go b/precompiles/ArbRetryableTx_test.go index ced3208d27..47450299ce 100644 --- a/precompiles/ArbRetryableTx_test.go +++ b/precompiles/ArbRetryableTx_test.go @@ -38,7 +38,7 @@ func TestGetCurrentRedeemer(t *testing.T) { } } -func TestRetryableRedeemAndGetBeneficiary(t *testing.T) { +func TestRetryableRedeem(t *testing.T) { evm := newMockEVMForTesting() precompileCtx := testContext(common.Address{}, evm) @@ -87,21 +87,4 @@ func TestRetryableRedeemAndGetBeneficiary(t *testing.T) { // to handle both cases, and some will be left over in this test's use case. Fail(t, "didn't consume all the expected gas") } - - getBeneficiaryCallData, err := retryABI.Pack("getBeneficiary", id) - Require(t, err) - retrievedBeneficiary, _, err := Precompiles()[retryAddress].Call( - getBeneficiaryCallData, - retryAddress, - retryAddress, - common.Address{}, - big.NewInt(0), - false, - 1000000, - evm, - ) - Require(t, err) - if common.BytesToAddress(retrievedBeneficiary).Cmp(beneficiary) != 0 { - Fail(t, "expected beneficiary to be ", beneficiary, " but got ", common.BytesToAddress(retrievedBeneficiary)) - } } diff --git a/system_tests/retryable_test.go b/system_tests/retryable_test.go index 01519cc2c0..89446e3c4b 100644 --- a/system_tests/retryable_test.go +++ b/system_tests/retryable_test.go @@ -502,6 +502,14 @@ func TestKeepaliveAndCancelRetryable(t *testing.T) { // checks that the ticket exists and gets current timeout timeoutBeforeKeepalive, err := arbRetryableTx.GetTimeout(&bind.CallOpts{}, ticketId) Require(t, err) + + // checks beneficiary + retrievedBeneficiaryAddress, err := arbRetryableTx.GetBeneficiary(&bind.CallOpts{}, ticketId) + Require(t, err) + if retrievedBeneficiaryAddress != beneficiaryAddress { + Fatal(t, "expected beneficiary to be", beneficiaryAddress, "but got", retrievedBeneficiaryAddress) + } + // checks that keepalive increases the timeout as expected _, err = arbRetryableTx.Keepalive(&ownerTxOpts, ticketId) Require(t, err) From acd3ab87b8ebb45329ad292153235e6b6ead9b06 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 14 Oct 2024 11:14:47 -0300 Subject: [PATCH 430/489] TestArbRetryableTxDoesntRevert --- precompiles/ArbRetryableTx.go | 1 - system_tests/retryable_test.go | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/precompiles/ArbRetryableTx.go b/precompiles/ArbRetryableTx.go index 58342fd825..0ca32e487a 100644 --- a/precompiles/ArbRetryableTx.go +++ b/precompiles/ArbRetryableTx.go @@ -222,7 +222,6 @@ func (con ArbRetryableTx) Cancel(c ctx, evm mech, ticketId bytes32) error { return con.Canceled(c, evm, ticketId) } -// TODO: add system test func (con ArbRetryableTx) GetCurrentRedeemer(c ctx, evm mech) (common.Address, error) { if c.txProcessor.CurrentRefundTo != nil { return *c.txProcessor.CurrentRefundTo, nil diff --git a/system_tests/retryable_test.go b/system_tests/retryable_test.go index 89446e3c4b..070902c155 100644 --- a/system_tests/retryable_test.go +++ b/system_tests/retryable_test.go @@ -445,6 +445,25 @@ func TestGetLifetime(t *testing.T) { } } +func TestArbRetryableTxDoesntRevert(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, false) + cleanup := builder.Build(t) + defer cleanup() + + callOpts := &bind.CallOpts{Context: ctx} + + arbRetryableTx, err := precompilesgen.NewArbRetryableTx(common.HexToAddress("6e"), builder.L2.Client) + Require(t, err) + + _, err = arbRetryableTx.GetCurrentRedeemer(callOpts) + Require(t, err) +} + func TestKeepaliveAndCancelRetryable(t *testing.T) { t.Parallel() builder, delayedInbox, lookupL2Tx, ctx, teardown := retryableSetup(t) From 6aa5b4e6463a8199ade646e6560049217d21bdb0 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 14 Oct 2024 11:15:03 -0300 Subject: [PATCH 431/489] TestSubmitRetryable --- precompiles/ArbRetryableTx.go | 1 - system_tests/retryable_test.go | 41 ++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/precompiles/ArbRetryableTx.go b/precompiles/ArbRetryableTx.go index 0ca32e487a..d925499180 100644 --- a/precompiles/ArbRetryableTx.go +++ b/precompiles/ArbRetryableTx.go @@ -229,7 +229,6 @@ func (con ArbRetryableTx) GetCurrentRedeemer(c ctx, evm mech) (common.Address, e return common.Address{}, nil } -// TODO: add system test func (con ArbRetryableTx) SubmitRetryable( c ctx, evm mech, requestId bytes32, l1BaseFee, deposit, callvalue, gasFeeCap huge, gasLimit uint64, maxSubmissionFee huge, diff --git a/system_tests/retryable_test.go b/system_tests/retryable_test.go index 070902c155..d7a7b08337 100644 --- a/system_tests/retryable_test.go +++ b/system_tests/retryable_test.go @@ -5,6 +5,7 @@ package arbtest import ( "context" + "fmt" "math/big" "strings" "testing" @@ -464,6 +465,46 @@ func TestArbRetryableTxDoesntRevert(t *testing.T) { Require(t, err) } +func TestSubmitRetryable(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, false) + cleanup := builder.Build(t) + defer cleanup() + + auth := builder.L2Info.GetDefaultTransactOpts("Owner", ctx) + + arbRetryableTx, err := precompilesgen.NewArbRetryableTx(common.HexToAddress("6e"), builder.L2.Client) + Require(t, err) + + _, err = arbRetryableTx.SubmitRetryable( + &auth, + [32]byte{}, + big.NewInt(0), + big.NewInt(0), + big.NewInt(0), + big.NewInt(0), + 0, + big.NewInt(0), + common.Address{}, + common.Address{}, + common.Address{}, + []byte{}, + ) + if err == nil { + t.Fatal("expected error") + } + observedMessage := err.Error() + expectedError := "NotCallable()" + expectedMessage := fmt.Sprintf("execution reverted: error %v: %v", expectedError, expectedError) + if observedMessage != expectedMessage { + Fatal(t, observedMessage) + } +} + func TestKeepaliveAndCancelRetryable(t *testing.T) { t.Parallel() builder, delayedInbox, lookupL2Tx, ctx, teardown := retryableSetup(t) From a2551a8e21eaa5122109073687210128c77f298e Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 14 Oct 2024 11:21:13 -0300 Subject: [PATCH 432/489] Moves TestSubmitRetryable to TestCustomSolidityErrors --- system_tests/precompile_test.go | 28 ++++++++++++++++++++++ system_tests/retryable_test.go | 41 --------------------------------- 2 files changed, 28 insertions(+), 41 deletions(-) diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index 69ed699dc8..430b4b4eab 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -79,6 +79,8 @@ func TestCustomSolidityErrors(t *testing.T) { defer cleanup() callOpts := &bind.CallOpts{Context: ctx} + auth := builder.L2Info.GetDefaultTransactOpts("Owner", ctx) + arbDebug, err := precompilesgen.NewArbDebug(common.HexToAddress("0xff"), builder.L2.Client) Require(t, err, "could not bind ArbDebug contract") customError := arbDebug.CustomRevert(callOpts, 1024) @@ -105,6 +107,32 @@ func TestCustomSolidityErrors(t *testing.T) { if observedMessage != expectedMessage { Fatal(t, observedMessage) } + + arbRetryableTx, err := precompilesgen.NewArbRetryableTx(common.HexToAddress("6e"), builder.L2.Client) + Require(t, err) + _, err = arbRetryableTx.SubmitRetryable( + &auth, + [32]byte{}, + big.NewInt(0), + big.NewInt(0), + big.NewInt(0), + big.NewInt(0), + 0, + big.NewInt(0), + common.Address{}, + common.Address{}, + common.Address{}, + []byte{}, + ) + if err == nil { + Fatal(t, "SubmitRetryable call should have errored") + } + observedMessage = err.Error() + expectedError = "NotCallable()" + expectedMessage = fmt.Sprintf("execution reverted: error %v: %v", expectedError, expectedError) + if observedMessage != expectedMessage { + Fatal(t, observedMessage) + } } func TestPrecompileErrorGasLeft(t *testing.T) { diff --git a/system_tests/retryable_test.go b/system_tests/retryable_test.go index d7a7b08337..070902c155 100644 --- a/system_tests/retryable_test.go +++ b/system_tests/retryable_test.go @@ -5,7 +5,6 @@ package arbtest import ( "context" - "fmt" "math/big" "strings" "testing" @@ -465,46 +464,6 @@ func TestArbRetryableTxDoesntRevert(t *testing.T) { Require(t, err) } -func TestSubmitRetryable(t *testing.T) { - t.Parallel() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - builder := NewNodeBuilder(ctx).DefaultConfig(t, false) - cleanup := builder.Build(t) - defer cleanup() - - auth := builder.L2Info.GetDefaultTransactOpts("Owner", ctx) - - arbRetryableTx, err := precompilesgen.NewArbRetryableTx(common.HexToAddress("6e"), builder.L2.Client) - Require(t, err) - - _, err = arbRetryableTx.SubmitRetryable( - &auth, - [32]byte{}, - big.NewInt(0), - big.NewInt(0), - big.NewInt(0), - big.NewInt(0), - 0, - big.NewInt(0), - common.Address{}, - common.Address{}, - common.Address{}, - []byte{}, - ) - if err == nil { - t.Fatal("expected error") - } - observedMessage := err.Error() - expectedError := "NotCallable()" - expectedMessage := fmt.Sprintf("execution reverted: error %v: %v", expectedError, expectedError) - if observedMessage != expectedMessage { - Fatal(t, observedMessage) - } -} - func TestKeepaliveAndCancelRetryable(t *testing.T) { t.Parallel() builder, delayedInbox, lookupL2Tx, ctx, teardown := retryableSetup(t) From 2f7fcfcfdc08cc38f464e2a7d4929faefa744cde Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 14 Oct 2024 11:27:17 -0300 Subject: [PATCH 433/489] ArbosActs system tests --- precompiles/ArbosActs.go | 2 -- system_tests/precompile_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/precompiles/ArbosActs.go b/precompiles/ArbosActs.go index 45808f35a8..e18aa43efb 100644 --- a/precompiles/ArbosActs.go +++ b/precompiles/ArbosActs.go @@ -11,12 +11,10 @@ type ArbosActs struct { CallerNotArbOSError func() error } -// TODO: add system test func (con ArbosActs) StartBlock(c ctx, evm mech, l1BaseFee huge, l1BlockNumber, l2BlockNumber, timeLastBlock uint64) error { return con.CallerNotArbOSError() } -// TODO: add system test func (con ArbosActs) BatchPostingReport(c ctx, evm mech, batchTimestamp huge, batchPosterAddress addr, batchNumber uint64, batchDataGas uint64, l1BaseFeeWei huge) error { return con.CallerNotArbOSError() } diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index 430b4b4eab..1e3c1adcee 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -133,6 +133,30 @@ func TestCustomSolidityErrors(t *testing.T) { if observedMessage != expectedMessage { Fatal(t, observedMessage) } + + arbosActs, err := precompilesgen.NewArbosActs(types.ArbosAddress, builder.L2.Client) + Require(t, err) + _, err = arbosActs.StartBlock(&auth, big.NewInt(0), 0, 0, 0) + if err == nil { + Fatal(t, "StartBlock call should have errored") + } + observedMessage = err.Error() + expectedError = "CallerNotArbOS()" + expectedMessage = fmt.Sprintf("execution reverted: error %v: %v", expectedError, expectedError) + if observedMessage != expectedMessage { + Fatal(t, observedMessage) + } + + _, err = arbosActs.BatchPostingReport(&auth, big.NewInt(0), common.Address{}, 0, 0, big.NewInt(0)) + if err == nil { + Fatal(t, "BatchPostingReport call should have errored") + } + observedMessage = err.Error() + expectedError = "CallerNotArbOS()" + expectedMessage = fmt.Sprintf("execution reverted: error %v: %v", expectedError, expectedError) + if observedMessage != expectedMessage { + Fatal(t, observedMessage) + } } func TestPrecompileErrorGasLeft(t *testing.T) { From 4ff5480fcd1b519b41f0ce097abf8aed58148a3a Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 14 Oct 2024 11:33:57 -0300 Subject: [PATCH 434/489] Refactors TestCustomSolidityErrors --- system_tests/precompile_test.go | 96 ++++++++++++++++----------------- 1 file changed, 45 insertions(+), 51 deletions(-) diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index 1e3c1adcee..eabbb60216 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -81,36 +81,42 @@ func TestCustomSolidityErrors(t *testing.T) { callOpts := &bind.CallOpts{Context: ctx} auth := builder.L2Info.GetDefaultTransactOpts("Owner", ctx) + ensure := func( + customError error, + expectedError string, + scenario string, + ) { + if customError == nil { + Fatal(t, "should have errored", "scenario", scenario) + } + observedMessage := customError.Error() + // The first error is server side. The second error is client side ABI decoding. + expectedMessage := fmt.Sprintf("execution reverted: error %v: %v", expectedError, expectedError) + if observedMessage != expectedMessage { + Fatal(t, observedMessage, "scenario", scenario) + } + } + arbDebug, err := precompilesgen.NewArbDebug(common.HexToAddress("0xff"), builder.L2.Client) Require(t, err, "could not bind ArbDebug contract") - customError := arbDebug.CustomRevert(callOpts, 1024) - if customError == nil { - Fatal(t, "customRevert call should have errored") - } - observedMessage := customError.Error() - expectedError := "Custom(1024, This spider family wards off bugs: /\\oo/\\ //\\(oo)//\\ /\\oo/\\, true)" - // The first error is server side. The second error is client side ABI decoding. - expectedMessage := fmt.Sprintf("execution reverted: error %v: %v", expectedError, expectedError) - if observedMessage != expectedMessage { - Fatal(t, observedMessage) - } + ensure( + arbDebug.CustomRevert(callOpts, 1024), + "Custom(1024, This spider family wards off bugs: /\\oo/\\ //\\(oo)//\\ /\\oo/\\, true)", + "arbDebug.CustomRevert", + ) arbSys, err := precompilesgen.NewArbSys(arbos.ArbSysAddress, builder.L2.Client) Require(t, err, "could not bind ArbSys contract") - _, customError = arbSys.ArbBlockHash(callOpts, big.NewInt(1e9)) - if customError == nil { - Fatal(t, "out of range ArbBlockHash call should have errored") - } - observedMessage = customError.Error() - expectedError = "InvalidBlockNumber(1000000000, 1)" - expectedMessage = fmt.Sprintf("execution reverted: error %v: %v", expectedError, expectedError) - if observedMessage != expectedMessage { - Fatal(t, observedMessage) - } + _, customError := arbSys.ArbBlockHash(callOpts, big.NewInt(1e9)) + ensure( + customError, + "InvalidBlockNumber(1000000000, 1)", + "arbSys.ArbBlockHash", + ) arbRetryableTx, err := precompilesgen.NewArbRetryableTx(common.HexToAddress("6e"), builder.L2.Client) Require(t, err) - _, err = arbRetryableTx.SubmitRetryable( + _, customError = arbRetryableTx.SubmitRetryable( &auth, [32]byte{}, big.NewInt(0), @@ -124,39 +130,27 @@ func TestCustomSolidityErrors(t *testing.T) { common.Address{}, []byte{}, ) - if err == nil { - Fatal(t, "SubmitRetryable call should have errored") - } - observedMessage = err.Error() - expectedError = "NotCallable()" - expectedMessage = fmt.Sprintf("execution reverted: error %v: %v", expectedError, expectedError) - if observedMessage != expectedMessage { - Fatal(t, observedMessage) - } + ensure( + customError, + "NotCallable()", + "arbRetryableTx.SubmitRetryable", + ) arbosActs, err := precompilesgen.NewArbosActs(types.ArbosAddress, builder.L2.Client) Require(t, err) - _, err = arbosActs.StartBlock(&auth, big.NewInt(0), 0, 0, 0) - if err == nil { - Fatal(t, "StartBlock call should have errored") - } - observedMessage = err.Error() - expectedError = "CallerNotArbOS()" - expectedMessage = fmt.Sprintf("execution reverted: error %v: %v", expectedError, expectedError) - if observedMessage != expectedMessage { - Fatal(t, observedMessage) - } + _, customError = arbosActs.StartBlock(&auth, big.NewInt(0), 0, 0, 0) + ensure( + customError, + "CallerNotArbOS()", + "arbosActs.StartBlock", + ) - _, err = arbosActs.BatchPostingReport(&auth, big.NewInt(0), common.Address{}, 0, 0, big.NewInt(0)) - if err == nil { - Fatal(t, "BatchPostingReport call should have errored") - } - observedMessage = err.Error() - expectedError = "CallerNotArbOS()" - expectedMessage = fmt.Sprintf("execution reverted: error %v: %v", expectedError, expectedError) - if observedMessage != expectedMessage { - Fatal(t, observedMessage) - } + _, customError = arbosActs.BatchPostingReport(&auth, big.NewInt(0), common.Address{}, 0, 0, big.NewInt(0)) + ensure( + customError, + "CallerNotArbOS()", + "arbosActs.BatchPostingReport", + ) } func TestPrecompileErrorGasLeft(t *testing.T) { From 54283080f2ee010927c5dafa433c5ee0d8f1814c Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 14 Oct 2024 11:44:41 -0300 Subject: [PATCH 435/489] TestArbDebugLegacyError --- precompiles/ArbDebug.go | 1 - system_tests/precompile_test.go | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/precompiles/ArbDebug.go b/precompiles/ArbDebug.go index 9bfc8c9815..d8b4ad0f8e 100644 --- a/precompiles/ArbDebug.go +++ b/precompiles/ArbDebug.go @@ -62,7 +62,6 @@ func (con ArbDebug) Panic(c ctx, evm mech) error { panic("called ArbDebug's debug-only Panic method") } -// TODO: add system test func (con ArbDebug) LegacyError(c ctx) error { return errors.New("example legacy error") } diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index eabbb60216..b4074171cb 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -70,6 +70,25 @@ func TestViewLogReverts(t *testing.T) { } } +func TestArbDebugLegacyError(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, false) + cleanup := builder.Build(t) + defer cleanup() + + callOpts := &bind.CallOpts{Context: ctx} + + arbDebug, err := precompilesgen.NewArbDebug(common.HexToAddress("0xff"), builder.L2.Client) + Require(t, err) + + err = arbDebug.LegacyError(callOpts) + if err == nil { + Fatal(t, "unexpected success") + } +} + func TestCustomSolidityErrors(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() From bfd0d0d2dd40c5655e9e5e8962c64864afd4fa90 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 14 Oct 2024 12:10:45 -0300 Subject: [PATCH 436/489] TestArbDebugPanic --- precompiles/ArbDebug.go | 1 - system_tests/precompile_test.go | 22 ++++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/precompiles/ArbDebug.go b/precompiles/ArbDebug.go index d8b4ad0f8e..bf85d5e18f 100644 --- a/precompiles/ArbDebug.go +++ b/precompiles/ArbDebug.go @@ -56,7 +56,6 @@ func (con ArbDebug) BecomeChainOwner(c ctx, evm mech) error { return c.State.ChainOwners().Add(c.caller) } -// TODO: add system test // Halts the chain by panicking in the STF func (con ArbDebug) Panic(c ctx, evm mech) error { panic("called ArbDebug's debug-only Panic method") diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index b4074171cb..426915c7d9 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -70,6 +70,28 @@ func TestViewLogReverts(t *testing.T) { } } +func TestArbDebugPanic(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, false) + cleanup := builder.Build(t) + defer cleanup() + + auth := builder.L2Info.GetDefaultTransactOpts("Owner", ctx) + + arbDebug, err := precompilesgen.NewArbDebug(common.HexToAddress("0xff"), builder.L2.Client) + Require(t, err) + + _, err = arbDebug.Panic(&auth) + if err == nil { + Fatal(t, "unexpected success") + } + if err.Error() != "method handler crashed" { + Fatal(t, "expected method handler to crash") + } +} + func TestArbDebugLegacyError(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() From 11996aa5c94cbd295675239eb030be4ee4a2923d Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 14 Oct 2024 13:15:12 -0300 Subject: [PATCH 437/489] TestArbSysDoesntRevert --- precompiles/ArbSys.go | 1 - system_tests/precompile_test.go | 21 +++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/precompiles/ArbSys.go b/precompiles/ArbSys.go index f48573fafe..149a95ec1f 100644 --- a/precompiles/ArbSys.go +++ b/precompiles/ArbSys.go @@ -75,7 +75,6 @@ func (con *ArbSys) IsTopLevelCall(c ctx, evm mech) (bool, error) { return evm.Depth() <= 2, nil } -// TODO: add system test // MapL1SenderContractAddressToL2Alias gets the contract's L2 alias func (con *ArbSys) MapL1SenderContractAddressToL2Alias(c ctx, sender addr, dest addr) (addr, error) { return util.RemapL1Address(sender), nil diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index 426915c7d9..8b90836635 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -610,6 +610,27 @@ func TestArbFunctionTable(t *testing.T) { } } +func TestArbSysDoesntRevert(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, false) + cleanup := builder.Build(t) + defer cleanup() + + callOpts := &bind.CallOpts{Context: ctx} + + arbSys, err := precompilesgen.NewArbSys(types.ArbSysAddress, builder.L2.Client) + Require(t, err) + + addr1 := common.BytesToAddress(crypto.Keccak256([]byte{})[:20]) + addr2 := common.BytesToAddress(crypto.Keccak256([]byte{})[:20]) + _, err = arbSys.MapL1SenderContractAddressToL2Alias(callOpts, addr1, addr2) + Require(t, err) +} + func TestArbGasInfoDoesntRevert(t *testing.T) { t.Parallel() From 4e2be7ec130f8bd9c906ba348953b10d08b49613 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 14 Oct 2024 13:18:32 -0300 Subject: [PATCH 438/489] Uses ArbSys.SendTxToL1 in a system test --- precompiles/ArbSys.go | 1 - system_tests/debugapi_test.go | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/precompiles/ArbSys.go b/precompiles/ArbSys.go index 149a95ec1f..689d3b18de 100644 --- a/precompiles/ArbSys.go +++ b/precompiles/ArbSys.go @@ -105,7 +105,6 @@ func (con *ArbSys) MyCallersAddressWithoutAliasing(c ctx, evm mech) (addr, error return address, err } -// TODO: add system test // SendTxToL1 sends a transaction to L1, adding it to the outbox func (con *ArbSys) SendTxToL1(c ctx, evm mech, value huge, destination addr, calldataForL1 []byte) (huge, error) { l1BlockNum, err := c.txProcessor.L1BlockNumber(vm.BlockContext{}) diff --git a/system_tests/debugapi_test.go b/system_tests/debugapi_test.go index 30a2bee03e..eb2bcd095d 100644 --- a/system_tests/debugapi_test.go +++ b/system_tests/debugapi_test.go @@ -43,7 +43,7 @@ func TestDebugAPI(t *testing.T) { arbSys, err := precompilesgen.NewArbSys(types.ArbSysAddress, builder.L2.Client) Require(t, err) auth := builder.L2Info.GetDefaultTransactOpts("Owner", ctx) - tx, err := arbSys.WithdrawEth(&auth, common.Address{}) + tx, err := arbSys.SendTxToL1(&auth, common.Address{}, []byte{}) Require(t, err) receipt, err := builder.L2.EnsureTxSucceeded(tx) Require(t, err) From 3927b8f8bb26e012bc6222c98f6dc6425621d37c Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 14 Oct 2024 13:22:33 -0300 Subject: [PATCH 439/489] Test for ArbosTest --- precompiles/ArbosTest.go | 1 - system_tests/precompile_test.go | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/precompiles/ArbosTest.go b/precompiles/ArbosTest.go index 8a770ea339..6e988f533a 100644 --- a/precompiles/ArbosTest.go +++ b/precompiles/ArbosTest.go @@ -12,7 +12,6 @@ type ArbosTest struct { Address addr // 0x69 } -// TODO: add system test // BurnArbGas unproductively burns the amount of L2 ArbGas func (con ArbosTest) BurnArbGas(c ctx, gasAmount huge) error { if !gasAmount.IsUint64() { diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index 8b90836635..7ea0cdbf4c 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -610,6 +610,25 @@ func TestArbFunctionTable(t *testing.T) { } } +func TestArbosTestDoesntRevert(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, false) + cleanup := builder.Build(t) + defer cleanup() + + callOpts := &bind.CallOpts{Context: ctx} + + arbosTest, err := precompilesgen.NewArbosTest(types.ArbosTestAddress, builder.L2.Client) + Require(t, err) + + err = arbosTest.BurnArbGas(callOpts, big.NewInt(1)) + Require(t, err) +} + func TestArbSysDoesntRevert(t *testing.T) { t.Parallel() From 5bd7a50b5a83f3f9a49e259f14bac90003717ae2 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 14 Oct 2024 15:28:22 -0300 Subject: [PATCH 440/489] TestArbAggregatorBatchPosters as a system test --- precompiles/ArbAggregator.go | 1 - precompiles/ArbAggregator_test.go | 28 ------------------ system_tests/precompile_test.go | 49 +++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 29 deletions(-) diff --git a/precompiles/ArbAggregator.go b/precompiles/ArbAggregator.go index c3ee19c12c..9ed9c94cf4 100644 --- a/precompiles/ArbAggregator.go +++ b/precompiles/ArbAggregator.go @@ -38,7 +38,6 @@ func (con ArbAggregator) GetBatchPosters(c ctx, evm mech) ([]addr, error) { return c.State.L1PricingState().BatchPosterTable().AllPosters(65536) } -// TODO: add system test func (con ArbAggregator) AddBatchPoster(c ctx, evm mech, newBatchPoster addr) error { isOwner, err := c.State.ChainOwners().IsMember(c.caller) if err != nil { diff --git a/precompiles/ArbAggregator_test.go b/precompiles/ArbAggregator_test.go index ce1cebde5d..879fc737e4 100644 --- a/precompiles/ArbAggregator_test.go +++ b/precompiles/ArbAggregator_test.go @@ -12,34 +12,6 @@ import ( "github.com/offchainlabs/nitro/arbos/l1pricing" ) -func TestArbAggregatorBatchPosters(t *testing.T) { - evm := newMockEVMForTesting() - context := testContext(common.Address{}, evm) - - addr := common.BytesToAddress(crypto.Keccak256([]byte{})[:20]) - - // initially should have one batch poster - bps, err := ArbAggregator{}.GetBatchPosters(context, evm) - Require(t, err) - if len(bps) != 1 { - Fail(t) - } - - // add addr as a batch poster - Require(t, ArbDebug{}.BecomeChainOwner(context, evm)) - Require(t, ArbAggregator{}.AddBatchPoster(context, evm, addr)) - - // there should now be two batch posters, and addr should be one of them - bps, err = ArbAggregator{}.GetBatchPosters(context, evm) - Require(t, err) - if len(bps) != 2 { - Fail(t) - } - if bps[0] != addr && bps[1] != addr { - Fail(t) - } -} - func TestFeeCollector(t *testing.T) { evm := newMockEVMForTesting() agg := ArbAggregator{} diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index 7ea0cdbf4c..3406637ed5 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -696,3 +696,52 @@ func TestArbGasInfoDoesntRevert(t *testing.T) { _, _, _, _, _, _, err = arbGasInfo.GetPricesInWeiWithAggregator(callOpts, addr) Require(t, err) } + +func TestArbAggregatorBatchPosters(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, false) + cleanup := builder.Build(t) + defer cleanup() + + auth := builder.L2Info.GetDefaultTransactOpts("Owner", ctx) + callOpts := &bind.CallOpts{Context: ctx} + + arbAggregator, err := precompilesgen.NewArbAggregator(types.ArbAggregatorAddress, builder.L2.Client) + Require(t, err) + + arbDebug, err := precompilesgen.NewArbDebug(types.ArbDebugAddress, builder.L2.Client) + Require(t, err) + + addr := common.BytesToAddress(crypto.Keccak256([]byte{})[:20]) + + // initially should have one batch poster + bps, err := arbAggregator.GetBatchPosters(callOpts) + Require(t, err) + if len(bps) != 1 { + Fatal(t, "expected one batch poster") + } + + // add addr as a batch poster + tx, err := arbDebug.BecomeChainOwner(&auth) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + tx, err = arbAggregator.AddBatchPoster(&auth, addr) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + + // there should now be two batch posters, and addr should be one of them + bps, err = arbAggregator.GetBatchPosters(callOpts) + Require(t, err) + if len(bps) != 2 { + Fatal(t, "expected two batch posters") + } + if bps[0] != addr && bps[1] != addr { + Fatal(t, "expected addr to be a batch poster") + } +} From 5927380d6f90fc13d38f06cb0fa1cbcbc5791571 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 14 Oct 2024 15:32:14 -0300 Subject: [PATCH 441/489] TestArbAggregatorGetPreferredAggregator --- precompiles/ArbAggregator.go | 2 -- system_tests/precompile_test.go | 34 +++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/precompiles/ArbAggregator.go b/precompiles/ArbAggregator.go index 9ed9c94cf4..9591c001d9 100644 --- a/precompiles/ArbAggregator.go +++ b/precompiles/ArbAggregator.go @@ -19,14 +19,12 @@ type ArbAggregator struct { var ErrNotOwner = errors.New("must be called by chain owner") -// TODO: add system test // GetPreferredAggregator returns the preferred aggregator address. // Deprecated: Do not use this method. func (con ArbAggregator) GetPreferredAggregator(c ctx, evm mech, address addr) (prefAgg addr, isDefault bool, err error) { return l1pricing.BatchPosterAddress, true, err } -// TODO: add system test // GetDefaultAggregator returns the default aggregator address. // Deprecated: Do not use this method. func (con ArbAggregator) GetDefaultAggregator(c ctx, evm mech) (addr, error) { diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index 3406637ed5..bd61fa3982 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -15,6 +15,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" "github.com/offchainlabs/nitro/arbos" + "github.com/offchainlabs/nitro/arbos/l1pricing" "github.com/offchainlabs/nitro/solgen/go/mocksgen" "github.com/offchainlabs/nitro/solgen/go/precompilesgen" "github.com/offchainlabs/nitro/util/arbmath" @@ -745,3 +746,36 @@ func TestArbAggregatorBatchPosters(t *testing.T) { Fatal(t, "expected addr to be a batch poster") } } + +func TestArbAggregatorGetPreferredAggregator(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, false) + cleanup := builder.Build(t) + defer cleanup() + + callOpts := &bind.CallOpts{Context: ctx} + + arbAggregator, err := precompilesgen.NewArbAggregator(types.ArbAggregatorAddress, builder.L2.Client) + Require(t, err) + + addr := common.BytesToAddress(crypto.Keccak256([]byte{})[:20]) + + prefAgg, isDefault, err := arbAggregator.GetPreferredAggregator(callOpts, addr) + Require(t, err) + if !isDefault { + Fatal(t, "expected default preferred aggregator") + } + if prefAgg != l1pricing.BatchPosterAddress { + Fatal(t, "expected default preferred aggregator to be", l1pricing.BatchPosterAddress, "got", prefAgg) + } + + prefAgg, err = arbAggregator.GetDefaultAggregator(callOpts) + Require(t, err) + if prefAgg != l1pricing.BatchPosterAddress { + Fatal(t, "expected default preferred aggregator to be", l1pricing.BatchPosterAddress, "got", prefAgg) + } +} From 82f21dea10f68156e89a5c9ca85b5599c8b06cef Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 14 Oct 2024 15:45:52 -0300 Subject: [PATCH 442/489] TestArbAggregatorBaseFee --- precompiles/ArbAggregator.go | 2 -- system_tests/precompile_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/precompiles/ArbAggregator.go b/precompiles/ArbAggregator.go index 9591c001d9..d515eb375a 100644 --- a/precompiles/ArbAggregator.go +++ b/precompiles/ArbAggregator.go @@ -91,14 +91,12 @@ func (con ArbAggregator) SetFeeCollector(c ctx, evm mech, batchPoster addr, newF return posterInfo.SetPayTo(newFeeCollector) } -// TODO: add system test // GetTxBaseFee gets an aggregator's current fixed fee to submit a tx func (con ArbAggregator) GetTxBaseFee(c ctx, evm mech, aggregator addr) (huge, error) { // This is deprecated and now always returns zero. return big.NewInt(0), nil } -// TODO: add system test // SetTxBaseFee sets an aggregator's fixed fee (caller must be the aggregator, its fee collector, or an owner) func (con ArbAggregator) SetTxBaseFee(c ctx, evm mech, aggregator addr, feeInL1Gas huge) error { // This is deprecated and is now a no-op. diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index bd61fa3982..80bc2abdc4 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -611,6 +611,34 @@ func TestArbFunctionTable(t *testing.T) { } } +func TestArbAggregatorBaseFee(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, false) + cleanup := builder.Build(t) + defer cleanup() + + auth := builder.L2Info.GetDefaultTransactOpts("Owner", ctx) + callOpts := &bind.CallOpts{Context: ctx} + + arbAggregator, err := precompilesgen.NewArbAggregator(types.ArbAggregatorAddress, builder.L2.Client) + Require(t, err) + + tx, err := arbAggregator.SetTxBaseFee(&auth, common.Address{}, big.NewInt(1)) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + + fee, err := arbAggregator.GetTxBaseFee(callOpts, common.Address{}) + Require(t, err) + if fee.Cmp(big.NewInt(0)) != 0 { + Fatal(t, "expected fee to be 0, got", fee) + } +} + func TestArbosTestDoesntRevert(t *testing.T) { t.Parallel() From ba50c008ec54503f2630d08aaffa113b2459020d Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 14 Oct 2024 15:51:03 -0300 Subject: [PATCH 443/489] TestArbAggregatorDoesntRevert --- precompiles/ArbAggregator.go | 2 -- system_tests/precompile_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/precompiles/ArbAggregator.go b/precompiles/ArbAggregator.go index d515eb375a..b74e280fe8 100644 --- a/precompiles/ArbAggregator.go +++ b/precompiles/ArbAggregator.go @@ -58,7 +58,6 @@ func (con ArbAggregator) AddBatchPoster(c ctx, evm mech, newBatchPoster addr) er return nil } -// TODO: add system test // GetFeeCollector gets a batch poster's fee collector func (con ArbAggregator) GetFeeCollector(c ctx, evm mech, batchPoster addr) (addr, error) { posterInfo, err := c.State.L1PricingState().BatchPosterTable().OpenPoster(batchPoster, false) @@ -68,7 +67,6 @@ func (con ArbAggregator) GetFeeCollector(c ctx, evm mech, batchPoster addr) (add return posterInfo.PayTo() } -// TODO: add system test // SetFeeCollector sets a batch poster's fee collector (caller must be the batch poster, its fee collector, or an owner) func (con ArbAggregator) SetFeeCollector(c ctx, evm mech, batchPoster addr, newFeeCollector addr) error { posterInfo, err := c.State.L1PricingState().BatchPosterTable().OpenPoster(batchPoster, false) diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index 80bc2abdc4..cc4cf4195f 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -639,6 +639,31 @@ func TestArbAggregatorBaseFee(t *testing.T) { } } +func TestArbAggregatorDoesntRevert(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, false) + cleanup := builder.Build(t) + defer cleanup() + + auth := builder.L2Info.GetDefaultTransactOpts("Owner", ctx) + callOpts := &bind.CallOpts{Context: ctx} + + arbAggregator, err := precompilesgen.NewArbAggregator(types.ArbAggregatorAddress, builder.L2.Client) + Require(t, err) + + tx, err := arbAggregator.SetFeeCollector(&auth, l1pricing.BatchPosterAddress, common.Address{}) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + + _, err = arbAggregator.GetFeeCollector(callOpts, l1pricing.BatchPosterAddress) + Require(t, err) +} + func TestArbosTestDoesntRevert(t *testing.T) { t.Parallel() From 3a75847a6954aea1445be7cda2288dc4296c15d7 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 14 Oct 2024 16:15:50 -0300 Subject: [PATCH 444/489] TestArbOwnerPublicGetAllChainOwners --- precompiles/ArbOwnerPublic.go | 2 -- system_tests/precompile_test.go | 35 +++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/precompiles/ArbOwnerPublic.go b/precompiles/ArbOwnerPublic.go index d901c77f12..451e18e1cc 100644 --- a/precompiles/ArbOwnerPublic.go +++ b/precompiles/ArbOwnerPublic.go @@ -16,13 +16,11 @@ type ArbOwnerPublic struct { ChainOwnerRectifiedGasCost func(addr) (uint64, error) } -// TODO: add system test // GetAllChainOwners retrieves the list of chain owners func (con ArbOwnerPublic) GetAllChainOwners(c ctx, evm mech) ([]common.Address, error) { return c.State.ChainOwners().AllMembers(65536) } -// TODO: add system test // RectifyChainOwner checks if the account is a chain owner func (con ArbOwnerPublic) RectifyChainOwner(c ctx, evm mech, addr addr) error { err := c.State.ChainOwners().RectifyMapping(addr) diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index cc4cf4195f..dc8cb842a3 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -704,6 +704,41 @@ func TestArbSysDoesntRevert(t *testing.T) { Require(t, err) } +func TestArbOwnerPublicGetAllChainOwners(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, false) + cleanup := builder.Build(t) + defer cleanup() + + auth := builder.L2Info.GetDefaultTransactOpts("Owner", ctx) + callOpts := &bind.CallOpts{Context: ctx} + + arbOwnerPublic, err := precompilesgen.NewArbOwnerPublic(types.ArbOwnerPublicAddress, builder.L2.Client) + Require(t, err) + + ownerAddr := builder.L2Info.GetAddress("Owner") + chainOwners, err := arbOwnerPublic.GetAllChainOwners(callOpts) + Require(t, err) + chainOwnerInChainOwners := false + for _, chainOwner := range chainOwners { + if chainOwner.Cmp(ownerAddr) == 0 { + chainOwnerInChainOwners = true + } + } + if !chainOwnerInChainOwners { + Fatal(t, "expected owner to be in chain owners") + } + + _, err = arbOwnerPublic.RectifyChainOwner(&auth, ownerAddr) + if (err == nil) || (err.Error() != "execution reverted") { + Fatal(t, "expected rectify chain owner to revert since it is already an owner") + } +} + func TestArbGasInfoDoesntRevert(t *testing.T) { t.Parallel() From b1d81100cd382c34cf17fb2c49ae6d080799b117 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 15 Oct 2024 10:16:35 -0300 Subject: [PATCH 445/489] TestArbAddressTableDoesntRevert --- precompiles/ArbAddressTable.go | 7 ----- system_tests/precompile_test.go | 54 +++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 7 deletions(-) diff --git a/precompiles/ArbAddressTable.go b/precompiles/ArbAddressTable.go index 45b708bd7d..102fd55c3b 100644 --- a/precompiles/ArbAddressTable.go +++ b/precompiles/ArbAddressTable.go @@ -13,19 +13,16 @@ type ArbAddressTable struct { Address addr // 0x66 } -// TODO: add system test // AddressExists checks if an address exists in the table func (con ArbAddressTable) AddressExists(c ctx, evm mech, addr addr) (bool, error) { return c.State.AddressTable().AddressExists(addr) } -// TODO: add system test // Compress and returns the bytes that represent the address func (con ArbAddressTable) Compress(c ctx, evm mech, addr addr) ([]uint8, error) { return c.State.AddressTable().Compress(addr) } -// TODO: add system test // Decompress the compressed bytes at the given offset with those of the corresponding account func (con ArbAddressTable) Decompress(c ctx, evm mech, buf []uint8, offset huge) (addr, huge, error) { if !offset.IsInt64() { @@ -39,7 +36,6 @@ func (con ArbAddressTable) Decompress(c ctx, evm mech, buf []uint8, offset huge) return result, new(big.Int).SetUint64(nbytes), err } -// TODO: add system test // Lookup the index of an address in the table func (con ArbAddressTable) Lookup(c ctx, evm mech, addr addr) (huge, error) { result, exists, err := c.State.AddressTable().Lookup(addr) @@ -52,7 +48,6 @@ func (con ArbAddressTable) Lookup(c ctx, evm mech, addr addr) (huge, error) { return new(big.Int).SetUint64(result), nil } -// TODO: add system test // LookupIndex for an address in the table by index func (con ArbAddressTable) LookupIndex(c ctx, evm mech, index huge) (addr, error) { if !index.IsUint64() { @@ -68,14 +63,12 @@ func (con ArbAddressTable) LookupIndex(c ctx, evm mech, index huge) (addr, error return result, nil } -// TODO: add system test // Register adds an account to the table, shrinking its compressed representation func (con ArbAddressTable) Register(c ctx, evm mech, addr addr) (huge, error) { slot, err := c.State.AddressTable().Register(addr) return new(big.Int).SetUint64(slot), err } -// TODO: add system test // Size gets the number of addresses in the table func (con ArbAddressTable) Size(c ctx, evm mech) (huge, error) { size, err := c.State.AddressTable().Size() diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index dc8cb842a3..5d5cfe69db 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -639,6 +639,60 @@ func TestArbAggregatorBaseFee(t *testing.T) { } } +func TestArbAddressTableDoesntRevert(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, false) + cleanup := builder.Build(t) + defer cleanup() + + auth := builder.L2Info.GetDefaultTransactOpts("Owner", ctx) + callOpts := &bind.CallOpts{Context: ctx} + + arbAddressTable, err := precompilesgen.NewArbAddressTable(types.ArbAddressTableAddress, builder.L2.Client) + Require(t, err) + + addr := common.BytesToAddress(crypto.Keccak256([]byte{})[:20]) + + exists, err := arbAddressTable.AddressExists(callOpts, addr) + Require(t, err) + if exists { + Fatal(t, "expected address to not exist") + } + + tx, err := arbAddressTable.Register(&auth, addr) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + + idx, err := arbAddressTable.Lookup(callOpts, addr) + Require(t, err) + + retrievedAddr, err := arbAddressTable.LookupIndex(callOpts, idx) + Require(t, err) + if retrievedAddr.Cmp(addr) != 0 { + Fatal(t, "expected retrieved address to be", addr, "got", retrievedAddr) + } + + size, err := arbAddressTable.Size(callOpts) + Require(t, err) + if size.Cmp(big.NewInt(1)) != 0 { + Fatal(t, "expected size to be 1, got", size) + } + + tx, err = arbAddressTable.Compress(&auth, addr) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + + res := []uint8{128} + _, _, err = arbAddressTable.Decompress(callOpts, res, big.NewInt(0)) + Require(t, err) +} + func TestArbAggregatorDoesntRevert(t *testing.T) { t.Parallel() From a5435b225843368584da0106f1e6b81e2e71e3ec Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 15 Oct 2024 10:37:00 -0300 Subject: [PATCH 446/489] System test for ArbOwner.GetAllChainOwners --- precompiles/ArbOwner.go | 1 - system_tests/precompile_test.go | 27 +++++++++++++++++++++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/precompiles/ArbOwner.go b/precompiles/ArbOwner.go index 7387b62a62..4f4f01d45c 100644 --- a/precompiles/ArbOwner.go +++ b/precompiles/ArbOwner.go @@ -54,7 +54,6 @@ func (con ArbOwner) IsChainOwner(c ctx, evm mech, addr addr) (bool, error) { return c.State.ChainOwners().IsMember(addr) } -// TODO: add system test // GetAllChainOwners retrieves the list of chain owners func (con ArbOwner) GetAllChainOwners(c ctx, evm mech) ([]common.Address, error) { return c.State.ChainOwners().AllMembers(65536) diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index 5d5cfe69db..5aa648cc21 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "math/big" + "sort" "testing" "github.com/ethereum/go-ethereum/accounts/abi/bind" @@ -758,7 +759,7 @@ func TestArbSysDoesntRevert(t *testing.T) { Require(t, err) } -func TestArbOwnerPublicGetAllChainOwners(t *testing.T) { +func TestGetAllChainOwners(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(context.Background()) @@ -774,11 +775,29 @@ func TestArbOwnerPublicGetAllChainOwners(t *testing.T) { arbOwnerPublic, err := precompilesgen.NewArbOwnerPublic(types.ArbOwnerPublicAddress, builder.L2.Client) Require(t, err) - ownerAddr := builder.L2Info.GetAddress("Owner") - chainOwners, err := arbOwnerPublic.GetAllChainOwners(callOpts) + arbOwner, err := precompilesgen.NewArbOwner(types.ArbOwnerAddress, builder.L2.Client) + Require(t, err) + + chainOwnersArbOwnerPublic, err := arbOwnerPublic.GetAllChainOwners(callOpts) Require(t, err) + chainOwnersArbOwner, err := arbOwner.GetAllChainOwners(callOpts) + Require(t, err) + if len(chainOwnersArbOwnerPublic) != len(chainOwnersArbOwner) { + Fatal(t, "expected chain owners to be the same length") + } + // sort the chain owners to ensure they are in the same order + sort.Slice(chainOwnersArbOwnerPublic, func(i, j int) bool { + return chainOwnersArbOwnerPublic[i].Cmp(chainOwnersArbOwnerPublic[j]) < 0 + }) + for i := 0; i < len(chainOwnersArbOwnerPublic); i += 1 { + if chainOwnersArbOwnerPublic[i].Cmp(chainOwnersArbOwner[i]) != 0 { + Fatal(t, "expected chain owners to be the same") + } + } + + ownerAddr := builder.L2Info.GetAddress("Owner") chainOwnerInChainOwners := false - for _, chainOwner := range chainOwners { + for _, chainOwner := range chainOwnersArbOwner { if chainOwner.Cmp(ownerAddr) == 0 { chainOwnerInChainOwners = true } From 3fd92d33f1c0cd71b540c947f401545d8d439c9a Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 15 Oct 2024 10:47:05 -0300 Subject: [PATCH 447/489] TestChainOwners --- precompiles/ArbOwner.go | 2 -- system_tests/precompile_test.go | 34 +++++++++++++++++++++++++++------ 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/precompiles/ArbOwner.go b/precompiles/ArbOwner.go index 4f4f01d45c..487f93b049 100644 --- a/precompiles/ArbOwner.go +++ b/precompiles/ArbOwner.go @@ -33,13 +33,11 @@ var ( ErrOutOfBounds = errors.New("value out of bounds") ) -// TODO: add system test // AddChainOwner adds account as a chain owner func (con ArbOwner) AddChainOwner(c ctx, evm mech, newOwner addr) error { return c.State.ChainOwners().Add(newOwner) } -// TODO: add system test // RemoveChainOwner removes account from the list of chain owners func (con ArbOwner) RemoveChainOwner(c ctx, evm mech, addr addr) error { member, _ := con.IsChainOwner(c, evm, addr) diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index 5aa648cc21..1ac52c1883 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -759,7 +759,7 @@ func TestArbSysDoesntRevert(t *testing.T) { Require(t, err) } -func TestGetAllChainOwners(t *testing.T) { +func TestChainOwners(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(context.Background()) @@ -774,10 +774,22 @@ func TestGetAllChainOwners(t *testing.T) { arbOwnerPublic, err := precompilesgen.NewArbOwnerPublic(types.ArbOwnerPublicAddress, builder.L2.Client) Require(t, err) - arbOwner, err := precompilesgen.NewArbOwner(types.ArbOwnerAddress, builder.L2.Client) Require(t, err) + builder.L2Info.GenerateAccount("Owner2") + chainOwnerAddr2 := builder.L2Info.GetAddress("Owner2") + tx, err := arbOwner.AddChainOwner(&auth, chainOwnerAddr2) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + isChainOwner, err := arbOwnerPublic.IsChainOwner(callOpts, chainOwnerAddr2) + Require(t, err) + if !isChainOwner { + Fatal(t, "expected owner2 to be a chain owner") + } + + // check that the chain owners retrieved from arbOwnerPublic and arbOwner are the same chainOwnersArbOwnerPublic, err := arbOwnerPublic.GetAllChainOwners(callOpts) Require(t, err) chainOwnersArbOwner, err := arbOwner.GetAllChainOwners(callOpts) @@ -794,11 +806,10 @@ func TestGetAllChainOwners(t *testing.T) { Fatal(t, "expected chain owners to be the same") } } - - ownerAddr := builder.L2Info.GetAddress("Owner") + chainOwnerAddr := builder.L2Info.GetAddress("Owner") chainOwnerInChainOwners := false for _, chainOwner := range chainOwnersArbOwner { - if chainOwner.Cmp(ownerAddr) == 0 { + if chainOwner.Cmp(chainOwnerAddr) == 0 { chainOwnerInChainOwners = true } } @@ -806,7 +817,18 @@ func TestGetAllChainOwners(t *testing.T) { Fatal(t, "expected owner to be in chain owners") } - _, err = arbOwnerPublic.RectifyChainOwner(&auth, ownerAddr) + // remove chain owner 2 + tx, err = arbOwner.RemoveChainOwner(&auth, chainOwnerAddr2) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + isChainOwner, err = arbOwnerPublic.IsChainOwner(callOpts, chainOwnerAddr2) + Require(t, err) + if isChainOwner { + Fatal(t, "expected owner2 to not be a chain owner") + } + + _, err = arbOwnerPublic.RectifyChainOwner(&auth, chainOwnerAddr) if (err == nil) || (err.Error() != "execution reverted") { Fatal(t, "expected rectify chain owner to revert since it is already an owner") } From d3e08616c1ea280b49a07d66dcc7a1c868b1e126 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 15 Oct 2024 11:32:25 -0300 Subject: [PATCH 448/489] TestFeeAccounts --- precompiles/ArbOwner.go | 2 -- system_tests/precompile_test.go | 42 +++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/precompiles/ArbOwner.go b/precompiles/ArbOwner.go index 487f93b049..de67bca045 100644 --- a/precompiles/ArbOwner.go +++ b/precompiles/ArbOwner.go @@ -96,13 +96,11 @@ func (con ArbOwner) SetL2GasBacklogTolerance(c ctx, evm mech, sec uint64) error return c.State.L2PricingState().SetBacklogTolerance(sec) } -// TODO: add system test // GetNetworkFeeAccount gets the network fee collector func (con ArbOwner) GetNetworkFeeAccount(c ctx, evm mech) (addr, error) { return c.State.NetworkFeeAccount() } -// TODO: add system test // GetInfraFeeAccount gets the infrastructure fee collector func (con ArbOwner) GetInfraFeeAccount(c ctx, evm mech) (addr, error) { return c.State.InfraFeeAccount() diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index 1ac52c1883..a775b21e2c 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -640,6 +640,48 @@ func TestArbAggregatorBaseFee(t *testing.T) { } } +func TestFeeAccounts(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, false) + cleanup := builder.Build(t) + defer cleanup() + + auth := builder.L2Info.GetDefaultTransactOpts("Owner", ctx) + callOpts := &bind.CallOpts{Context: ctx} + + arbOwner, err := precompilesgen.NewArbOwner(types.ArbOwnerAddress, builder.L2.Client) + Require(t, err) + + builder.L2Info.GenerateAccount("User2") + addr := builder.L2Info.GetAddress("User2") + + tx, err := arbOwner.SetNetworkFeeAccount(&auth, addr) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + + feeAccount, err := arbOwner.GetNetworkFeeAccount(callOpts) + Require(t, err) + if feeAccount.Cmp(addr) != 0 { + Fatal(t, "expected fee account to be", addr, "got", feeAccount) + } + + tx, err = arbOwner.SetInfraFeeAccount(&auth, addr) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + + feeAccount, err = arbOwner.GetInfraFeeAccount(callOpts) + Require(t, err) + if feeAccount.Cmp(addr) != 0 { + Fatal(t, "expected fee account to be", addr, "got", feeAccount) + } +} + func TestArbAddressTableDoesntRevert(t *testing.T) { t.Parallel() From 0fd636c929777bb1b2d0bc45e6cdab284a1a5da3 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 15 Oct 2024 11:47:00 -0300 Subject: [PATCH 449/489] TestArbOwnerDoesntRevert --- precompiles/ArbOwner.go | 4 ---- system_tests/precompile_test.go | 41 +++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/precompiles/ArbOwner.go b/precompiles/ArbOwner.go index de67bca045..8b87445e0e 100644 --- a/precompiles/ArbOwner.go +++ b/precompiles/ArbOwner.go @@ -62,7 +62,6 @@ func (con ArbOwner) SetL1BaseFeeEstimateInertia(c ctx, evm mech, inertia uint64) return c.State.L1PricingState().SetInertia(inertia) } -// TODO: add system test // SetL2BaseFee sets the L2 gas price directly, bypassing the pool calculus func (con ArbOwner) SetL2BaseFee(c ctx, evm mech, priceInWei huge) error { return c.State.L2PricingState().SetBaseFeeWei(priceInWei) @@ -145,7 +144,6 @@ func (con ArbOwner) SetPerBatchGasCharge(c ctx, evm mech, cost int64) error { return c.State.L1PricingState().SetPerBatchGasCost(cost) } -// TODO: add system test func (con ArbOwner) SetAmortizedCostCapBips(c ctx, evm mech, cap uint64) error { return c.State.L1PricingState().SetAmortizedCostCapBips(cap) } @@ -154,7 +152,6 @@ func (con ArbOwner) SetBrotliCompressionLevel(c ctx, evm mech, level uint64) err return c.State.SetBrotliCompressionLevel(level) } -// TODO: add system test func (con ArbOwner) ReleaseL1PricerSurplusFunds(c ctx, evm mech, maxWeiToRelease huge) (huge, error) { balance := evm.StateDB.GetBalance(l1pricing.L1PricerFundsPoolAddress) l1p := c.State.L1PricingState() @@ -298,7 +295,6 @@ func (con ArbOwner) RemoveWasmCacheManager(c ctx, _ mech, manager addr) error { return managers.Remove(manager, c.State.ArbOSVersion()) } -// TODO: add system test func (con ArbOwner) SetChainConfig(c ctx, evm mech, serializedChainConfig []byte) error { if c == nil { return errors.New("nil context") diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index a775b21e2c..acdaa88af7 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -5,6 +5,7 @@ package arbtest import ( "context" + "encoding/json" "fmt" "math/big" "sort" @@ -876,6 +877,46 @@ func TestChainOwners(t *testing.T) { } } +func TestArbOwnerDoesntRevert(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, false) + cleanup := builder.Build(t) + defer cleanup() + + auth := builder.L2Info.GetDefaultTransactOpts("Owner", ctx) + + arbOwner, err := precompilesgen.NewArbOwner(types.ArbOwnerAddress, builder.L2.Client) + Require(t, err) + + chainConfig := params.ArbitrumDevTestChainConfig() + chainConfig.ArbitrumChainParams.MaxCodeSize = 100 + serializedChainConfig, err := json.Marshal(chainConfig) + Require(t, err) + tx, err := arbOwner.SetChainConfig(&auth, string(serializedChainConfig)) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + + tx, err = arbOwner.SetAmortizedCostCapBips(&auth, 77734) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + + tx, err = arbOwner.ReleaseL1PricerSurplusFunds(&auth, big.NewInt(1)) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + + tx, err = arbOwner.SetL2BaseFee(&auth, big.NewInt(1)) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) +} + func TestArbGasInfoDoesntRevert(t *testing.T) { t.Parallel() From 3e2fdfc6031d11744d9da1536f504507896681bb Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 15 Oct 2024 15:37:25 -0300 Subject: [PATCH 450/489] Commet about DoesntRevert tests --- system_tests/precompile_test.go | 18 ++++++++++++++++++ system_tests/retryable_test.go | 3 +++ 2 files changed, 21 insertions(+) diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index acdaa88af7..379d939835 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -683,6 +683,9 @@ func TestFeeAccounts(t *testing.T) { } } +// DoesntRevert tests are useful to check if precompile calls revert due to differences in the +// return types of a contract between go and solidity. +// They are not a substitute for unit tests, as they don't test the actual functionality of the precompile. func TestArbAddressTableDoesntRevert(t *testing.T) { t.Parallel() @@ -737,6 +740,9 @@ func TestArbAddressTableDoesntRevert(t *testing.T) { Require(t, err) } +// DoesntRevert tests are useful to check if precompile calls revert due to differences in the +// return types of a contract between go and solidity. +// They are not a substitute for unit tests, as they don't test the actual functionality of the precompile. func TestArbAggregatorDoesntRevert(t *testing.T) { t.Parallel() @@ -762,6 +768,9 @@ func TestArbAggregatorDoesntRevert(t *testing.T) { Require(t, err) } +// DoesntRevert tests are useful to check if precompile calls revert due to differences in the +// return types of a contract between go and solidity. +// They are not a substitute for unit tests, as they don't test the actual functionality of the precompile. func TestArbosTestDoesntRevert(t *testing.T) { t.Parallel() @@ -781,6 +790,9 @@ func TestArbosTestDoesntRevert(t *testing.T) { Require(t, err) } +// DoesntRevert tests are useful to check if precompile calls revert due to differences in the +// return types of a contract between go and solidity. +// They are not a substitute for unit tests, as they don't test the actual functionality of the precompile. func TestArbSysDoesntRevert(t *testing.T) { t.Parallel() @@ -877,6 +889,9 @@ func TestChainOwners(t *testing.T) { } } +// DoesntRevert tests are useful to check if precompile calls revert due to differences in the +// return types of a contract between go and solidity. +// They are not a substitute for unit tests, as they don't test the actual functionality of the precompile. func TestArbOwnerDoesntRevert(t *testing.T) { t.Parallel() @@ -917,6 +932,9 @@ func TestArbOwnerDoesntRevert(t *testing.T) { Require(t, err) } +// DoesntRevert tests are useful to check if precompile calls revert due to differences in the +// return types of a contract between go and solidity. +// They are not a substitute for unit tests, as they don't test the actual functionality of the precompile. func TestArbGasInfoDoesntRevert(t *testing.T) { t.Parallel() diff --git a/system_tests/retryable_test.go b/system_tests/retryable_test.go index 070902c155..5c90d3c7d5 100644 --- a/system_tests/retryable_test.go +++ b/system_tests/retryable_test.go @@ -445,6 +445,9 @@ func TestGetLifetime(t *testing.T) { } } +// DoesntRevert tests are useful to check if precompile calls revert due to differences in the +// return types of a contract between go and solidity. +// They are not a substitute for unit tests, as they don't test the actual functionality of the precompile. func TestArbRetryableTxDoesntRevert(t *testing.T) { t.Parallel() From c3b667735468027063b841144a9b5d3ba90f23f2 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 15 Oct 2024 15:56:36 -0300 Subject: [PATCH 451/489] Reuse precompile addresses defined in types --- system_tests/precompile_test.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index 379d939835..cdf63ffc98 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -83,7 +83,7 @@ func TestArbDebugPanic(t *testing.T) { auth := builder.L2Info.GetDefaultTransactOpts("Owner", ctx) - arbDebug, err := precompilesgen.NewArbDebug(common.HexToAddress("0xff"), builder.L2.Client) + arbDebug, err := precompilesgen.NewArbDebug(types.ArbDebugAddress, builder.L2.Client) Require(t, err) _, err = arbDebug.Panic(&auth) @@ -141,7 +141,7 @@ func TestCustomSolidityErrors(t *testing.T) { } } - arbDebug, err := precompilesgen.NewArbDebug(common.HexToAddress("0xff"), builder.L2.Client) + arbDebug, err := precompilesgen.NewArbDebug(types.ArbDebugAddress, builder.L2.Client) Require(t, err, "could not bind ArbDebug contract") ensure( arbDebug.CustomRevert(callOpts, 1024), @@ -158,7 +158,7 @@ func TestCustomSolidityErrors(t *testing.T) { "arbSys.ArbBlockHash", ) - arbRetryableTx, err := precompilesgen.NewArbRetryableTx(common.HexToAddress("6e"), builder.L2.Client) + arbRetryableTx, err := precompilesgen.NewArbRetryableTx(types.ArbRetryableTxAddress, builder.L2.Client) Require(t, err) _, customError = arbRetryableTx.SubmitRetryable( &auth, @@ -455,7 +455,7 @@ func TestCurrentTxL1GasFees(t *testing.T) { cleanup := builder.Build(t) defer cleanup() - arbGasInfo, err := precompilesgen.NewArbGasInfo(common.HexToAddress("0x6c"), builder.L2.Client) + arbGasInfo, err := precompilesgen.NewArbGasInfo(types.ArbGasInfoAddress, builder.L2.Client) Require(t, err) currTxL1GasFees, err := arbGasInfo.GetCurrentTxL1GasFees(&bind.CallOpts{Context: ctx}) @@ -478,11 +478,11 @@ func TestGetBrotliCompressionLevel(t *testing.T) { auth := builder.L2Info.GetDefaultTransactOpts("Owner", ctx) - arbOwnerPublic, err := precompilesgen.NewArbOwnerPublic(common.HexToAddress("0x6b"), builder.L2.Client) - Require(t, err, "could not bind ArbOwner contract") + arbOwnerPublic, err := precompilesgen.NewArbOwnerPublic(types.ArbOwnerPublicAddress, builder.L2.Client) + Require(t, err) - arbOwner, err := precompilesgen.NewArbOwner(common.HexToAddress("0x70"), builder.L2.Client) - Require(t, err, "could not bind ArbOwner contract") + arbOwner, err := precompilesgen.NewArbOwner(types.ArbOwnerAddress, builder.L2.Client) + Require(t, err) brotliCompressionLevel := uint64(11) @@ -495,7 +495,7 @@ func TestGetBrotliCompressionLevel(t *testing.T) { // retrieves brotli compression level callOpts := &bind.CallOpts{Context: ctx} retrievedBrotliCompressionLevel, err := arbOwnerPublic.GetBrotliCompressionLevel(callOpts) - Require(t, err, "failed to call GetBrotliCompressionLevel") + Require(t, err) if retrievedBrotliCompressionLevel != brotliCompressionLevel { Fatal(t, "expected brotli compression level to be", brotliCompressionLevel, "got", retrievedBrotliCompressionLevel) } From c4a6548f2afadeba3afdd49e5fb0983a6ff85631 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 17 Oct 2024 09:47:31 -0300 Subject: [PATCH 452/489] Moves precompile DoesntRevert tests to its own file --- system_tests/precompile_doesnt_revert_test.go | 248 ++++++++++++++++++ system_tests/precompile_test.go | 225 ---------------- system_tests/retryable_test.go | 22 -- 3 files changed, 248 insertions(+), 247 deletions(-) create mode 100644 system_tests/precompile_doesnt_revert_test.go diff --git a/system_tests/precompile_doesnt_revert_test.go b/system_tests/precompile_doesnt_revert_test.go new file mode 100644 index 0000000000..e6751d347d --- /dev/null +++ b/system_tests/precompile_doesnt_revert_test.go @@ -0,0 +1,248 @@ +// Copyright 2021-2024, Offchain Labs, Inc. +// For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE + +package arbtest + +import ( + "context" + "encoding/json" + "math/big" + "testing" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/params" + "github.com/offchainlabs/nitro/arbos/l1pricing" + "github.com/offchainlabs/nitro/solgen/go/precompilesgen" +) + +// DoesntRevert tests are useful to check if precompile calls revert due to differences in the +// return types of a contract between go and solidity. +// They are not a substitute for unit tests, as they don't test the actual functionality of the precompile. + +func TestArbAddressTableDoesntRevert(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, false) + cleanup := builder.Build(t) + defer cleanup() + + auth := builder.L2Info.GetDefaultTransactOpts("Owner", ctx) + callOpts := &bind.CallOpts{Context: ctx} + + arbAddressTable, err := precompilesgen.NewArbAddressTable(types.ArbAddressTableAddress, builder.L2.Client) + Require(t, err) + + addr := common.BytesToAddress(crypto.Keccak256([]byte{})[:20]) + + exists, err := arbAddressTable.AddressExists(callOpts, addr) + Require(t, err) + if exists { + Fatal(t, "expected address to not exist") + } + + tx, err := arbAddressTable.Register(&auth, addr) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + + idx, err := arbAddressTable.Lookup(callOpts, addr) + Require(t, err) + + retrievedAddr, err := arbAddressTable.LookupIndex(callOpts, idx) + Require(t, err) + if retrievedAddr.Cmp(addr) != 0 { + Fatal(t, "expected retrieved address to be", addr, "got", retrievedAddr) + } + + size, err := arbAddressTable.Size(callOpts) + Require(t, err) + if size.Cmp(big.NewInt(1)) != 0 { + Fatal(t, "expected size to be 1, got", size) + } + + tx, err = arbAddressTable.Compress(&auth, addr) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + + res := []uint8{128} + _, _, err = arbAddressTable.Decompress(callOpts, res, big.NewInt(0)) + Require(t, err) +} + +func TestArbAggregatorDoesntRevert(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, false) + cleanup := builder.Build(t) + defer cleanup() + + auth := builder.L2Info.GetDefaultTransactOpts("Owner", ctx) + callOpts := &bind.CallOpts{Context: ctx} + + arbAggregator, err := precompilesgen.NewArbAggregator(types.ArbAggregatorAddress, builder.L2.Client) + Require(t, err) + + tx, err := arbAggregator.SetFeeCollector(&auth, l1pricing.BatchPosterAddress, common.Address{}) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + + _, err = arbAggregator.GetFeeCollector(callOpts, l1pricing.BatchPosterAddress) + Require(t, err) +} + +func TestArbosTestDoesntRevert(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, false) + cleanup := builder.Build(t) + defer cleanup() + + callOpts := &bind.CallOpts{Context: ctx} + + arbosTest, err := precompilesgen.NewArbosTest(types.ArbosTestAddress, builder.L2.Client) + Require(t, err) + + err = arbosTest.BurnArbGas(callOpts, big.NewInt(1)) + Require(t, err) +} + +func TestArbSysDoesntRevert(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, false) + cleanup := builder.Build(t) + defer cleanup() + + callOpts := &bind.CallOpts{Context: ctx} + + arbSys, err := precompilesgen.NewArbSys(types.ArbSysAddress, builder.L2.Client) + Require(t, err) + + addr1 := common.BytesToAddress(crypto.Keccak256([]byte{})[:20]) + addr2 := common.BytesToAddress(crypto.Keccak256([]byte{})[:20]) + _, err = arbSys.MapL1SenderContractAddressToL2Alias(callOpts, addr1, addr2) + Require(t, err) +} + +func TestArbOwnerDoesntRevert(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, false) + cleanup := builder.Build(t) + defer cleanup() + + auth := builder.L2Info.GetDefaultTransactOpts("Owner", ctx) + + arbOwner, err := precompilesgen.NewArbOwner(types.ArbOwnerAddress, builder.L2.Client) + Require(t, err) + + chainConfig := params.ArbitrumDevTestChainConfig() + chainConfig.ArbitrumChainParams.MaxCodeSize = 100 + serializedChainConfig, err := json.Marshal(chainConfig) + Require(t, err) + tx, err := arbOwner.SetChainConfig(&auth, string(serializedChainConfig)) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + + tx, err = arbOwner.SetAmortizedCostCapBips(&auth, 77734) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + + tx, err = arbOwner.ReleaseL1PricerSurplusFunds(&auth, big.NewInt(1)) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + + tx, err = arbOwner.SetL2BaseFee(&auth, big.NewInt(1)) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) +} + +func TestArbGasInfoDoesntRevert(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, false) + cleanup := builder.Build(t) + defer cleanup() + + callOpts := &bind.CallOpts{Context: ctx} + addr := common.BytesToAddress(crypto.Keccak256([]byte{})[:20]) + + arbGasInfo, err := precompilesgen.NewArbGasInfo(types.ArbGasInfoAddress, builder.L2.Client) + Require(t, err) + + _, err = arbGasInfo.GetGasBacklog(callOpts) + Require(t, err) + + _, err = arbGasInfo.GetLastL1PricingUpdateTime(callOpts) + Require(t, err) + + _, err = arbGasInfo.GetL1PricingFundsDueForRewards(callOpts) + Require(t, err) + + _, err = arbGasInfo.GetL1PricingUnitsSinceUpdate(callOpts) + Require(t, err) + + _, err = arbGasInfo.GetLastL1PricingSurplus(callOpts) + Require(t, err) + + _, _, _, err = arbGasInfo.GetPricesInArbGas(callOpts) + Require(t, err) + + _, _, _, err = arbGasInfo.GetPricesInArbGasWithAggregator(callOpts, addr) + Require(t, err) + + _, err = arbGasInfo.GetAmortizedCostCapBips(callOpts) + Require(t, err) + + _, err = arbGasInfo.GetL1FeesAvailable(callOpts) + Require(t, err) + + _, _, _, _, _, _, err = arbGasInfo.GetPricesInWeiWithAggregator(callOpts, addr) + Require(t, err) +} + +func TestArbRetryableTxDoesntRevert(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, false) + cleanup := builder.Build(t) + defer cleanup() + + callOpts := &bind.CallOpts{Context: ctx} + + arbRetryableTx, err := precompilesgen.NewArbRetryableTx(common.HexToAddress("6e"), builder.L2.Client) + Require(t, err) + + _, err = arbRetryableTx.GetCurrentRedeemer(callOpts) + Require(t, err) +} diff --git a/system_tests/precompile_test.go b/system_tests/precompile_test.go index cdf63ffc98..9d5737c249 100644 --- a/system_tests/precompile_test.go +++ b/system_tests/precompile_test.go @@ -5,7 +5,6 @@ package arbtest import ( "context" - "encoding/json" "fmt" "math/big" "sort" @@ -683,137 +682,6 @@ func TestFeeAccounts(t *testing.T) { } } -// DoesntRevert tests are useful to check if precompile calls revert due to differences in the -// return types of a contract between go and solidity. -// They are not a substitute for unit tests, as they don't test the actual functionality of the precompile. -func TestArbAddressTableDoesntRevert(t *testing.T) { - t.Parallel() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - builder := NewNodeBuilder(ctx).DefaultConfig(t, false) - cleanup := builder.Build(t) - defer cleanup() - - auth := builder.L2Info.GetDefaultTransactOpts("Owner", ctx) - callOpts := &bind.CallOpts{Context: ctx} - - arbAddressTable, err := precompilesgen.NewArbAddressTable(types.ArbAddressTableAddress, builder.L2.Client) - Require(t, err) - - addr := common.BytesToAddress(crypto.Keccak256([]byte{})[:20]) - - exists, err := arbAddressTable.AddressExists(callOpts, addr) - Require(t, err) - if exists { - Fatal(t, "expected address to not exist") - } - - tx, err := arbAddressTable.Register(&auth, addr) - Require(t, err) - _, err = builder.L2.EnsureTxSucceeded(tx) - Require(t, err) - - idx, err := arbAddressTable.Lookup(callOpts, addr) - Require(t, err) - - retrievedAddr, err := arbAddressTable.LookupIndex(callOpts, idx) - Require(t, err) - if retrievedAddr.Cmp(addr) != 0 { - Fatal(t, "expected retrieved address to be", addr, "got", retrievedAddr) - } - - size, err := arbAddressTable.Size(callOpts) - Require(t, err) - if size.Cmp(big.NewInt(1)) != 0 { - Fatal(t, "expected size to be 1, got", size) - } - - tx, err = arbAddressTable.Compress(&auth, addr) - Require(t, err) - _, err = builder.L2.EnsureTxSucceeded(tx) - Require(t, err) - - res := []uint8{128} - _, _, err = arbAddressTable.Decompress(callOpts, res, big.NewInt(0)) - Require(t, err) -} - -// DoesntRevert tests are useful to check if precompile calls revert due to differences in the -// return types of a contract between go and solidity. -// They are not a substitute for unit tests, as they don't test the actual functionality of the precompile. -func TestArbAggregatorDoesntRevert(t *testing.T) { - t.Parallel() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - builder := NewNodeBuilder(ctx).DefaultConfig(t, false) - cleanup := builder.Build(t) - defer cleanup() - - auth := builder.L2Info.GetDefaultTransactOpts("Owner", ctx) - callOpts := &bind.CallOpts{Context: ctx} - - arbAggregator, err := precompilesgen.NewArbAggregator(types.ArbAggregatorAddress, builder.L2.Client) - Require(t, err) - - tx, err := arbAggregator.SetFeeCollector(&auth, l1pricing.BatchPosterAddress, common.Address{}) - Require(t, err) - _, err = builder.L2.EnsureTxSucceeded(tx) - Require(t, err) - - _, err = arbAggregator.GetFeeCollector(callOpts, l1pricing.BatchPosterAddress) - Require(t, err) -} - -// DoesntRevert tests are useful to check if precompile calls revert due to differences in the -// return types of a contract between go and solidity. -// They are not a substitute for unit tests, as they don't test the actual functionality of the precompile. -func TestArbosTestDoesntRevert(t *testing.T) { - t.Parallel() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - builder := NewNodeBuilder(ctx).DefaultConfig(t, false) - cleanup := builder.Build(t) - defer cleanup() - - callOpts := &bind.CallOpts{Context: ctx} - - arbosTest, err := precompilesgen.NewArbosTest(types.ArbosTestAddress, builder.L2.Client) - Require(t, err) - - err = arbosTest.BurnArbGas(callOpts, big.NewInt(1)) - Require(t, err) -} - -// DoesntRevert tests are useful to check if precompile calls revert due to differences in the -// return types of a contract between go and solidity. -// They are not a substitute for unit tests, as they don't test the actual functionality of the precompile. -func TestArbSysDoesntRevert(t *testing.T) { - t.Parallel() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - builder := NewNodeBuilder(ctx).DefaultConfig(t, false) - cleanup := builder.Build(t) - defer cleanup() - - callOpts := &bind.CallOpts{Context: ctx} - - arbSys, err := precompilesgen.NewArbSys(types.ArbSysAddress, builder.L2.Client) - Require(t, err) - - addr1 := common.BytesToAddress(crypto.Keccak256([]byte{})[:20]) - addr2 := common.BytesToAddress(crypto.Keccak256([]byte{})[:20]) - _, err = arbSys.MapL1SenderContractAddressToL2Alias(callOpts, addr1, addr2) - Require(t, err) -} - func TestChainOwners(t *testing.T) { t.Parallel() @@ -889,99 +757,6 @@ func TestChainOwners(t *testing.T) { } } -// DoesntRevert tests are useful to check if precompile calls revert due to differences in the -// return types of a contract between go and solidity. -// They are not a substitute for unit tests, as they don't test the actual functionality of the precompile. -func TestArbOwnerDoesntRevert(t *testing.T) { - t.Parallel() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - builder := NewNodeBuilder(ctx).DefaultConfig(t, false) - cleanup := builder.Build(t) - defer cleanup() - - auth := builder.L2Info.GetDefaultTransactOpts("Owner", ctx) - - arbOwner, err := precompilesgen.NewArbOwner(types.ArbOwnerAddress, builder.L2.Client) - Require(t, err) - - chainConfig := params.ArbitrumDevTestChainConfig() - chainConfig.ArbitrumChainParams.MaxCodeSize = 100 - serializedChainConfig, err := json.Marshal(chainConfig) - Require(t, err) - tx, err := arbOwner.SetChainConfig(&auth, string(serializedChainConfig)) - Require(t, err) - _, err = builder.L2.EnsureTxSucceeded(tx) - Require(t, err) - - tx, err = arbOwner.SetAmortizedCostCapBips(&auth, 77734) - Require(t, err) - _, err = builder.L2.EnsureTxSucceeded(tx) - Require(t, err) - - tx, err = arbOwner.ReleaseL1PricerSurplusFunds(&auth, big.NewInt(1)) - Require(t, err) - _, err = builder.L2.EnsureTxSucceeded(tx) - Require(t, err) - - tx, err = arbOwner.SetL2BaseFee(&auth, big.NewInt(1)) - Require(t, err) - _, err = builder.L2.EnsureTxSucceeded(tx) - Require(t, err) -} - -// DoesntRevert tests are useful to check if precompile calls revert due to differences in the -// return types of a contract between go and solidity. -// They are not a substitute for unit tests, as they don't test the actual functionality of the precompile. -func TestArbGasInfoDoesntRevert(t *testing.T) { - t.Parallel() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - builder := NewNodeBuilder(ctx).DefaultConfig(t, false) - cleanup := builder.Build(t) - defer cleanup() - - callOpts := &bind.CallOpts{Context: ctx} - addr := common.BytesToAddress(crypto.Keccak256([]byte{})[:20]) - - arbGasInfo, err := precompilesgen.NewArbGasInfo(types.ArbGasInfoAddress, builder.L2.Client) - Require(t, err) - - _, err = arbGasInfo.GetGasBacklog(callOpts) - Require(t, err) - - _, err = arbGasInfo.GetLastL1PricingUpdateTime(callOpts) - Require(t, err) - - _, err = arbGasInfo.GetL1PricingFundsDueForRewards(callOpts) - Require(t, err) - - _, err = arbGasInfo.GetL1PricingUnitsSinceUpdate(callOpts) - Require(t, err) - - _, err = arbGasInfo.GetLastL1PricingSurplus(callOpts) - Require(t, err) - - _, _, _, err = arbGasInfo.GetPricesInArbGas(callOpts) - Require(t, err) - - _, _, _, err = arbGasInfo.GetPricesInArbGasWithAggregator(callOpts, addr) - Require(t, err) - - _, err = arbGasInfo.GetAmortizedCostCapBips(callOpts) - Require(t, err) - - _, err = arbGasInfo.GetL1FeesAvailable(callOpts) - Require(t, err) - - _, _, _, _, _, _, err = arbGasInfo.GetPricesInWeiWithAggregator(callOpts, addr) - Require(t, err) -} - func TestArbAggregatorBatchPosters(t *testing.T) { t.Parallel() diff --git a/system_tests/retryable_test.go b/system_tests/retryable_test.go index 5c90d3c7d5..89446e3c4b 100644 --- a/system_tests/retryable_test.go +++ b/system_tests/retryable_test.go @@ -445,28 +445,6 @@ func TestGetLifetime(t *testing.T) { } } -// DoesntRevert tests are useful to check if precompile calls revert due to differences in the -// return types of a contract between go and solidity. -// They are not a substitute for unit tests, as they don't test the actual functionality of the precompile. -func TestArbRetryableTxDoesntRevert(t *testing.T) { - t.Parallel() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - builder := NewNodeBuilder(ctx).DefaultConfig(t, false) - cleanup := builder.Build(t) - defer cleanup() - - callOpts := &bind.CallOpts{Context: ctx} - - arbRetryableTx, err := precompilesgen.NewArbRetryableTx(common.HexToAddress("6e"), builder.L2.Client) - Require(t, err) - - _, err = arbRetryableTx.GetCurrentRedeemer(callOpts) - Require(t, err) -} - func TestKeepaliveAndCancelRetryable(t *testing.T) { t.Parallel() builder, delayedInbox, lookupL2Tx, ctx, teardown := retryableSetup(t) From bb92595db74dd3d9b4790d36cddde922e8d8074b Mon Sep 17 00:00:00 2001 From: Joshua Colvin Date: Thu, 17 Oct 2024 23:52:01 -0700 Subject: [PATCH 453/489] Update check-build Updated dependencies for MacOS and fixed issues with running on Ubuntu --- {util => scripts}/check-build.sh | 54 +++++++++++--------------------- 1 file changed, 19 insertions(+), 35 deletions(-) rename {util => scripts}/check-build.sh (66%) diff --git a/util/check-build.sh b/scripts/check-build.sh similarity index 66% rename from util/check-build.sh rename to scripts/check-build.sh index ddf540e45c..d654405c49 100755 --- a/util/check-build.sh +++ b/scripts/check-build.sh @@ -27,7 +27,7 @@ echo -e "${BLUE}Checking prerequisites for building Nitro locally...${NC}" if command_exists docker; then echo -e "${GREEN}Docker is installed.${NC}" else - echo -e "${RED}Docker is not installed. $INSTALLATION_DOCS_URL${NC}" + echo -e "${RED}Docker is not installed.${NC}" EXIT_CODE=1 fi @@ -58,51 +58,31 @@ fi if docker images | grep -q "nitro-node"; then echo -e "${GREEN}Nitro Docker image is built.${NC}" else - echo -e "${RED}Nitro Docker image is not built. Build it using: docker build . --tag nitro-node${NC}" - EXIT_CODE=1 + echo -e "${YELLOW}Nitro Docker image is not built. Build it using: docker build . --tag nitro-node${NC}" fi # Step 5: Check prerequisites for building binaries echo -e "${BLUE}Checking prerequisites for building Nitro's binaries...${NC}" if [[ "$OS" == "Linux" ]]; then - prerequisites=(git curl build-essential cmake npm golang clang make gotestsum wasm2wat lld-13 python3 yarn) + prerequisites=(git curl make cmake npm golang clang make gotestsum wasm2wat wasm-ld python3 yarn) else - prerequisites=(git curl make cmake npm go gvm golangci-lint wasm2wat clang gotestsum yarn) + prerequisites=(git curl make cmake npm go golangci-lint wasm2wat clang wasm-ld gotestsum yarn) fi for pkg in "${prerequisites[@]}"; do - if command_exists "$pkg"; then + EXISTS=$(command_exists "$pkg") + [[ "$pkg" == "make" ]] && pkg="build-essential" + [[ "$pkg" == "wasm2wat" ]] && pkg="wabt" + [[ "$pkg" == "clang" ]] && pkg="llvm" + [[ "$pkg" == "wasm-ld" ]] && pkg="lld" + if $EXISTS; then # There is no way to check for wabt / llvm directly, since they install multiple tools # So instead, we check for wasm2wat and clang, which are part of wabt and llvm respectively # and if they are installed, we assume wabt / llvm is installed else we ask the user to install wabt / llvm - [[ "$pkg" == "wasm2wat" ]] && pkg="wabt" - [[ "$pkg" == "clang" ]] && pkg="llvm" - - # Check for specific symbolic links related to wasm-ld on Linux and macOS - if [[ "$pkg" == "llvm" ]]; then - if [[ "$OS" == "Linux" ]]; then - if [ ! -L /usr/local/bin/wasm-ld ]; then - echo -e "${YELLOW}Creating symbolic link for wasm-ld on Linux.${NC}" - sudo ln -s /usr/bin/wasm-ld-13 /usr/local/bin/wasm-ld - else - echo -e "${GREEN}Symbolic link for wasm-ld on Linux is already present.${NC}" - fi - elif [[ "$OS" == "Darwin" ]]; then - if [ ! -L /usr/local/bin/wasm-ld ]; then - echo -e "${YELLOW}Creating symbolic link for wasm-ld on macOS.${NC}" - sudo mkdir -p /usr/local/bin - sudo ln -s /opt/homebrew/opt/llvm/bin/wasm-ld /usr/local/bin/wasm-ld - else - echo -e "${GREEN}Symbolic link for wasm-ld on macOS is already present.${NC}" - fi - fi - fi echo -e "${GREEN}$pkg is installed.${NC}" else - [[ "$pkg" == "wasm2wat" ]] && pkg="wabt" - [[ "$pkg" == "clang" ]] && pkg="llvm" - echo -e "${RED}$pkg is not installed. Please install $pkg. $INSTALLATION_DOCS_URL${NC}" + echo -e "${RED}$pkg is not installed. Please install $pkg.${NC}" EXIT_CODE=1 fi done @@ -111,7 +91,7 @@ done if command_exists node && node -v | grep -q "v18"; then echo -e "${GREEN}Node.js version 18 is installed.${NC}" else - echo -e "${RED}Node.js version 18 not installed. $INSTALLATION_DOCS_URL${NC}" + echo -e "${RED}Node.js version 18 not installed.${NC}" EXIT_CODE=1 fi @@ -119,7 +99,7 @@ fi if command_exists rustc && rustc --version | grep -q "1.80.1"; then echo -e "${GREEN}Rust version 1.80.1 is installed.${NC}" else - echo -e "${RED}Rust version 1.80.1 not installed. $INSTALLATION_DOCS_URL${NC}" + echo -e "${RED}Rust version 1.80.1 not installed.${NC}" EXIT_CODE=1 fi @@ -136,7 +116,7 @@ go_version_needed=$(grep "^go " go.mod | awk '{print $2}') if command_exists go && go version | grep -q "$go_version_needed"; then echo -e "${GREEN}Go version $go_version_needed is installed.${NC}" else - echo -e "${RED}Go version $go_version_needed not installed. $INSTALLATION_DOCS_URL${NC}" + echo -e "${RED}Go version $go_version_needed not installed.${NC}" EXIT_CODE=1 fi @@ -144,11 +124,15 @@ fi if command_exists foundryup; then echo -e "${GREEN}Foundry is installed.${NC}" else - echo -e "${RED}Foundry is not installed. $INSTALLATION_DOCS_URL${NC}" + echo -e "${RED}Foundry is not installed.${NC}" EXIT_CODE=1 fi echo -e "${BLUE}Verification complete.${NC}" +if [ $EXIT_CODE != 0 ]; then + echo -e "${RED}One or more dependencies missing. $INSTALLATION_DOCS_URL${NC}" +fi + exit $EXIT_CODE From 38ff8272a67acdcd8edbf34e0ba55695d365ea7e Mon Sep 17 00:00:00 2001 From: Joshua Colvin Date: Fri, 18 Oct 2024 11:46:47 -0700 Subject: [PATCH 454/489] More check-build tweaks * remove last usage of `sudo` * improve version tag messaging * cleanup informational messages --- scripts/check-build.sh | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/scripts/check-build.sh b/scripts/check-build.sh index d654405c49..518d12fbc2 100755 --- a/scripts/check-build.sh +++ b/scripts/check-build.sh @@ -21,7 +21,6 @@ EXIT_CODE=0 # Detect operating system OS=$(uname -s) echo -e "${BLUE}Detected OS: $OS${NC}" -echo -e "${BLUE}Checking prerequisites for building Nitro locally...${NC}" # Step 1: Check Docker Installation if command_exists docker; then @@ -32,7 +31,7 @@ else fi # Step 2: Check if Docker service is running -if [[ "$OS" == "Linux" ]] && ! sudo service docker status >/dev/null; then +if [[ "$OS" == "Linux" ]] && ! pidof dockerd >/dev/null; then echo -e "${YELLOW}Docker service is not running on Linux. Start it with: sudo service docker start${NC}" EXIT_CODE=1 elif [[ "$OS" == "Darwin" ]] && ! docker info >/dev/null 2>&1; then @@ -43,8 +42,12 @@ else fi # Step 3: Check the version tag -VERSION_TAG=$(git tag --points-at HEAD | sed '/-/!s/$/_/' | sort -rV | sed 's/_$//' | head -n 1 | grep ^ || git show -s --pretty=%D | sed 's/, /\n/g' | grep -v '^origin/' | grep -v '^grafted\|HEAD\|master\|main$' || echo "dev") -echo -e "${YELLOW}You are on the version tag: $VERSION_TAG${NC}" +VERSION_TAG=$(git tag --points-at HEAD | sed '/-/!s/$/_/' | sort -rV | sed 's/_$//' | head -n 1 | grep ^ || git show -s --pretty=%D | sed 's/, /\n/g' | grep -v '^origin/' | grep -v '^grafted\|HEAD\|master\|main$' || echo "") +if [[ -z "${VAR}" ]]; then + echo -e "${YELLOW}Untagged version of Nitro checked out, not guaranteed to build successfully.${NC}" +else + echo -e "${GREEN}You are on Nitro version tag: $VERSION_TAG${NC}" +fi # Check if submodules are properly initialized and updated if git submodule status | grep -qE '^-|\+'; then @@ -62,7 +65,6 @@ else fi # Step 5: Check prerequisites for building binaries -echo -e "${BLUE}Checking prerequisites for building Nitro's binaries...${NC}" if [[ "$OS" == "Linux" ]]; then prerequisites=(git curl make cmake npm golang clang make gotestsum wasm2wat wasm-ld python3 yarn) else @@ -128,10 +130,11 @@ else EXIT_CODE=1 fi -echo -e "${BLUE}Verification complete.${NC}" if [ $EXIT_CODE != 0 ]; then echo -e "${RED}One or more dependencies missing. $INSTALLATION_DOCS_URL${NC}" +else + echo -e "${BLUE}Build readiness check passed.${NC}" fi exit $EXIT_CODE From 2ba9a55756b0215de2ef39371dd676c0703e49a6 Mon Sep 17 00:00:00 2001 From: Joshua Colvin Date: Fri, 18 Oct 2024 11:50:57 -0700 Subject: [PATCH 455/489] Improve message for untagged version --- scripts/check-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check-build.sh b/scripts/check-build.sh index 518d12fbc2..ee79d12b8b 100755 --- a/scripts/check-build.sh +++ b/scripts/check-build.sh @@ -44,7 +44,7 @@ fi # Step 3: Check the version tag VERSION_TAG=$(git tag --points-at HEAD | sed '/-/!s/$/_/' | sort -rV | sed 's/_$//' | head -n 1 | grep ^ || git show -s --pretty=%D | sed 's/, /\n/g' | grep -v '^origin/' | grep -v '^grafted\|HEAD\|master\|main$' || echo "") if [[ -z "${VAR}" ]]; then - echo -e "${YELLOW}Untagged version of Nitro checked out, not guaranteed to build successfully.${NC}" + echo -e "${YELLOW}Untagged version of Nitro checked out, may not be fully tested.${NC}" else echo -e "${GREEN}You are on Nitro version tag: $VERSION_TAG${NC}" fi From 1a1e179c737dc41424ab634fd481ead15c6485e4 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Mon, 21 Oct 2024 16:58:28 +0530 Subject: [PATCH 456/489] Log from batch poster for which das backends are using chunked vs legacy store --- das/dasRpcClient.go | 1 + 1 file changed, 1 insertion(+) diff --git a/das/dasRpcClient.go b/das/dasRpcClient.go index 241f2196b1..d6e2c389c9 100644 --- a/das/dasRpcClient.go +++ b/das/dasRpcClient.go @@ -101,6 +101,7 @@ func (c *DASRPCClient) Store(ctx context.Context, message []byte, timeout uint64 var startChunkedStoreResult StartChunkedStoreResult if err := c.clnt.CallContext(ctx, &startChunkedStoreResult, "das_startChunkedStore", hexutil.Uint64(timestamp), hexutil.Uint64(nChunks), hexutil.Uint64(c.chunkSize), hexutil.Uint64(totalSize), hexutil.Uint64(timeout), hexutil.Bytes(startReqSig)); err != nil { if strings.Contains(err.Error(), "the method das_startChunkedStore does not exist") { + log.Info("Legacy store is used by the DAS client", "url", c.url) return c.legacyStore(ctx, message, timeout) } return nil, err From 044f70bd6312d0e71fe1dff386d9325db71f02db Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Tue, 22 Oct 2024 12:52:24 +0530 Subject: [PATCH 457/489] address PR comments --- system_tests/program_test.go | 94 +++++++++++++++++++----------------- 1 file changed, 50 insertions(+), 44 deletions(-) diff --git a/system_tests/program_test.go b/system_tests/program_test.go index 9c8fe949ab..c240aaad6b 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -1268,66 +1268,75 @@ func TestStylusPrecompileMethodsSimple(t *testing.T) { ownerAuth := builder.L2Info.GetDefaultTransactOpts("Owner", ctx) ensure(arbDebug.BecomeChainOwner(&ownerAuth)) - // ArbOwner precompile methods - testConst := uint16(10) - ensure(arbOwner.SetInkPrice(&ownerAuth, uint32(testConst))) - ensure(arbOwner.SetWasmMaxStackDepth(&ownerAuth, uint32(testConst))) - ensure(arbOwner.SetWasmFreePages(&ownerAuth, testConst)) - ensure(arbOwner.SetWasmPageGas(&ownerAuth, testConst)) - ensure(arbOwner.SetWasmPageLimit(&ownerAuth, testConst)) - // Setting low values of gas and cached parameters ensures when MinInitGas is called on ArbWasm precompile, - // the returned values would be programs.MinInitGasUnits and programs.MinCachedGasUnits - ensure(arbOwner.SetWasmMinInitGas(&ownerAuth, 1, 1)) - ensure(arbOwner.SetWasmInitCostScalar(&ownerAuth, uint64(testConst))) - expectedExpiryDays := uint16(1) - ensure(arbOwner.SetWasmExpiryDays(&ownerAuth, expectedExpiryDays)) - ensure(arbOwner.SetWasmKeepaliveDays(&ownerAuth, 0)) - ensure(arbOwner.SetWasmBlockCacheSize(&ownerAuth, testConst)) - - // ArbWasm precompile methods wasm, _ := readWasmFile(t, rustFile("keccak")) - codehash := crypto.Keccak256Hash(wasm) programAddress := deployContract(t, ctx, ownerAuth, builder.L2.Client, wasm) activateAuth := ownerAuth activateAuth.Value = oneEth ensure(arbWasm.ActivateProgram(&activateAuth, programAddress)) - bcs, err := arbWasm.BlockCacheSize(nil) - Require(t, err) - if bcs != testConst { - t.Errorf("BlockCacheSize from arbWasm precompile didnt match the value set by arbowner. have: %d, want: %d", bcs, testConst) - } + expectedExpiryDays := uint16(1) + ensure(arbOwner.SetWasmExpiryDays(&ownerAuth, expectedExpiryDays)) ed, err := arbWasm.ExpiryDays(nil) Require(t, err) if ed != expectedExpiryDays { t.Errorf("ExpiryDays from arbWasm precompile didnt match the value set by arbowner. have: %d, want: %d", ed, expectedExpiryDays) } + ptl, err := arbWasm.ProgramTimeLeft(nil, programAddress) + Require(t, err) + expectedExpirySeconds := (uint64(expectedExpiryDays) * 24 * 3600) + // ProgramTimeLeft returns time in seconds to expiry and the current ExpiryDays is set to 1 day + // We expect the lag of 3600 seconds to exist because program.activatedAt uses hoursSinceArbitrum that + // rounds down (the current time since ArbitrumStartTime in hours)/3600 + if expectedExpirySeconds-ptl > 3600 { + t.Errorf("ProgramTimeLeft from arbWasm precompile returned value lesser than expected. %d <= want <= %d, have: %d", expectedExpirySeconds-3600, expectedExpirySeconds, ptl) + } + + ensure(arbOwner.SetWasmBlockCacheSize(&ownerAuth, 100)) + bcs, err := arbWasm.BlockCacheSize(nil) + Require(t, err) + if bcs != 100 { + t.Errorf("BlockCacheSize from arbWasm precompile didnt match the value set by arbowner. have: %d, want: %d", bcs, 100) + } + + ensure(arbOwner.SetWasmFreePages(&ownerAuth, 3)) fp, err := arbWasm.FreePages(nil) Require(t, err) - if fp != testConst { - t.Errorf("FreePages from arbWasm precompile didnt match the value set by arbowner. have: %d, want: %d", fp, testConst) + if fp != 3 { + t.Errorf("FreePages from arbWasm precompile didnt match the value set by arbowner. have: %d, want: %d", fp, 3) } + + ensure(arbOwner.SetWasmInitCostScalar(&ownerAuth, uint64(4))) ics, err := arbWasm.InitCostScalar(nil) Require(t, err) - if ics != uint64(testConst) { - t.Errorf("InitCostScalar from arbWasm precompile didnt match the value set by arbowner. have: %d, want: %d", ics, testConst) + if ics != uint64(4) { + t.Errorf("InitCostScalar from arbWasm precompile didnt match the value set by arbowner. have: %d, want: %d", ics, 4) } + + ensure(arbOwner.SetInkPrice(&ownerAuth, uint32(5))) ip, err := arbWasm.InkPrice(nil) Require(t, err) - if ip != uint32(testConst) { - t.Errorf("InkPrice from arbWasm precompile didnt match the value set by arbowner. have: %d, want: %d", ip, testConst) + if ip != uint32(5) { + t.Errorf("InkPrice from arbWasm precompile didnt match the value set by arbowner. have: %d, want: %d", ip, 5) } + + ensure(arbOwner.SetWasmKeepaliveDays(&ownerAuth, 0)) kad, err := arbWasm.KeepaliveDays(nil) Require(t, err) if kad != 0 { t.Errorf("KeepaliveDays from arbWasm precompile didnt match the value set by arbowner. have: %d, want: 0", kad) } + + ensure(arbOwner.SetWasmMaxStackDepth(&ownerAuth, uint32(6))) msd, err := arbWasm.MaxStackDepth(nil) Require(t, err) - if msd != uint32(testConst) { - t.Errorf("MaxStackDepth from arbWasm precompile didnt match the value set by arbowner. have: %d, want: %d", msd, testConst) + if msd != uint32(6) { + t.Errorf("MaxStackDepth from arbWasm precompile didnt match the value set by arbowner. have: %d, want: %d", msd, 6) } + + // Setting low values of gas and cached parameters ensures when MinInitGas is called on ArbWasm precompile, + // the returned values would be programs.MinInitGasUnits and programs.MinCachedGasUnits + ensure(arbOwner.SetWasmMinInitGas(&ownerAuth, 1, 1)) mig, err := arbWasm.MinInitGas(nil) Require(t, err) if mig.Gas != programs.MinInitGasUnits { @@ -1336,34 +1345,31 @@ func TestStylusPrecompileMethodsSimple(t *testing.T) { if mig.Cached != programs.MinCachedGasUnits { t.Errorf("MinInitGas from arbWasm precompile didnt match the Cached value set by arbowner. have: %d, want: %d", mig.Cached, programs.MinCachedGasUnits) } + + ensure(arbOwner.SetWasmPageGas(&ownerAuth, 7)) pg, err := arbWasm.PageGas(nil) Require(t, err) - if pg != testConst { - t.Errorf("PageGas from arbWasm precompile didnt match the value set by arbowner. have: %d, want: %d", pg, testConst) + if pg != 7 { + t.Errorf("PageGas from arbWasm precompile didnt match the value set by arbowner. have: %d, want: %d", pg, 7) } + + ensure(arbOwner.SetWasmPageLimit(&ownerAuth, 8)) pl, err := arbWasm.PageLimit(nil) Require(t, err) - if pl != testConst { - t.Errorf("PageLimit from arbWasm precompile didnt match the value set by arbowner. have: %d, want: %d", pl, testConst) + if pl != 8 { + t.Errorf("PageLimit from arbWasm precompile didnt match the value set by arbowner. have: %d, want: %d", pl, 8) } + // pageramp currently is initialPageRamp = 620674314 value in programs package _, err = arbWasm.PageRamp(nil) Require(t, err) + codehash := crypto.Keccak256Hash(wasm) cas, err := arbWasm.CodehashAsmSize(nil, codehash) Require(t, err) if cas == 0 { t.Error("CodehashAsmSize from arbWasm precompile returned 0 value") } - ptl, err := arbWasm.ProgramTimeLeft(nil, programAddress) - Require(t, err) - expectedExpirySeconds := (uint64(expectedExpiryDays) * 24 * 3600) - // ProgramTimeLeft returns time in seconds to expiry and the current ExpiryDays is set to 1 day - // We expect the lag of 3600 seconds to exist because program.activatedAt uses hoursSinceArbitrum that - // rounds down (the current time since ArbitrumStartTime in hours)/3600 - if expectedExpirySeconds-ptl > 3600 { - t.Errorf("ProgramTimeLeft from arbWasm precompile returned value lesser than expected. %d <= want <= %d, have: %d", expectedExpirySeconds-3600, expectedExpirySeconds, ptl) - } // Since ArbOwner has set wasm KeepaliveDays to 0, it enables us to do this, though this shouldn't have any effect codehashKeepaliveAuth := ownerAuth codehashKeepaliveAuth.Value = oneEth From 94e252aa24d8f7587f6faf9583a19769d742fed5 Mon Sep 17 00:00:00 2001 From: Nick Hovsmith Date: Tue, 22 Oct 2024 11:01:26 -0400 Subject: [PATCH 458/489] add flag to remove docker in docker --- Makefile | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 4c5550d199..64d5acfd08 100644 --- a/Makefile +++ b/Makefile @@ -579,9 +579,15 @@ contracts/test/prover/proofs/%.json: $(arbitrator_cases)/%.wasm $(prover_bin) @touch $@ .make/cbrotli-wasm: $(DEP_PREDICATE) $(ORDER_ONLY_PREDICATE) .make - test -f target/lib-wasm/libbrotlicommon-static.a || ./scripts/build-brotli.sh -w - test -f target/lib-wasm/libbrotlienc-static.a || ./scripts/build-brotli.sh -w - test -f target/lib-wasm/libbrotlidec-static.a || ./scripts/build-brotli.sh -w +ifdef NODOCKERINDOCKER + test -f target/lib-wasm/libbrotlicommon-static.a || ./scripts/build-brotli.sh -w + test -f target/lib-wasm/libbrotlienc-static.a || ./scripts/build-brotli.sh -w + test -f target/lib-wasm/libbrotlidec-static.a || ./scripts/build-brotli.sh -w +else + test -f target/lib-wasm/libbrotlicommon-static.a || ./scripts/build-brotli.sh -w -d + test -f target/lib-wasm/libbrotlienc-static.a || ./scripts/build-brotli.sh -w -d + test -f target/lib-wasm/libbrotlidec-static.a || ./scripts/build-brotli.sh -w -d +endif @touch $@ .make/wasm-lib: $(DEP_PREDICATE) arbitrator/wasm-libraries/soft-float/SoftFloat/build/Wasm-Clang/softfloat.a $(ORDER_ONLY_PREDICATE) .make From 67d55aa3aae5125960b1b656b150afbdf69300bd Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Tue, 22 Oct 2024 10:29:33 -0500 Subject: [PATCH 459/489] Change trace logs to debug --- arbnode/inbox_reader.go | 20 ++++++++++---------- arbnode/inbox_tracker.go | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/arbnode/inbox_reader.go b/arbnode/inbox_reader.go index 42e99fad66..816d6d7a31 100644 --- a/arbnode/inbox_reader.go +++ b/arbnode/inbox_reader.go @@ -353,7 +353,7 @@ func (r *InboxReader) run(ctx context.Context, hadError bool) error { return err } if ourLatestDelayedCount < checkingDelayedCount { - log.Trace("Expecting to find delayed messages", "checkingDelayedCount", checkingDelayedCount, "ourLatestDelayedCount", ourLatestDelayedCount, "currentHeight", currentHeight) + log.Debug("Expecting to find delayed messages", "checkingDelayedCount", checkingDelayedCount, "ourLatestDelayedCount", ourLatestDelayedCount, "currentHeight", currentHeight) checkingDelayedCount = ourLatestDelayedCount missingDelayed = true } else if ourLatestDelayedCount > checkingDelayedCount { @@ -374,7 +374,7 @@ func (r *InboxReader) run(ctx context.Context, hadError bool) error { return err } if dbDelayedAcc != l1DelayedAcc { - log.Trace("Latest delayed accumulator mismatch", "delayedSeqNum", checkingDelayedSeqNum, "dbDelayedAcc", dbDelayedAcc, "l1DelayedAcc", l1DelayedAcc) + log.Debug("Latest delayed accumulator mismatch", "delayedSeqNum", checkingDelayedSeqNum, "dbDelayedAcc", dbDelayedAcc, "l1DelayedAcc", l1DelayedAcc) reorgingDelayed = true } } @@ -392,7 +392,7 @@ func (r *InboxReader) run(ctx context.Context, hadError bool) error { return err } if ourLatestBatchCount < checkingBatchCount { - log.Trace("Expecting to find sequencer batches", "checkingBatchCount", checkingBatchCount, "ourLatestBatchCount", ourLatestBatchCount, "currentHeight", currentHeight) + log.Debug("Expecting to find sequencer batches", "checkingBatchCount", checkingBatchCount, "ourLatestBatchCount", ourLatestBatchCount, "currentHeight", currentHeight) checkingBatchCount = ourLatestBatchCount missingSequencer = true } else if ourLatestBatchCount > checkingBatchCount && config.HardReorg { @@ -412,7 +412,7 @@ func (r *InboxReader) run(ctx context.Context, hadError bool) error { return err } if dbBatchAcc != l1BatchAcc { - log.Trace("Latest sequencer batch accumulator mismatch", "batchSeqNum", checkingBatchSeqNum, "dbBatchAcc", dbBatchAcc, "l1BatchAcc", l1BatchAcc) + log.Debug("Latest sequencer batch accumulator mismatch", "batchSeqNum", checkingBatchSeqNum, "dbBatchAcc", dbBatchAcc, "l1BatchAcc", l1BatchAcc) reorgingSequencer = true } } @@ -455,7 +455,7 @@ func (r *InboxReader) run(ctx context.Context, hadError bool) error { if to.Cmp(currentHeight) > 0 { to.Set(currentHeight) } - log.Trace( + log.Debug( "Looking up messages", "from", from.String(), "to", to.String(), @@ -526,7 +526,7 @@ func (r *InboxReader) run(ctx context.Context, hadError bool) error { } } } - log.Trace( + log.Debug( "Found sequencer batches", "firstSequenceNumber", firstBatch.SequenceNumber, "newBatchesCount", len(sequencerBatches), @@ -537,7 +537,7 @@ func (r *InboxReader) run(ctx context.Context, hadError bool) error { "readLastAcc", readLastAcc, ) } else if missingSequencer && to.Cmp(currentHeight) >= 0 { - log.Trace("Didn't find expected sequencer batches", "from", from, "to", to, "currentHeight", currentHeight) + log.Debug("Didn't find expected sequencer batches", "from", from, "to", to, "currentHeight", currentHeight) // We were missing sequencer batches but didn't find any. // This must mean that the sequencer batches are in the past. reorgingSequencer = true @@ -564,7 +564,7 @@ func (r *InboxReader) run(ctx context.Context, hadError bool) error { } havePrevAcc = haveAcc } - log.Trace( + log.Debug( "Found delayed messages", "firstSequenceNumber", beforeCount, "count", len(delayedMessages), @@ -572,12 +572,12 @@ func (r *InboxReader) run(ctx context.Context, hadError bool) error { "readBeforeAcc", beforeAcc, "haveBeforeAcc", havePrevAcc, "readLastAcc", lazyHashLogging{func() common.Hash { - // Only compute this if we need to log it, as it's expensive + // Only compute this if we need to log it, as it's somewhat expensive return delayedMessages[len(delayedMessages)-1].AfterInboxAcc() }}, ) } else if missingDelayed && to.Cmp(currentHeight) >= 0 { - log.Trace("Didn't find expected delayed messages", "from", from, "to", to, "currentHeight", currentHeight) + log.Debug("Didn't find expected delayed messages", "from", from, "to", to, "currentHeight", currentHeight) // We were missing delayed messages but didn't find any. // This must mean that the delayed messages are in the past. reorgingDelayed = true diff --git a/arbnode/inbox_tracker.go b/arbnode/inbox_tracker.go index 862610100c..d2ab5ef1f5 100644 --- a/arbnode/inbox_tracker.go +++ b/arbnode/inbox_tracker.go @@ -709,7 +709,7 @@ func (t *InboxTracker) AddSequencerBatches(ctx context.Context, client arbutil.L return err } if notFound || haveDelayedAcc != batch.AfterDelayedAcc { - log.Trace( + log.Debug( "Delayed message accumulator doesn't match sequencer batch", "batch", batch.SequenceNumber, "delayedPosition", batch.AfterDelayedCount-1, From 6680d818dbeaadf8f56729ba1bcc919d861a05ee Mon Sep 17 00:00:00 2001 From: Nick Hovsmith Date: Tue, 22 Oct 2024 12:30:59 -0400 Subject: [PATCH 460/489] update flags --- Dockerfile | 2 +- Makefile | 12 +++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index aba5432254..aec8e831f3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -115,7 +115,7 @@ COPY --from=brotli-wasm-export / target/ COPY scripts/build-brotli.sh scripts/ COPY brotli brotli RUN apt-get update && apt-get install -y cmake -RUN NITRO_BUILD_IGNORE_TIMESTAMPS=1 make build-prover-header +RUN NITRO_BUILD_IGNORE_TIMESTAMPS=1 make build-prover-header ARGS="-w -d" FROM scratch AS prover-header-export COPY --from=prover-header-builder /workspace/target/ / diff --git a/Makefile b/Makefile index 64d5acfd08..2eb6ea54be 100644 --- a/Makefile +++ b/Makefile @@ -579,15 +579,9 @@ contracts/test/prover/proofs/%.json: $(arbitrator_cases)/%.wasm $(prover_bin) @touch $@ .make/cbrotli-wasm: $(DEP_PREDICATE) $(ORDER_ONLY_PREDICATE) .make -ifdef NODOCKERINDOCKER - test -f target/lib-wasm/libbrotlicommon-static.a || ./scripts/build-brotli.sh -w - test -f target/lib-wasm/libbrotlienc-static.a || ./scripts/build-brotli.sh -w - test -f target/lib-wasm/libbrotlidec-static.a || ./scripts/build-brotli.sh -w -else - test -f target/lib-wasm/libbrotlicommon-static.a || ./scripts/build-brotli.sh -w -d - test -f target/lib-wasm/libbrotlienc-static.a || ./scripts/build-brotli.sh -w -d - test -f target/lib-wasm/libbrotlidec-static.a || ./scripts/build-brotli.sh -w -d -endif + test -f target/lib-wasm/libbrotlicommon-static.a || ./scripts/build-brotli.sh $(ARGS) + test -f target/lib-wasm/libbrotlienc-static.a || ./scripts/build-brotli.sh $(ARGS) + test -f target/lib-wasm/libbrotlidec-static.a || ./scripts/build-brotli.sh $(ARGS) @touch $@ .make/wasm-lib: $(DEP_PREDICATE) arbitrator/wasm-libraries/soft-float/SoftFloat/build/Wasm-Clang/softfloat.a $(ORDER_ONLY_PREDICATE) .make From 08618e2bd61f221b950efc3e74ac6c6eea41406d Mon Sep 17 00:00:00 2001 From: Nick Hovsmith Date: Tue, 22 Oct 2024 14:08:14 -0400 Subject: [PATCH 461/489] update names and defaults --- Dockerfile | 2 +- Makefile | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index aec8e831f3..aba5432254 100644 --- a/Dockerfile +++ b/Dockerfile @@ -115,7 +115,7 @@ COPY --from=brotli-wasm-export / target/ COPY scripts/build-brotli.sh scripts/ COPY brotli brotli RUN apt-get update && apt-get install -y cmake -RUN NITRO_BUILD_IGNORE_TIMESTAMPS=1 make build-prover-header ARGS="-w -d" +RUN NITRO_BUILD_IGNORE_TIMESTAMPS=1 make build-prover-header FROM scratch AS prover-header-export COPY --from=prover-header-builder /workspace/target/ / diff --git a/Makefile b/Makefile index 2eb6ea54be..5a7227ab86 100644 --- a/Makefile +++ b/Makefile @@ -579,9 +579,10 @@ contracts/test/prover/proofs/%.json: $(arbitrator_cases)/%.wasm $(prover_bin) @touch $@ .make/cbrotli-wasm: $(DEP_PREDICATE) $(ORDER_ONLY_PREDICATE) .make - test -f target/lib-wasm/libbrotlicommon-static.a || ./scripts/build-brotli.sh $(ARGS) - test -f target/lib-wasm/libbrotlienc-static.a || ./scripts/build-brotli.sh $(ARGS) - test -f target/lib-wasm/libbrotlidec-static.a || ./scripts/build-brotli.sh $(ARGS) + CBROTLI_WASM_BUILD_ARGS ?= -w -d + test -f target/lib-wasm/libbrotlicommon-static.a || ./scripts/build-brotli.sh $(CBROTLI_WASM_BUILD_ARGS) + test -f target/lib-wasm/libbrotlienc-static.a || ./scripts/build-brotli.sh $(CBROTLI_WASM_BUILD_ARGS) + test -f target/lib-wasm/libbrotlidec-static.a || ./scripts/build-brotli.sh $(CBROTLI_WASM_BUILD_ARGS) @touch $@ .make/wasm-lib: $(DEP_PREDICATE) arbitrator/wasm-libraries/soft-float/SoftFloat/build/Wasm-Clang/softfloat.a $(ORDER_ONLY_PREDICATE) .make From 82c01951094e84de8f0115765465d4c43ddf079c Mon Sep 17 00:00:00 2001 From: Nick Hovsmith Date: Wed, 23 Oct 2024 10:40:15 -0400 Subject: [PATCH 462/489] move var --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5a7227ab86..ff285ba0fb 100644 --- a/Makefile +++ b/Makefile @@ -155,6 +155,8 @@ stylus_test_hostio-test_src = $(call get_stylus_test_rust,hostio-test) stylus_test_wasms = $(stylus_test_keccak_wasm) $(stylus_test_keccak-100_wasm) $(stylus_test_fallible_wasm) $(stylus_test_storage_wasm) $(stylus_test_multicall_wasm) $(stylus_test_log_wasm) $(stylus_test_create_wasm) $(stylus_test_math_wasm) $(stylus_test_sdk-storage_wasm) $(stylus_test_erc20_wasm) $(stylus_test_read-return-data_wasm) $(stylus_test_evm-data_wasm) $(stylus_test_hostio-test_wasm) $(stylus_test_bfs:.b=.wasm) stylus_benchmarks = $(wildcard $(stylus_dir)/*.toml $(stylus_dir)/src/*.rs) $(stylus_test_wasms) +CBROTLI_WASM_BUILD_ARGS ?= -w -d + # user targets .PHONY: push @@ -579,7 +581,6 @@ contracts/test/prover/proofs/%.json: $(arbitrator_cases)/%.wasm $(prover_bin) @touch $@ .make/cbrotli-wasm: $(DEP_PREDICATE) $(ORDER_ONLY_PREDICATE) .make - CBROTLI_WASM_BUILD_ARGS ?= -w -d test -f target/lib-wasm/libbrotlicommon-static.a || ./scripts/build-brotli.sh $(CBROTLI_WASM_BUILD_ARGS) test -f target/lib-wasm/libbrotlienc-static.a || ./scripts/build-brotli.sh $(CBROTLI_WASM_BUILD_ARGS) test -f target/lib-wasm/libbrotlidec-static.a || ./scripts/build-brotli.sh $(CBROTLI_WASM_BUILD_ARGS) From 0790c9d0d4a747a5b5d8e91cac9986804d0d3a05 Mon Sep 17 00:00:00 2001 From: Nick Hovsmith Date: Wed, 23 Oct 2024 12:25:37 -0400 Subject: [PATCH 463/489] always include -w --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index ff285ba0fb..12dfb07cf8 100644 --- a/Makefile +++ b/Makefile @@ -155,7 +155,7 @@ stylus_test_hostio-test_src = $(call get_stylus_test_rust,hostio-test) stylus_test_wasms = $(stylus_test_keccak_wasm) $(stylus_test_keccak-100_wasm) $(stylus_test_fallible_wasm) $(stylus_test_storage_wasm) $(stylus_test_multicall_wasm) $(stylus_test_log_wasm) $(stylus_test_create_wasm) $(stylus_test_math_wasm) $(stylus_test_sdk-storage_wasm) $(stylus_test_erc20_wasm) $(stylus_test_read-return-data_wasm) $(stylus_test_evm-data_wasm) $(stylus_test_hostio-test_wasm) $(stylus_test_bfs:.b=.wasm) stylus_benchmarks = $(wildcard $(stylus_dir)/*.toml $(stylus_dir)/src/*.rs) $(stylus_test_wasms) -CBROTLI_WASM_BUILD_ARGS ?= -w -d +CBROTLI_WASM_BUILD_ARGS ?=-d # user targets @@ -581,9 +581,9 @@ contracts/test/prover/proofs/%.json: $(arbitrator_cases)/%.wasm $(prover_bin) @touch $@ .make/cbrotli-wasm: $(DEP_PREDICATE) $(ORDER_ONLY_PREDICATE) .make - test -f target/lib-wasm/libbrotlicommon-static.a || ./scripts/build-brotli.sh $(CBROTLI_WASM_BUILD_ARGS) - test -f target/lib-wasm/libbrotlienc-static.a || ./scripts/build-brotli.sh $(CBROTLI_WASM_BUILD_ARGS) - test -f target/lib-wasm/libbrotlidec-static.a || ./scripts/build-brotli.sh $(CBROTLI_WASM_BUILD_ARGS) + test -f target/lib-wasm/libbrotlicommon-static.a || ./scripts/build-brotli.sh -w $(CBROTLI_WASM_BUILD_ARGS) + test -f target/lib-wasm/libbrotlienc-static.a || ./scripts/build-brotli.sh -w $(CBROTLI_WASM_BUILD_ARGS) + test -f target/lib-wasm/libbrotlidec-static.a || ./scripts/build-brotli.sh -w $(CBROTLI_WASM_BUILD_ARGS) @touch $@ .make/wasm-lib: $(DEP_PREDICATE) arbitrator/wasm-libraries/soft-float/SoftFloat/build/Wasm-Clang/softfloat.a $(ORDER_ONLY_PREDICATE) .make From 55480695262cd91722aa45955429f0c58aee9df4 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 23 Oct 2024 14:05:40 -0300 Subject: [PATCH 464/489] Do not use SetFinalizer to remove entries from preimageResolvers in a hostio machine --- validator/server_arb/machine.go | 19 +++++++++++++++++-- validator/server_arb/validator_spawner.go | 1 + 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/validator/server_arb/machine.go b/validator/server_arb/machine.go index 1e73e6b212..9617de5fcb 100644 --- a/validator/server_arb/machine.go +++ b/validator/server_arb/machine.go @@ -53,6 +53,7 @@ type ArbitratorMachine struct { ptr *C.struct_Machine contextId *int64 // has a finalizer attached to remove the preimage resolver from the global map frozen bool // does not allow anything that changes machine state, not cloned with the machine + isHostIo bool } // Assert that ArbitratorMachine implements MachineInterface @@ -71,7 +72,14 @@ func (m *ArbitratorMachine) Destroy() { // We no longer need a finalizer runtime.SetFinalizer(m, nil) } - m.contextId = nil + + // there is no finalizer related to contextId when the machine is HostIo, + // so we need to manually remove the preimage resolver + if m.isHostIo && (m.contextId != nil) { + preimageResolvers.Delete(*m.contextId) + } else { + m.contextId = nil + } } func freeContextId(context *int64) { @@ -110,6 +118,11 @@ func (m *ArbitratorMachine) Clone() *ArbitratorMachine { defer runtime.KeepAlive(m) m.mutex.Lock() defer m.mutex.Unlock() + + if m.isHostIo { + panic("it is not allowed to clone a hostio machine") + } + newMach := machineFromPointer(C.arbitrator_clone_machine(m.ptr)) newMach.contextId = m.contextId return newMach @@ -385,7 +398,9 @@ func (m *ArbitratorMachine) SetPreimageResolver(resolver GoPreimageResolver) err id := lastPreimageResolverId.Add(1) preimageResolvers.Store(id, resolver) m.contextId = &id - runtime.SetFinalizer(m.contextId, freeContextId) + if !m.isHostIo { + runtime.SetFinalizer(m.contextId, freeContextId) + } C.arbitrator_set_context(m.ptr, u64(id)) return nil } diff --git a/validator/server_arb/validator_spawner.go b/validator/server_arb/validator_spawner.go index 07971e2ba5..fc25ebac75 100644 --- a/validator/server_arb/validator_spawner.go +++ b/validator/server_arb/validator_spawner.go @@ -160,6 +160,7 @@ func (v *ArbitratorSpawner) execute( } mach := basemachine.Clone() + mach.isHostIo = true defer mach.Destroy() err = v.loadEntryToMachine(ctx, entry, mach) if err != nil { From dccdae1306a4695f92f80872851722a4fa6b8f80 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Fri, 25 Oct 2024 09:46:01 +0530 Subject: [PATCH 465/489] Attempt to clear from PEL again after successfull deletion of Lower --- pubsub/producer.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pubsub/producer.go b/pubsub/producer.go index dacaeba7d0..722c145a09 100644 --- a/pubsub/producer.go +++ b/pubsub/producer.go @@ -201,8 +201,9 @@ func (p *Producer[Request, Response]) clearMessages(ctx context.Context) time.Du } if _, err := p.client.XDel(ctx, p.redisStream, pelData.Lower).Result(); err != nil { log.Error("error deleting PEL's lower message thats past its TTL", "msgID", pelData.Lower, "err", err) - return 0 + return 5 * p.cfg.CheckResultInterval } + return 0 } } return 5 * p.cfg.CheckResultInterval From 7c595e2ddfc059d6d7e130b2937af4d49679142b Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 25 Oct 2024 10:14:25 -0300 Subject: [PATCH 466/489] Fix check that verifies if enough time has passed to allow posting a batch with only batch posting report --- arbnode/batch_poster.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arbnode/batch_poster.go b/arbnode/batch_poster.go index dcca27e1c6..a928aa6ba4 100644 --- a/arbnode/batch_poster.go +++ b/arbnode/batch_poster.go @@ -1306,7 +1306,8 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) b.building.muxBackend.delayedInbox = append(b.building.muxBackend.delayedInbox, msg) } } - if (msg.Message.Header.Kind != arbostypes.L1MessageType_BatchPostingReport) || (time.Since(firstMsgTime) >= config.MaxEmptyBatchDelay) { + timeSinceMsg := time.Since(time.Unix(int64(msg.Message.Header.Timestamp), 0)) + if (msg.Message.Header.Kind != arbostypes.L1MessageType_BatchPostingReport) || (timeSinceMsg >= config.MaxEmptyBatchDelay) { b.building.haveUsefulMessage = true if b.building.firstUsefulMsg == nil { b.building.firstUsefulMsg = msg From 489e1529f8e13164b03777f8850bfa1999581f2e Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 25 Oct 2024 11:58:53 -0300 Subject: [PATCH 467/489] Adds missing nosec G115 --- arbnode/batch_poster.go | 1 + 1 file changed, 1 insertion(+) diff --git a/arbnode/batch_poster.go b/arbnode/batch_poster.go index a928aa6ba4..46a0160b71 100644 --- a/arbnode/batch_poster.go +++ b/arbnode/batch_poster.go @@ -1306,6 +1306,7 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) b.building.muxBackend.delayedInbox = append(b.building.muxBackend.delayedInbox, msg) } } + // #nosec G115 timeSinceMsg := time.Since(time.Unix(int64(msg.Message.Header.Timestamp), 0)) if (msg.Message.Header.Kind != arbostypes.L1MessageType_BatchPostingReport) || (timeSinceMsg >= config.MaxEmptyBatchDelay) { b.building.haveUsefulMessage = true From 8f4998c881dc46ab41ae7b460d3bad33c477248c Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Mon, 28 Oct 2024 21:17:43 -0500 Subject: [PATCH 468/489] Use geth's in-tree BLS again after v1.14.0 --- blsSignatures/blsSignatures.go | 22 ++++++++++++---------- go-ethereum | 2 +- go.mod | 1 - go.sum | 3 --- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/blsSignatures/blsSignatures.go b/blsSignatures/blsSignatures.go index cfcbc34d80..b597d6a07e 100644 --- a/blsSignatures/blsSignatures.go +++ b/blsSignatures/blsSignatures.go @@ -7,8 +7,10 @@ import ( cryptorand "crypto/rand" "encoding/base64" "errors" + "math/big" + "github.com/ethereum/go-ethereum/crypto" - bls12381 "github.com/kilic/bls12-381" + "github.com/ethereum/go-ethereum/crypto/bls12381" ) type PublicKey struct { @@ -16,13 +18,13 @@ type PublicKey struct { validityProof *bls12381.PointG1 // if this is nil, key came from a trusted source } -type PrivateKey *bls12381.Fr +type PrivateKey *big.Int type Signature *bls12381.PointG1 func GeneratePrivKeyString() (string, error) { - fr := bls12381.NewFr() - privKey, err := fr.Rand(cryptorand.Reader) + g2 := bls12381.NewG2() + privKey, err := cryptorand.Int(cryptorand.Reader, g2.Q()) if err != nil { return "", err } @@ -33,8 +35,8 @@ func GeneratePrivKeyString() (string, error) { } func GenerateKeys() (PublicKey, PrivateKey, error) { - fr := bls12381.NewFr() - privateKey, err := fr.Rand(cryptorand.Reader) + g2 := bls12381.NewG2() + privateKey, err := cryptorand.Int(cryptorand.Reader, g2.Q()) if err != nil { return PublicKey{}, nil, err } @@ -118,7 +120,7 @@ func verifySignature2(sig Signature, message []byte, publicKey PublicKey, keyVal return false, err } - engine := bls12381.NewEngine() + engine := bls12381.NewPairingEngine() engine.Reset() engine.AddPair(pointOnCurve, publicKey.key) leftSide := engine.Result() @@ -154,7 +156,7 @@ func VerifyAggregatedSignatureDifferentMessages(sig Signature, messages [][]byte if len(messages) != len(pubKeys) { return false, errors.New("len(messages) does not match (len(pub keys) in verification") } - engine := bls12381.NewEngine() + engine := bls12381.NewPairingEngine() engine.Reset() for i, msg := range messages { pointOnCurve, err := hashToG1Curve(msg, false) @@ -240,11 +242,11 @@ func PublicKeyFromBytes(in []byte, trustedSource bool) (PublicKey, error) { } func PrivateKeyToBytes(priv PrivateKey) []byte { - return bls12381.NewFr().Set(priv).ToBytes() + return ((*big.Int)(priv)).Bytes() } func PrivateKeyFromBytes(in []byte) (PrivateKey, error) { - return bls12381.NewFr().FromBytes(in), nil + return new(big.Int).SetBytes(in), nil } func SignatureToBytes(sig Signature) []byte { diff --git a/go-ethereum b/go-ethereum index 2b01a1c83e..0fd8140a61 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 2b01a1c83e022983547559c070b200b95cfcab4e +Subproject commit 0fd8140a6127e77c228bc1beb0b9374244478dcd diff --git a/go.mod b/go.mod index f5a6768eb2..cbe473d15b 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,6 @@ require ( github.com/google/uuid v1.6.0 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/holiman/uint256 v1.2.4 - github.com/kilic/bls12-381 v0.1.0 github.com/knadh/koanf v1.4.0 github.com/mailru/easygo v0.0.0-20190618140210-3c14a0dc985f github.com/mitchellh/mapstructure v1.4.1 diff --git a/go.sum b/go.sum index 419b1acfae..25b594cc25 100644 --- a/go.sum +++ b/go.sum @@ -441,8 +441,6 @@ github.com/juju/utils v0.0.0-20180808125547-9dfc6dbfb02b/go.mod h1:6/KLg8Wz/y2KV github.com/juju/version v0.0.0-20161031051906-1f41e27e54f2/go.mod h1:kE8gK5X0CImdr7qpSKl3xB2PmpySSmfj7zVbkZFs81U= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kilic/bls12-381 v0.1.0 h1:encrdjqKMEvabVQ7qYOKu1OvhqpK4s47wDYtNiPtlp4= -github.com/kilic/bls12-381 v0.1.0/go.mod h1:vDTTHJONJ6G+P2R74EhnyotQDTliQDnFEwhdmfzw1ig= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= @@ -810,7 +808,6 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From cffb7930b05c064b37a4452297367f44ed0061e3 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 30 Oct 2024 09:14:08 -0300 Subject: [PATCH 469/489] Revert "Do not use SetFinalizer to remove entries from preimageResolvers in a hostio machine" This reverts commit 55480695262cd91722aa45955429f0c58aee9df4. --- validator/server_arb/machine.go | 19 ++----------------- validator/server_arb/validator_spawner.go | 1 - 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/validator/server_arb/machine.go b/validator/server_arb/machine.go index 9617de5fcb..1e73e6b212 100644 --- a/validator/server_arb/machine.go +++ b/validator/server_arb/machine.go @@ -53,7 +53,6 @@ type ArbitratorMachine struct { ptr *C.struct_Machine contextId *int64 // has a finalizer attached to remove the preimage resolver from the global map frozen bool // does not allow anything that changes machine state, not cloned with the machine - isHostIo bool } // Assert that ArbitratorMachine implements MachineInterface @@ -72,14 +71,7 @@ func (m *ArbitratorMachine) Destroy() { // We no longer need a finalizer runtime.SetFinalizer(m, nil) } - - // there is no finalizer related to contextId when the machine is HostIo, - // so we need to manually remove the preimage resolver - if m.isHostIo && (m.contextId != nil) { - preimageResolvers.Delete(*m.contextId) - } else { - m.contextId = nil - } + m.contextId = nil } func freeContextId(context *int64) { @@ -118,11 +110,6 @@ func (m *ArbitratorMachine) Clone() *ArbitratorMachine { defer runtime.KeepAlive(m) m.mutex.Lock() defer m.mutex.Unlock() - - if m.isHostIo { - panic("it is not allowed to clone a hostio machine") - } - newMach := machineFromPointer(C.arbitrator_clone_machine(m.ptr)) newMach.contextId = m.contextId return newMach @@ -398,9 +385,7 @@ func (m *ArbitratorMachine) SetPreimageResolver(resolver GoPreimageResolver) err id := lastPreimageResolverId.Add(1) preimageResolvers.Store(id, resolver) m.contextId = &id - if !m.isHostIo { - runtime.SetFinalizer(m.contextId, freeContextId) - } + runtime.SetFinalizer(m.contextId, freeContextId) C.arbitrator_set_context(m.ptr, u64(id)) return nil } diff --git a/validator/server_arb/validator_spawner.go b/validator/server_arb/validator_spawner.go index fc25ebac75..07971e2ba5 100644 --- a/validator/server_arb/validator_spawner.go +++ b/validator/server_arb/validator_spawner.go @@ -160,7 +160,6 @@ func (v *ArbitratorSpawner) execute( } mach := basemachine.Clone() - mach.isHostIo = true defer mach.Destroy() err = v.loadEntryToMachine(ctx, entry, mach) if err != nil { From 5390b0e09fbd9e1160aebbaedb2fe52e2b382988 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 30 Oct 2024 10:00:17 -0300 Subject: [PATCH 470/489] Uses preimageResolverRefCounter instead of relying on finalizers to remove preimage resolvers from global map --- validator/server_arb/machine.go | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/validator/server_arb/machine.go b/validator/server_arb/machine.go index 1e73e6b212..5e381db82f 100644 --- a/validator/server_arb/machine.go +++ b/validator/server_arb/machine.go @@ -51,14 +51,15 @@ type MachineInterface interface { type ArbitratorMachine struct { mutex sync.Mutex // needed because go finalizers don't synchronize (meaning they aren't thread safe) ptr *C.struct_Machine - contextId *int64 // has a finalizer attached to remove the preimage resolver from the global map - frozen bool // does not allow anything that changes machine state, not cloned with the machine + contextId int64 + frozen bool // does not allow anything that changes machine state, not cloned with the machine } // Assert that ArbitratorMachine implements MachineInterface var _ MachineInterface = (*ArbitratorMachine)(nil) var preimageResolvers containers.SyncMap[int64, GoPreimageResolver] +var preimageResolverRefCounter containers.SyncMap[int64, *atomic.Int64] var lastPreimageResolverId atomic.Int64 // atomic // Any future calls to this machine will result in a panic @@ -68,14 +69,14 @@ func (m *ArbitratorMachine) Destroy() { if m.ptr != nil { C.arbitrator_free_machine(m.ptr) m.ptr = nil - // We no longer need a finalizer - runtime.SetFinalizer(m, nil) } - m.contextId = nil -} - -func freeContextId(context *int64) { - preimageResolvers.Delete(*context) + refCounter, ok := preimageResolverRefCounter.Load(m.contextId) + if ok { + if refCounter.Add(-1) == 0 { + preimageResolverRefCounter.Delete(m.contextId) + preimageResolvers.Delete(m.contextId) + } + } } func machineFromPointer(ptr *C.struct_Machine) *ArbitratorMachine { @@ -112,6 +113,10 @@ func (m *ArbitratorMachine) Clone() *ArbitratorMachine { defer m.mutex.Unlock() newMach := machineFromPointer(C.arbitrator_clone_machine(m.ptr)) newMach.contextId = m.contextId + refCounter, ok := preimageResolverRefCounter.Load(m.contextId) + if ok { + refCounter.Add(1) + } return newMach } @@ -384,8 +389,10 @@ func (m *ArbitratorMachine) SetPreimageResolver(resolver GoPreimageResolver) err } id := lastPreimageResolverId.Add(1) preimageResolvers.Store(id, resolver) - m.contextId = &id - runtime.SetFinalizer(m.contextId, freeContextId) + refCounter := atomic.Int64{} + refCounter.Store(1) + preimageResolverRefCounter.Store(id, &refCounter) + m.contextId = id C.arbitrator_set_context(m.ptr, u64(id)) return nil } From b7ba9c306b9dc3124979664ee578f37ea7307f0c Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 30 Oct 2024 10:16:11 -0300 Subject: [PATCH 471/489] Reuse same global map to keep preimage resolver reference counter --- validator/server_arb/machine.go | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/validator/server_arb/machine.go b/validator/server_arb/machine.go index 5e381db82f..aa4ab85706 100644 --- a/validator/server_arb/machine.go +++ b/validator/server_arb/machine.go @@ -58,8 +58,7 @@ type ArbitratorMachine struct { // Assert that ArbitratorMachine implements MachineInterface var _ MachineInterface = (*ArbitratorMachine)(nil) -var preimageResolvers containers.SyncMap[int64, GoPreimageResolver] -var preimageResolverRefCounter containers.SyncMap[int64, *atomic.Int64] +var preimageResolvers containers.SyncMap[int64, goPreimageResolverWithRefCounter] var lastPreimageResolverId atomic.Int64 // atomic // Any future calls to this machine will result in a panic @@ -70,10 +69,9 @@ func (m *ArbitratorMachine) Destroy() { C.arbitrator_free_machine(m.ptr) m.ptr = nil } - refCounter, ok := preimageResolverRefCounter.Load(m.contextId) + resolverWithRefCounter, ok := preimageResolvers.Load(m.contextId) if ok { - if refCounter.Add(-1) == 0 { - preimageResolverRefCounter.Delete(m.contextId) + if resolverWithRefCounter.refCounter.Add(-1) == 0 { preimageResolvers.Delete(m.contextId) } } @@ -113,10 +111,12 @@ func (m *ArbitratorMachine) Clone() *ArbitratorMachine { defer m.mutex.Unlock() newMach := machineFromPointer(C.arbitrator_clone_machine(m.ptr)) newMach.contextId = m.contextId - refCounter, ok := preimageResolverRefCounter.Load(m.contextId) + + resolverWithRefCounter, ok := preimageResolvers.Load(m.contextId) if ok { - refCounter.Add(1) + resolverWithRefCounter.refCounter.Add(1) } + return newMach } @@ -355,19 +355,23 @@ func (m *ArbitratorMachine) AddDelayedInboxMessage(index uint64, data []byte) er } type GoPreimageResolver = func(arbutil.PreimageType, common.Hash) ([]byte, error) +type goPreimageResolverWithRefCounter struct { + resolver GoPreimageResolver + refCounter *atomic.Int64 +} //export preimageResolver func preimageResolver(context C.size_t, ty C.uint8_t, ptr unsafe.Pointer) C.ResolvedPreimage { var hash common.Hash input := (*[1 << 30]byte)(ptr)[:32] copy(hash[:], input) - resolver, ok := preimageResolvers.Load(int64(context)) + resolverWithRefCounter, ok := preimageResolvers.Load(int64(context)) if !ok { return C.ResolvedPreimage{ len: -1, } } - preimage, err := resolver(arbutil.PreimageType(ty), hash) + preimage, err := resolverWithRefCounter.resolver(arbutil.PreimageType(ty), hash) if err != nil { log.Error("preimage resolution failed", "err", err) return C.ResolvedPreimage{ @@ -388,10 +392,15 @@ func (m *ArbitratorMachine) SetPreimageResolver(resolver GoPreimageResolver) err return errors.New("machine frozen") } id := lastPreimageResolverId.Add(1) - preimageResolvers.Store(id, resolver) + refCounter := atomic.Int64{} refCounter.Store(1) - preimageResolverRefCounter.Store(id, &refCounter) + resolverWithRefCounter := goPreimageResolverWithRefCounter{ + resolver: resolver, + refCounter: &refCounter, + } + preimageResolvers.Store(id, resolverWithRefCounter) + m.contextId = id C.arbitrator_set_context(m.ptr, u64(id)) return nil From 98ccb2dfda661686b2bec7c6ec25fa0ae6dd69b2 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 30 Oct 2024 14:45:03 -0300 Subject: [PATCH 472/489] TestEntriesAreDeletedFromPreimageResolversGlobalMap --- util/containers/syncmap.go | 10 ++++ validator/server_arb/machine_test.go | 75 ++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 validator/server_arb/machine_test.go diff --git a/util/containers/syncmap.go b/util/containers/syncmap.go index 7952a32252..e24d56fda6 100644 --- a/util/containers/syncmap.go +++ b/util/containers/syncmap.go @@ -22,3 +22,13 @@ func (m *SyncMap[K, V]) Store(key K, val V) { func (m *SyncMap[K, V]) Delete(key K) { m.internal.Delete(key) } + +// Only used for testing +func (m *SyncMap[K, V]) Keys() []K { + s := make([]K, 0) + m.internal.Range(func(k, v interface{}) bool { + s = append(s, k.(K)) + return true + }) + return s +} diff --git a/validator/server_arb/machine_test.go b/validator/server_arb/machine_test.go new file mode 100644 index 0000000000..3255970350 --- /dev/null +++ b/validator/server_arb/machine_test.go @@ -0,0 +1,75 @@ +// Copyright 2021-2024, Offchain Labs, Inc. +// For license information, see https://github.com/nitro/blob/master/LICENSE + +package server_arb + +import ( + "path" + "reflect" + "runtime" + "sort" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/offchainlabs/nitro/arbutil" + "github.com/offchainlabs/nitro/util/testhelpers" +) + +func TestEntriesAreDeletedFromPreimageResolversGlobalMap(t *testing.T) { + resolver := func(arbutil.PreimageType, common.Hash) ([]byte, error) { + return nil, nil + } + + sortedKeys := func() []int64 { + keys := preimageResolvers.Keys() + sort.Slice(keys, func(i, j int) bool { + return keys[i] < keys[j] + }) + return keys + } + + // clear global map before running test + preimageKeys := sortedKeys() + for _, key := range preimageKeys { + preimageResolvers.Delete(key) + } + + _, filename, _, _ := runtime.Caller(0) + wasmDir := path.Join(path.Dir(filename), "../../arbitrator/prover/test-cases/") + wasmPath := path.Join(wasmDir, "global-state.wasm") + modulePaths := []string{path.Join(wasmDir, "global-state-wrapper.wasm")} + + machine1, err := LoadSimpleMachine(wasmPath, modulePaths, true) + testhelpers.RequireImpl(t, err) + err = machine1.SetPreimageResolver(resolver) + testhelpers.RequireImpl(t, err) + + machine2, err := LoadSimpleMachine(wasmPath, modulePaths, true) + testhelpers.RequireImpl(t, err) + err = machine2.SetPreimageResolver(resolver) + testhelpers.RequireImpl(t, err) + + machine1_clone1 := machine1.Clone() + machine1_clone2 := machine1.Clone() + + checkKeys := func(expectedKeys []int64, scenario string) { + keys := sortedKeys() + if !reflect.DeepEqual(keys, expectedKeys) { + t.Fatal("Unexpected preimageResolversKeys got", keys, "expected", expectedKeys, "scenario", scenario) + } + } + + checkKeys([]int64{machine1.contextId, machine2.contextId}, "initial") + + machine1_clone1.Destroy() + checkKeys([]int64{machine1.contextId, machine2.contextId}, "after machine1_clone1 destroy") + + machine1_clone2.Destroy() + checkKeys([]int64{machine1.contextId, machine2.contextId}, "after machine1_clone2 destroy") + + machine2.Destroy() + checkKeys([]int64{machine1.contextId}, "after machine2 destroy") + + machine1.Destroy() + checkKeys([]int64{}, "after machine1 destroy") +} From efc592f2c37029d55ab9f1686c3d0f9d996efab8 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 30 Oct 2024 15:18:09 -0300 Subject: [PATCH 473/489] Adds machine's finalizer back --- validator/server_arb/machine.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/validator/server_arb/machine.go b/validator/server_arb/machine.go index aa4ab85706..a9475c7f7c 100644 --- a/validator/server_arb/machine.go +++ b/validator/server_arb/machine.go @@ -68,6 +68,8 @@ func (m *ArbitratorMachine) Destroy() { if m.ptr != nil { C.arbitrator_free_machine(m.ptr) m.ptr = nil + // We no longer need a finalizer + runtime.SetFinalizer(m, nil) } resolverWithRefCounter, ok := preimageResolvers.Load(m.contextId) if ok { From 7ebf2dcf4b6e1fe0979ccee29f1a402fc2fc06bc Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 30 Oct 2024 16:16:31 -0300 Subject: [PATCH 474/489] Change machine destroy order in TestEntriesAreDeletedFromPreimageResolversGlobalMap --- validator/server_arb/machine_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/validator/server_arb/machine_test.go b/validator/server_arb/machine_test.go index 3255970350..b6b64c4b53 100644 --- a/validator/server_arb/machine_test.go +++ b/validator/server_arb/machine_test.go @@ -64,12 +64,12 @@ func TestEntriesAreDeletedFromPreimageResolversGlobalMap(t *testing.T) { machine1_clone1.Destroy() checkKeys([]int64{machine1.contextId, machine2.contextId}, "after machine1_clone1 destroy") + machine1.Destroy() + checkKeys([]int64{machine1.contextId, machine2.contextId}, "after machine1 destroy") + machine1_clone2.Destroy() - checkKeys([]int64{machine1.contextId, machine2.contextId}, "after machine1_clone2 destroy") + checkKeys([]int64{machine2.contextId}, "after machine1_clone2 destroy") machine2.Destroy() - checkKeys([]int64{machine1.contextId}, "after machine2 destroy") - - machine1.Destroy() - checkKeys([]int64{}, "after machine1 destroy") + checkKeys([]int64{}, "after machine2 destroy") } From 0b4dbef3a3e566aa376bde81ad66bd86dcacdfd9 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 30 Oct 2024 16:36:08 -0600 Subject: [PATCH 475/489] fix MockTransfer in trace --- arbos/util/transfer.go | 6 ++++-- system_tests/program_gas_test.go | 6 +----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/arbos/util/transfer.go b/arbos/util/transfer.go index e186dee0fa..ed5e07c8cf 100644 --- a/arbos/util/transfer.go +++ b/arbos/util/transfer.go @@ -36,8 +36,10 @@ func TransferBalance( return errors.New("tracing scenario mismatch") } - if scenario != TracingDuringEVM && tracer.CaptureArbitrumTransfer != nil { - tracer.CaptureArbitrumTransfer(from, to, amount, scenario == TracingBeforeEVM, purpose) + if scenario != TracingDuringEVM { + if tracer.CaptureArbitrumTransfer != nil { + tracer.CaptureArbitrumTransfer(from, to, amount, scenario == TracingBeforeEVM, purpose) + } } else { fromCopy := from toCopy := to diff --git a/system_tests/program_gas_test.go b/system_tests/program_gas_test.go index 03f63ff39e..10a371532d 100644 --- a/system_tests/program_gas_test.go +++ b/system_tests/program_gas_test.go @@ -353,7 +353,7 @@ func compareGasUsage( switch mode { case compareGasForEach: if len(evmGasUsage[opcode]) != len(stylusGasUsage[hostio]) { - Fatal(t, "mismatch between hostios and opcodes", evmGasUsage, stylusGasUsage) + Fatal(t, "mismatch between opcode ", opcode, " - ", evmGasUsage[opcode], " and hostio ", hostio, " - ", stylusGasUsage[hostio]) } for i := range evmGasUsage[opcode] { opcodeGas := evmGasUsage[opcode][i] @@ -390,10 +390,6 @@ func evmOpcodesGasUsage(ctx context.Context, rpcClient rpc.ClientInterface, tx * op := vm.StringToOp(result.StructLogs[i].Op) gasUsed := uint64(0) if op == vm.CALL || op == vm.STATICCALL || op == vm.DELEGATECALL || op == vm.CREATE || op == vm.CREATE2 { - if result.StructLogs[i].GasCost == 0 { - // ignore mock call emitted by arbos - continue - } // For the CALL* opcodes, the GasCost in the tracer represents the gas sent // to the callee contract, which is 63/64 of the remaining gas. This happens // because the tracer is evaluated before the call is executed, so the EVM From 493f45bce37060a030bad7286524c00dc978b727 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 1 Nov 2024 14:48:54 -0300 Subject: [PATCH 476/489] Sets contextId to zero when destroying a machine --- validator/server_arb/machine.go | 1 + validator/server_arb/machine_test.go | 24 +++++++++++++++--------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/validator/server_arb/machine.go b/validator/server_arb/machine.go index a9475c7f7c..198e76370c 100644 --- a/validator/server_arb/machine.go +++ b/validator/server_arb/machine.go @@ -77,6 +77,7 @@ func (m *ArbitratorMachine) Destroy() { preimageResolvers.Delete(m.contextId) } } + m.contextId = 0 } func machineFromPointer(ptr *C.struct_Machine) *ArbitratorMachine { diff --git a/validator/server_arb/machine_test.go b/validator/server_arb/machine_test.go index b6b64c4b53..cd3e326385 100644 --- a/validator/server_arb/machine_test.go +++ b/validator/server_arb/machine_test.go @@ -49,8 +49,8 @@ func TestEntriesAreDeletedFromPreimageResolversGlobalMap(t *testing.T) { err = machine2.SetPreimageResolver(resolver) testhelpers.RequireImpl(t, err) - machine1_clone1 := machine1.Clone() - machine1_clone2 := machine1.Clone() + machine1Clone1 := machine1.Clone() + machine1Clone2 := machine1.Clone() checkKeys := func(expectedKeys []int64, scenario string) { keys := sortedKeys() @@ -59,17 +59,23 @@ func TestEntriesAreDeletedFromPreimageResolversGlobalMap(t *testing.T) { } } - checkKeys([]int64{machine1.contextId, machine2.contextId}, "initial") + machine1ContextId := machine1.contextId + machine2ContextId := machine2.contextId - machine1_clone1.Destroy() - checkKeys([]int64{machine1.contextId, machine2.contextId}, "after machine1_clone1 destroy") + checkKeys([]int64{machine1ContextId, machine2ContextId}, "initial") + + machine1Clone1.Destroy() + checkKeys([]int64{machine1ContextId, machine2ContextId}, "after machine1Clone1 is destroyed") + + machine1.Destroy() + checkKeys([]int64{machine1ContextId, machine2ContextId}, "after machine1 is destroyed") machine1.Destroy() - checkKeys([]int64{machine1.contextId, machine2.contextId}, "after machine1 destroy") + checkKeys([]int64{machine1ContextId, machine2ContextId}, "after machine1 is destroyed again") - machine1_clone2.Destroy() - checkKeys([]int64{machine2.contextId}, "after machine1_clone2 destroy") + machine1Clone2.Destroy() + checkKeys([]int64{machine2ContextId}, "after machine1Clone2 is destroyed") machine2.Destroy() - checkKeys([]int64{}, "after machine2 destroy") + checkKeys([]int64{}, "after machine2 is destroyed") } From 7ca0f246b82fa180dc2d2f80d3f70c0a089fcc1e Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 1 Nov 2024 14:55:49 -0300 Subject: [PATCH 477/489] contextId as pointer to avoid worrying about what are the valid context ids --- validator/server_arb/machine.go | 25 +++++++++++++++---------- validator/server_arb/machine_test.go | 4 ++-- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/validator/server_arb/machine.go b/validator/server_arb/machine.go index 198e76370c..b2312368c3 100644 --- a/validator/server_arb/machine.go +++ b/validator/server_arb/machine.go @@ -51,7 +51,7 @@ type MachineInterface interface { type ArbitratorMachine struct { mutex sync.Mutex // needed because go finalizers don't synchronize (meaning they aren't thread safe) ptr *C.struct_Machine - contextId int64 + contextId *int64 frozen bool // does not allow anything that changes machine state, not cloned with the machine } @@ -71,13 +71,16 @@ func (m *ArbitratorMachine) Destroy() { // We no longer need a finalizer runtime.SetFinalizer(m, nil) } - resolverWithRefCounter, ok := preimageResolvers.Load(m.contextId) - if ok { - if resolverWithRefCounter.refCounter.Add(-1) == 0 { - preimageResolvers.Delete(m.contextId) + + if m.contextId != nil { + resolverWithRefCounter, ok := preimageResolvers.Load(*m.contextId) + if ok { + if resolverWithRefCounter.refCounter.Add(-1) == 0 { + preimageResolvers.Delete(*m.contextId) + } } } - m.contextId = 0 + m.contextId = nil } func machineFromPointer(ptr *C.struct_Machine) *ArbitratorMachine { @@ -115,9 +118,11 @@ func (m *ArbitratorMachine) Clone() *ArbitratorMachine { newMach := machineFromPointer(C.arbitrator_clone_machine(m.ptr)) newMach.contextId = m.contextId - resolverWithRefCounter, ok := preimageResolvers.Load(m.contextId) - if ok { - resolverWithRefCounter.refCounter.Add(1) + if m.contextId != nil { + resolverWithRefCounter, ok := preimageResolvers.Load(*m.contextId) + if ok { + resolverWithRefCounter.refCounter.Add(1) + } } return newMach @@ -404,7 +409,7 @@ func (m *ArbitratorMachine) SetPreimageResolver(resolver GoPreimageResolver) err } preimageResolvers.Store(id, resolverWithRefCounter) - m.contextId = id + m.contextId = &id C.arbitrator_set_context(m.ptr, u64(id)) return nil } diff --git a/validator/server_arb/machine_test.go b/validator/server_arb/machine_test.go index cd3e326385..e18ee5788c 100644 --- a/validator/server_arb/machine_test.go +++ b/validator/server_arb/machine_test.go @@ -59,8 +59,8 @@ func TestEntriesAreDeletedFromPreimageResolversGlobalMap(t *testing.T) { } } - machine1ContextId := machine1.contextId - machine2ContextId := machine2.contextId + machine1ContextId := *machine1.contextId + machine2ContextId := *machine2.contextId checkKeys([]int64{machine1ContextId, machine2ContextId}, "initial") From d71865a2231bd815b97beff2a824b5069c8b9a8a Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 1 Nov 2024 14:58:45 -0300 Subject: [PATCH 478/489] Simplifies if --- validator/server_arb/machine.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/validator/server_arb/machine.go b/validator/server_arb/machine.go index b2312368c3..e8a4164ed5 100644 --- a/validator/server_arb/machine.go +++ b/validator/server_arb/machine.go @@ -74,10 +74,8 @@ func (m *ArbitratorMachine) Destroy() { if m.contextId != nil { resolverWithRefCounter, ok := preimageResolvers.Load(*m.contextId) - if ok { - if resolverWithRefCounter.refCounter.Add(-1) == 0 { - preimageResolvers.Delete(*m.contextId) - } + if ok && (resolverWithRefCounter.refCounter.Add(-1) == 0) { + preimageResolvers.Delete(*m.contextId) } } m.contextId = nil From 4aaee14bfe8eec32578da8e2a8e90d998ba827b4 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 4 Nov 2024 09:41:33 -0300 Subject: [PATCH 479/489] Handle SetPreimageResolver being called multiple times for the same machine --- validator/server_arb/machine.go | 19 ++++++++++++------- validator/server_arb/machine_test.go | 12 ++++++++++++ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/validator/server_arb/machine.go b/validator/server_arb/machine.go index e8a4164ed5..0291185a54 100644 --- a/validator/server_arb/machine.go +++ b/validator/server_arb/machine.go @@ -61,6 +61,15 @@ var _ MachineInterface = (*ArbitratorMachine)(nil) var preimageResolvers containers.SyncMap[int64, goPreimageResolverWithRefCounter] var lastPreimageResolverId atomic.Int64 // atomic +func dereferenceContextId(contextId *int64) { + if contextId != nil { + resolverWithRefCounter, ok := preimageResolvers.Load(*contextId) + if ok && (resolverWithRefCounter.refCounter.Add(-1) == 0) { + preimageResolvers.Delete(*contextId) + } + } +} + // Any future calls to this machine will result in a panic func (m *ArbitratorMachine) Destroy() { m.mutex.Lock() @@ -72,12 +81,7 @@ func (m *ArbitratorMachine) Destroy() { runtime.SetFinalizer(m, nil) } - if m.contextId != nil { - resolverWithRefCounter, ok := preimageResolvers.Load(*m.contextId) - if ok && (resolverWithRefCounter.refCounter.Add(-1) == 0) { - preimageResolvers.Delete(*m.contextId) - } - } + dereferenceContextId(m.contextId) m.contextId = nil } @@ -397,8 +401,9 @@ func (m *ArbitratorMachine) SetPreimageResolver(resolver GoPreimageResolver) err if m.frozen { return errors.New("machine frozen") } - id := lastPreimageResolverId.Add(1) + dereferenceContextId(m.contextId) + id := lastPreimageResolverId.Add(1) refCounter := atomic.Int64{} refCounter.Store(1) resolverWithRefCounter := goPreimageResolverWithRefCounter{ diff --git a/validator/server_arb/machine_test.go b/validator/server_arb/machine_test.go index e18ee5788c..e3ffb28b42 100644 --- a/validator/server_arb/machine_test.go +++ b/validator/server_arb/machine_test.go @@ -64,15 +64,27 @@ func TestEntriesAreDeletedFromPreimageResolversGlobalMap(t *testing.T) { checkKeys([]int64{machine1ContextId, machine2ContextId}, "initial") + // the machine's contextId should change when setting preimage resolver for the second time, + // and the entry for the old context id should be deleted + err = machine2.SetPreimageResolver(resolver) + testhelpers.RequireImpl(t, err) + if machine2ContextId == *machine2.contextId { + t.Fatal("Context id didn't change after setting preimage resolver for the second time") + } + machine2ContextId = *machine2.contextId + checkKeys([]int64{machine1ContextId, machine2ContextId}, "after setting preimage resolver for machine2 for the second time") + machine1Clone1.Destroy() checkKeys([]int64{machine1ContextId, machine2ContextId}, "after machine1Clone1 is destroyed") machine1.Destroy() checkKeys([]int64{machine1ContextId, machine2ContextId}, "after machine1 is destroyed") + // it is possible to destroy the same machine multiple times machine1.Destroy() checkKeys([]int64{machine1ContextId, machine2ContextId}, "after machine1 is destroyed again") + // entry for machine1ContextId should be deleted only after machine1 and all its clones are destroyed machine1Clone2.Destroy() checkKeys([]int64{machine2ContextId}, "after machine1Clone2 is destroyed") From 8b81224a58c8169c430efed30c6342e01448c6e0 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 4 Nov 2024 12:21:03 -0300 Subject: [PATCH 480/489] panic in case when resolver is not found, or when ref counter is negative --- validator/server_arb/machine.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/validator/server_arb/machine.go b/validator/server_arb/machine.go index 0291185a54..20b236e5a7 100644 --- a/validator/server_arb/machine.go +++ b/validator/server_arb/machine.go @@ -64,7 +64,14 @@ var lastPreimageResolverId atomic.Int64 // atomic func dereferenceContextId(contextId *int64) { if contextId != nil { resolverWithRefCounter, ok := preimageResolvers.Load(*contextId) - if ok && (resolverWithRefCounter.refCounter.Add(-1) == 0) { + if !ok { + panic(fmt.Sprintf("dereferenceContextId: resolver with ref counter not found, contextId: %v", *contextId)) + } + + refCounterAfter := resolverWithRefCounter.refCounter.Add(-1) + if refCounterAfter < 0 { + panic(fmt.Sprintf("dereferenceContextId: ref counter is negative, contextId: %v", *contextId)) + } else if refCounterAfter == 0 { preimageResolvers.Delete(*contextId) } } @@ -124,6 +131,8 @@ func (m *ArbitratorMachine) Clone() *ArbitratorMachine { resolverWithRefCounter, ok := preimageResolvers.Load(*m.contextId) if ok { resolverWithRefCounter.refCounter.Add(1) + } else { + panic(fmt.Sprintf("Clone: resolver with ref counter not found, contextId: %v", *m.contextId)) } } @@ -377,6 +386,7 @@ func preimageResolver(context C.size_t, ty C.uint8_t, ptr unsafe.Pointer) C.Reso copy(hash[:], input) resolverWithRefCounter, ok := preimageResolvers.Load(int64(context)) if !ok { + log.Error("preimageResolver: resolver with ref counter not found", "context", int64(context)) return C.ResolvedPreimage{ len: -1, } From f7f90d20a22ebeacf579f72686176c0ad6635b99 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 4 Nov 2024 12:35:45 -0300 Subject: [PATCH 481/489] renames refCounterAfter to refCount --- validator/server_arb/machine.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/validator/server_arb/machine.go b/validator/server_arb/machine.go index 20b236e5a7..f882fe65a6 100644 --- a/validator/server_arb/machine.go +++ b/validator/server_arb/machine.go @@ -68,10 +68,10 @@ func dereferenceContextId(contextId *int64) { panic(fmt.Sprintf("dereferenceContextId: resolver with ref counter not found, contextId: %v", *contextId)) } - refCounterAfter := resolverWithRefCounter.refCounter.Add(-1) - if refCounterAfter < 0 { + refCount := resolverWithRefCounter.refCounter.Add(-1) + if refCount < 0 { panic(fmt.Sprintf("dereferenceContextId: ref counter is negative, contextId: %v", *contextId)) - } else if refCounterAfter == 0 { + } else if refCount == 0 { preimageResolvers.Delete(*contextId) } } From 84dfaf17ba9524b02ae5f4e912ab49389f0529ec Mon Sep 17 00:00:00 2001 From: Joshua Colvin Date: Mon, 4 Nov 2024 11:44:20 -0700 Subject: [PATCH 482/489] fix variable name --- scripts/check-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check-build.sh b/scripts/check-build.sh index ee79d12b8b..6084900f96 100755 --- a/scripts/check-build.sh +++ b/scripts/check-build.sh @@ -43,7 +43,7 @@ fi # Step 3: Check the version tag VERSION_TAG=$(git tag --points-at HEAD | sed '/-/!s/$/_/' | sort -rV | sed 's/_$//' | head -n 1 | grep ^ || git show -s --pretty=%D | sed 's/, /\n/g' | grep -v '^origin/' | grep -v '^grafted\|HEAD\|master\|main$' || echo "") -if [[ -z "${VAR}" ]]; then +if [[ -z "${VERSION_TAG}" ]]; then echo -e "${YELLOW}Untagged version of Nitro checked out, may not be fully tested.${NC}" else echo -e "${GREEN}You are on Nitro version tag: $VERSION_TAG${NC}" From d56470333ae20de5330404963207012919d18303 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Tue, 5 Nov 2024 16:17:52 +0530 Subject: [PATCH 483/489] Update geth --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index cdbadc1c4d..fc0b413ef7 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit cdbadc1c4d6df98dc2e7b07a6fcdb0259e58a046 +Subproject commit fc0b413ef764c368a7e6e0af17c47d3e73bbb361 From 60e7281f6eeb4c4b1b2adf09878d23b69d077612 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Tue, 5 Nov 2024 16:21:36 +0530 Subject: [PATCH 484/489] fix build --- arbos/arbosState/initialization_test.go | 4 +--- arbos/arbosState/initialize.go | 6 +----- system_tests/state_fuzz_test.go | 2 +- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/arbos/arbosState/initialization_test.go b/arbos/arbosState/initialization_test.go index 4bee02f9df..697a41e4fd 100644 --- a/arbos/arbosState/initialization_test.go +++ b/arbos/arbosState/initialization_test.go @@ -66,9 +66,7 @@ func tryMarshalUnmarshal(input *statetransfer.ArbosInitializationInfo, t *testin cacheConfig := core.DefaultCacheConfigWithScheme(env.GetTestStateScheme()) stateroot, err := InitializeArbosInDatabase(raw, cacheConfig, initReader, chainConfig, arbostypes.TestInitMessage, 0, 0) Require(t, err) - number, err := initReader.GetNextBlockNumber() - Require(t, err) - triedbConfig := cacheConfig.TriedbConfig(chainConfig.IsVerkle(new(big.Int).SetUint64(number), 0)) + triedbConfig := cacheConfig.TriedbConfig() stateDb, err := state.New(stateroot, state.NewDatabaseWithConfig(raw, triedbConfig), nil) Require(t, err) diff --git a/arbos/arbosState/initialize.go b/arbos/arbosState/initialize.go index eef15869a2..374eff2cea 100644 --- a/arbos/arbosState/initialize.go +++ b/arbos/arbosState/initialize.go @@ -56,11 +56,7 @@ func MakeGenesisBlock(parentHash common.Hash, blockNumber uint64, timestamp uint } func InitializeArbosInDatabase(db ethdb.Database, cacheConfig *core.CacheConfig, initData statetransfer.InitDataReader, chainConfig *params.ChainConfig, initMessage *arbostypes.ParsedInitMessage, timestamp uint64, accountsPerSync uint) (root common.Hash, err error) { - number, err := initData.GetNextBlockNumber() - if err != nil { - return common.Hash{}, err - } - triedbConfig := cacheConfig.TriedbConfig(chainConfig.IsVerkle(new(big.Int).SetUint64(number), timestamp)) + triedbConfig := cacheConfig.TriedbConfig() triedbConfig.Preimages = false stateDatabase := state.NewDatabaseWithConfig(db, triedbConfig) defer func() { diff --git a/system_tests/state_fuzz_test.go b/system_tests/state_fuzz_test.go index 132bb7cc44..c8312350e6 100644 --- a/system_tests/state_fuzz_test.go +++ b/system_tests/state_fuzz_test.go @@ -159,7 +159,7 @@ func FuzzStateTransition(f *testing.F) { if err != nil { panic(err) } - trieDBConfig := cacheConfig.TriedbConfig(chainConfig.IsVerkle(new(big.Int), 0)) + trieDBConfig := cacheConfig.TriedbConfig() statedb, err := state.New(stateRoot, state.NewDatabaseWithConfig(chainDb, trieDBConfig), nil) if err != nil { panic(err) From 09f9fde8938197fbd3cedc06de70ba6e9574e297 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Tue, 5 Nov 2024 16:46:36 +0530 Subject: [PATCH 485/489] Update geth --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index fc0b413ef7..c4d8abf274 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit fc0b413ef764c368a7e6e0af17c47d3e73bbb361 +Subproject commit c4d8abf2741bb628fc905b24a2d50c6f56539f4a From 6566ff06e8df0c92a1528da98a685dd3aa0a55dd Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Tue, 5 Nov 2024 20:02:59 +0530 Subject: [PATCH 486/489] Changes based on PR comments --- arbnode/batch_poster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arbnode/batch_poster.go b/arbnode/batch_poster.go index e5f31a1eee..46a0160b71 100644 --- a/arbnode/batch_poster.go +++ b/arbnode/batch_poster.go @@ -1126,7 +1126,7 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) } var use4844 bool config := b.config() - if config.Post4844Blobs && b.dapWriter == nil && latestHeader.ExcessBlobGas != nil && *latestHeader.ExcessBlobGas != 0 && latestHeader.BlobGasUsed != nil && *latestHeader.BlobGasUsed != 0 { + if config.Post4844Blobs && b.dapWriter == nil && latestHeader.ExcessBlobGas != nil && latestHeader.BlobGasUsed != nil { arbOSVersion, err := b.arbOSVersionGetter.ArbOSVersionForMessageNumber(arbutil.MessageIndex(arbmath.SaturatingUSub(uint64(batchPosition.MessageCount), 1))) if err != nil { return false, err From c7f4f916bcbfcd937e6423c5d2a2df1e0aea230e Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Tue, 5 Nov 2024 21:48:26 +0530 Subject: [PATCH 487/489] Update geth --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index c4d8abf274..c4334737c9 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit c4d8abf2741bb628fc905b24a2d50c6f56539f4a +Subproject commit c4334737c95dc2b1bd398d2fe7feefee72da2f0f From 800f0226e1ddecac2585c8c2b7674d15704a0655 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Tue, 5 Nov 2024 22:37:13 +0530 Subject: [PATCH 488/489] disable blob posting in the test batch poster config --- arbnode/batch_poster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arbnode/batch_poster.go b/arbnode/batch_poster.go index 46a0160b71..185f43fce7 100644 --- a/arbnode/batch_poster.go +++ b/arbnode/batch_poster.go @@ -280,7 +280,7 @@ var TestBatchPosterConfig = BatchPosterConfig{ DASRetentionPeriod: daprovider.DefaultDASRetentionPeriod, GasRefunderAddress: "", ExtraBatchGas: 10_000, - Post4844Blobs: true, + Post4844Blobs: false, IgnoreBlobPrice: false, DataPoster: dataposter.TestDataPosterConfig, ParentChainWallet: DefaultBatchPosterL1WalletConfig, From f878323143e1ee61bfe51a7c606349d16fa6104a Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Tue, 5 Nov 2024 17:02:48 -0700 Subject: [PATCH 489/489] common_test: remove race condition --- system_tests/common_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 4fac71e3f9..9d491f522b 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -1224,7 +1224,6 @@ func createTestL1BlockChain(t *testing.T, l1info info) (info, *ethclient.Client, stack.RegisterAPIs(tracers.APIs(l1backend.APIBackend)) Require(t, stack.Start()) - Require(t, l1backend.Start()) rpcClient := stack.Attach()