diff --git a/pkg/tests/reward_test.go b/pkg/tests/reward_test.go index 321bffd05..280581604 100644 --- a/pkg/tests/reward_test.go +++ b/pkg/tests/reward_test.go @@ -124,7 +124,7 @@ func Test_Delegation_DelayedClaimingDestroyOutputWithoutRewards(t *testing.T) { delegationEndEpoch = futureBoundedEpochIndex + 1 } - tx2 := ts.DefaultWallet().DelayedClaimingTransition("TX2", ts.DefaultWallet().OutputData("TX1:0"), delegationEndEpoch) + tx2 := ts.DefaultWallet().DelayedClaimingTransition("TX2", ts.DefaultWallet().OutputData("TX1:0"), delegationEndEpoch) block2 := lo.PanicOnErr(ts.IssueBasicBlockWithOptions("block2", ts.DefaultWallet(), tx2, mock.WithStrongParents(block1.ID()))) latestParents := ts.CommitUntilSlot(block1_2Slot, block2.ID()) @@ -351,7 +351,7 @@ func Test_Account_StakeAmountCalculation(t *testing.T) { ) block5 := lo.PanicOnErr(ts.IssueBasicBlockWithOptions("block5", ts.DefaultWallet(), tx5, mock.WithStrongParents(latestParents...))) blockIssuanceResp = ts.DefaultWallet().GetNewBlockIssuanceResponse() - tx6 := ts.DefaultWallet().DelayedClaimingTransition("TX6", ts.DefaultWallet().OutputData("TX5:0"), ts.DefaultWallet().DelegationEndFromSlot(block5_6Slot, blockIssuanceResp.LatestCommitment.Slot)) + tx6 := ts.DefaultWallet().DelayedClaimingTransition("TX6", ts.DefaultWallet().OutputData("TX5:0"), ts.DefaultWallet().DelegationEndFromSlot(block5_6Slot, blockIssuanceResp.LatestCommitment.Slot)) block6 := lo.PanicOnErr(ts.IssueBasicBlockWithOptions("block6", ts.DefaultWallet(), tx6, mock.WithStrongParents(block5.ID()))) latestParents = ts.CommitUntilSlot(block5_6Slot, block6.ID()) @@ -387,7 +387,7 @@ func Test_Account_StakeAmountCalculation(t *testing.T) { // STEP 6: REMOVE A DELEGATION BY TRANSITIONING TO DELAYED CLAIMING. block9Slot := ts.CurrentSlot() blockIssuanceResp = ts.DefaultWallet().GetNewBlockIssuanceResponse() - tx9 := ts.DefaultWallet().DelayedClaimingTransition("TX9", ts.DefaultWallet().OutputData("TX4:0"), ts.DefaultWallet().DelegationEndFromSlot(block9Slot, blockIssuanceResp.LatestCommitment.Slot)) + tx9 := ts.DefaultWallet().DelayedClaimingTransition("TX9", ts.DefaultWallet().OutputData("TX4:0"), ts.DefaultWallet().DelegationEndFromSlot(block9Slot, blockIssuanceResp.LatestCommitment.Slot)) block9 := lo.PanicOnErr(ts.IssueBasicBlockWithOptions("block9", ts.DefaultWallet(), tx9, mock.WithStrongParents(latestParents...))) // Commit until the claiming epoch so we can remove the staking feature from the account in the next step. latestParents = ts.CommitUntilSlot(block9Slot, block9.ID()) @@ -418,7 +418,7 @@ func Test_Account_StakeAmountCalculation(t *testing.T) { block12Slot := ts.CurrentSlot() blockIssuanceResp = ts.DefaultWallet().GetNewBlockIssuanceResponse() - tx12 := ts.DefaultWallet().DelayedClaimingTransition("TX12", ts.DefaultWallet().OutputData("TX2:0"), ts.DefaultWallet().DelegationEndFromSlot(block12Slot, blockIssuanceResp.LatestCommitment.Slot)) + tx12 := ts.DefaultWallet().DelayedClaimingTransition("TX12", ts.DefaultWallet().OutputData("TX2:0"), ts.DefaultWallet().DelegationEndFromSlot(block12Slot, blockIssuanceResp.LatestCommitment.Slot)) block12 := lo.PanicOnErr(ts.IssueBasicBlockWithOptions("block12", ts.DefaultWallet(), tx12, mock.WithStrongParents(latestParents...))) ts.CommitUntilSlot(block12Slot, block12.ID()) diff --git a/tools/docker-network/.env b/tools/docker-network/.env index 92dd849c2..0de9f4407 100644 --- a/tools/docker-network/.env +++ b/tools/docker-network/.env @@ -6,8 +6,6 @@ COMMON_CONFIG=" --p2p.identityPrivateKeyFilePath=/app/data/p2p/identity.key --profiling.enabled=true --profiling.bindAddress=0.0.0.0:6061 ---db.path=/app/data/database ---protocol.snapshot.path=/app/data/snapshot.bin --restAPI.publicRoutes=/health,/api/routes,/api/core/v3/info,/api/core/v3/network*,/api/core/v3/blocks*,/api/core/v3/transactions*,/api/core/v3/commitments*,/api/core/v3/outputs*,/api/core/v3/accounts*,/api/core/v3/validators*,/api/core/v3/rewards*,/api/core/v3/committee*,/api/debug/v2/*,/api/indexer/v2/*,/api/mqtt/v2,/api/blockissuer/v1/*,/api/management/v1/* --debugAPI.enabled=false --p2p.autopeering.allowLocalIPs=true diff --git a/tools/docker-network/docker-compose.yml b/tools/docker-network/docker-compose.yml index cf57fb0f6..2864be3a3 100644 --- a/tools/docker-network/docker-compose.yml +++ b/tools/docker-network/docker-compose.yml @@ -23,10 +23,12 @@ services: - "6051:6061/tcp" # pprof - "9059:9029/tcp" # INX volumes: - - ./docker-network.snapshot:/app/data/snapshot.bin + - ./docker-network-snapshots/:/app/data/snapshots/ - ./config.json:/app/config.json:ro command: > ${COMMON_CONFIG} + --db.path==${DB_PATH_V1:-/app/data/database} + --protocol.snapshot.path=${SNAPSHOT_PATH_V1:-/app/data/snapshots/snapshot.bin} --p2p.identityPrivateKey=08735375679f3d8031353e94282ed1d65119e5c288fe56d6639d9184a3f978fee8febfedff11cc376daea0f59c395ae2e9a870a25ac4e36093000fbf4d0e8f18 --node.alias=node-1-validator --inx.enabled=true @@ -53,11 +55,13 @@ services: - "6061:6061/tcp" # pprof - "9069:9029/tcp" # INX volumes: - - ./docker-network.snapshot:/app/data/snapshot.bin + - ./docker-network-snapshots/:/app/data/snapshots/ - ./config.json:/app/config.json:ro command: > ${COMMON_CONFIG} ${AUTOPEERING_CONFIG} + --db.path==${DB_PATH_V2:-/app/data/database} + --protocol.snapshot.path=${SNAPSHOT_PATH_V2:-/app/data/snapshots/snapshot.bin} --p2p.identityPrivateKey=ba771419c52132a0dfb2521ed18667813f398da159010a55a0a482af939affb92d3338789ad4a07a7631b91791deb11f82ed5dc612822f24275e9f7a313b691f --node.alias=node-2-validator --inx.enabled=true @@ -83,11 +87,13 @@ services: - "6071:6061/tcp" # pprof - "9079:9029/tcp" # INX volumes: - - ./docker-network.snapshot:/app/data/snapshot.bin + - ./docker-network-snapshots/:/app/data/snapshots/ - ./config.json:/app/config.json:ro command: > ${COMMON_CONFIG} ${AUTOPEERING_CONFIG} + --db.path==${DB_PATH_V3:-/app/data/database} + --protocol.snapshot.path=${SNAPSHOT_PATH_V3:-/app/data/snapshots/snapshot.bin} --p2p.identityPrivateKey=a6261ac049755675ff1437654ca9f83b305055f01ff08c4f039209ef5a4a7d96d06fb61df77a8815209a8f4d204226dee593e50d0ec897ec440a2c1fbde77656 --node.alias=node-3-validator --inx.enabled=true @@ -113,11 +119,13 @@ services: - "6041:6061/tcp" # pprof - "9049:9029/tcp" # INX volumes: - - ./docker-network.snapshot:/app/data/snapshot.bin + - ./docker-network-snapshots/:/app/data/snapshots/ - ./config.json:/app/config.json:ro command: > ${COMMON_CONFIG} ${AUTOPEERING_CONFIG} + --db.path==${DB_PATH_V4:-/app/data/database} + --protocol.snapshot.path=${SNAPSHOT_PATH_V4:-/app/data/snapshots/snapshot.bin} --p2p.identityPrivateKey=f205f6c4525069f71f9c7e987d72421a16c7900056b494a2b85fdf7942cf906aefbdc580f5d1ce4ae3f86ccfe109c6cd76df9b0e710a437b2aa964358c7b9449 --node.alias=node-4-validator --inx.enabled=true @@ -141,11 +149,13 @@ services: - "6081:6061/tcp" # pprof - "9089:9029/tcp" # INX volumes: - - ./docker-network.snapshot:/app/data/snapshot.bin + - ./docker-network-snapshots/:/app/data/snapshots/ - ./config.json:/app/config.json:ro command: > ${COMMON_CONFIG} ${AUTOPEERING_CONFIG} + --db.path==${DB_PATH_node5:-/app/data/database} + --protocol.snapshot.path=${SNAPSHOT_PATH_node5:-/app/data/snapshots/snapshot.bin} --p2p.identityPrivateKey=03feb3bcd25e57f75697bb329e6e0100680431e4c45c85bc013da2aea9e9d0345e08a0c37407dc62369deebc64cb0fb3ea26127d19d141ee7fb8eaa6b92019d7 --node.alias=node-5 --inx.enabled=true @@ -171,11 +181,13 @@ services: - "6091:6061/tcp" # pprof - "9099:9029/tcp" # INX volumes: - - ./docker-network.snapshot:/app/data/snapshot.bin + - ./docker-network-snapshots/:/app/data/snapshots/ - ./config.json:/app/config.json:ro command: > ${COMMON_CONFIG} ${AUTOPEERING_CONFIG} + --db.path==${DB_PATH_node6:-/app/data/database} + --protocol.snapshot.path=${SNAPSHOT_PATH_node6:-/app/data/snapshots/snapshot.bin} --p2p.identityPrivateKey=7d1491df3ef334dee988d6cdfc4b430b996d520bd63375a01d6754f8cee979b855b200fbea8c936ea1937a27e6ad72a7c9a21c1b17c2bd3c11f1f6994d813446 --node.alias=node-6 --inx.enabled=true diff --git a/tools/docker-network/run.sh b/tools/docker-network/run.sh index 2ece1590e..396c6b9ff 100755 --- a/tools/docker-network/run.sh +++ b/tools/docker-network/run.sh @@ -24,6 +24,10 @@ if [[ "$WITH_GO_WORK" -eq 1 ]]; then export DOCKERFILE_PATH="./iota-core/Dockerfile.dev" fi +mkdir -p docker-network-snapshots/ +# Allow 'others' to write, so a snapshot can be created via the management API from within docker containers. +chmod o+w docker-network-snapshots/ + # Allow docker compose to build and cache an image echo $DOCKER_BUILD_CONTEXT $DOCKERFILE_PATH docker compose build --build-arg WITH_GO_WORK=${WITH_GO_WORK:-0} --build-arg DOCKER_BUILD_CONTEXT=${DOCKER_BUILD_CONTEXT} --build-arg DOCKERFILE_PATH=${DOCKERFILE_PATH} @@ -37,7 +41,7 @@ fi docker compose pull inx-indexer inx-mqtt inx-blockissuer inx-faucet inx-validator-1 inx-dashboard-1 # Remove old snapshot file -rm *.snapshot +rm docker-network-snapshots/*.snapshot # create snapshot file echo "Create snapshot" @@ -54,12 +58,12 @@ docker run --rm \ golang:1.22-bookworm go run -tags=rocksdb . --config docker --seed 7R1itJx5hVuo9w9hjg5cwKFmek4HMSoBDgJZN8hKGxih # Move and set permissions for the .snapshot file -if ! mv -f ../genesis-snapshot/*.snapshot .; then +if ! mv -f ../genesis-snapshot/docker-network.snapshot docker-network-snapshots/snapshot.bin; then echo "Failed to create or move snapshot file" exit 1 fi -chmod o+r *.snapshot +chmod o+r docker-network-snapshots/snapshot.bin echo "Run iota-core network" # IOTA_CORE_PEER_REPLICAS is used in docker-compose.yml to determine how many replicas to create diff --git a/tools/docker-network/tests/api_management_test.go b/tools/docker-network/tests/api_management_test.go index 64c4fbb41..a89e2d0bc 100644 --- a/tools/docker-network/tests/api_management_test.go +++ b/tools/docker-network/tests/api_management_test.go @@ -210,16 +210,6 @@ func Test_ManagementAPI_Pruning(t *testing.T) { managementClient, err := nodeClientV1.Management(getContextWithTimeout(5 * time.Second)) require.NoError(t, err) - awaitNextEpoch := func() { - info, err := nodeClientV1.Info(getContextWithTimeout(5 * time.Second)) - require.NoError(t, err) - - currentEpoch := nodeClientV1.CommittedAPI().TimeProvider().EpochFromSlot(info.Status.LatestFinalizedSlot) - - // await the start slot of the next epoch - d.AwaitFinalization(nodeClientV1.CommittedAPI().TimeProvider().EpochStart(currentEpoch + 1)) - } - type test struct { name string testFunc func(t *testing.T) @@ -230,9 +220,9 @@ func Test_ManagementAPI_Pruning(t *testing.T) { name: "Test_PruneDatabase_ByEpoch", testFunc: func(t *testing.T) { // we need to wait until epoch 3 to be able to prune epoch 1 - awaitNextEpoch() - awaitNextEpoch() - awaitNextEpoch() + d.AwaitNextEpoch() + d.AwaitNextEpoch() + d.AwaitNextEpoch() // prune database by epoch pruneDatabaseResponse, err := managementClient.PruneDatabaseByEpoch(getContextWithTimeout(5*time.Second), 1) @@ -244,7 +234,7 @@ func Test_ManagementAPI_Pruning(t *testing.T) { name: "Test_PruneDatabase_ByDepth", testFunc: func(t *testing.T) { // wait for the next epoch to start - awaitNextEpoch() + d.AwaitNextEpoch() // prune database by depth pruneDatabaseResponse, err := managementClient.PruneDatabaseByDepth(getContextWithTimeout(5*time.Second), 1) @@ -256,7 +246,7 @@ func Test_ManagementAPI_Pruning(t *testing.T) { name: "Test_PruneDatabase_BySize", testFunc: func(t *testing.T) { // wait for the next epoch to start - awaitNextEpoch() + d.AwaitNextEpoch() // prune database by size pruneDatabaseResponse, err := managementClient.PruneDatabaseBySize(getContextWithTimeout(5*time.Second), "5G") @@ -300,7 +290,7 @@ func Test_ManagementAPI_Snapshots(t *testing.T) { managementClient, err := nodeClientV1.Management(getContextWithTimeout(5 * time.Second)) require.NoError(t, err) - awaitNextEpoch := func() { + awaitNextCommittedEpoch := func() { info, err := nodeClientV1.Info(getContextWithTimeout(5 * time.Second)) require.NoError(t, err) @@ -320,7 +310,7 @@ func Test_ManagementAPI_Snapshots(t *testing.T) { name: "Test_CreateSnapshot", testFunc: func(t *testing.T) { // wait for the next epoch to start - awaitNextEpoch() + awaitNextCommittedEpoch() // create snapshot snapshotResponse, err := managementClient.CreateSnapshot(getContextWithTimeout(5 * time.Second)) diff --git a/tools/docker-network/tests/dockerframework.go b/tools/docker-network/tests/dockerframework.go index d4c754c07..7b9543f94 100644 --- a/tools/docker-network/tests/dockerframework.go +++ b/tools/docker-network/tests/dockerframework.go @@ -6,6 +6,7 @@ import ( "context" "fmt" "log" + "math/rand" "net/http" "os" "os/exec" @@ -32,7 +33,7 @@ import ( var ( // need to build snapshotfile in tools/docker-network. - snapshotFilePath = "../docker-network.snapshot" + snapshotFilePath = "../docker-network-snapshots/snapshot.bin" keyManager = func() *wallet.KeyManager { genesisSeed, err := base58.Decode("7R1itJx5hVuo9w9hjg5cwKFmek4HMSoBDgJZN8hKGxih") if err != nil { @@ -55,6 +56,8 @@ type Node struct { ContainerConfigs string PrivateKey string IssueCandidacyPayload bool + DatabasePath string + SnapshotPath string } func (n *Node) AccountAddress(t *testing.T) *iotago.AccountAddress { @@ -145,6 +148,14 @@ func (d *DockerTestFramework) DockerComposeUp(detach ...bool) error { cmd.Env = os.Environ() for _, node := range d.Nodes() { cmd.Env = append(cmd.Env, fmt.Sprintf("ISSUE_CANDIDACY_PAYLOAD_%s=%t", node.Name, node.IssueCandidacyPayload)) + if node.DatabasePath != "" { + fmt.Println("Setting Database Path for", node.Name, " to", node.DatabasePath) + cmd.Env = append(cmd.Env, fmt.Sprintf("DB_PATH_%s=%s", node.Name, node.DatabasePath)) + } + if node.SnapshotPath != "" { + fmt.Println("Setting snapshot path for", node.Name, " to", node.SnapshotPath) + cmd.Env = append(cmd.Env, fmt.Sprintf("SNAPSHOT_PATH_%s=%s", node.Name, node.SnapshotPath)) + } } var out strings.Builder @@ -347,6 +358,30 @@ func (d *DockerTestFramework) Node(name string) *Node { return node } +func (d *DockerTestFramework) ModifyNode(name string, fun func(*Node)) { + d.nodesLock.Lock() + defer d.nodesLock.Unlock() + + node, exist := d.nodes[name] + require.True(d.Testing, exist) + + fun(node) +} + +// Restarts a node with another database path, conceptually deleting the database and +// restarts it with the given snapshot path. +func (d *DockerTestFramework) ResetNode(alias string, newSnapshotPath string) { + fmt.Println("Reset node", alias) + + d.ModifyNode(alias, func(n *Node) { + n.DatabasePath = fmt.Sprintf("/app/database/database%d", rand.Int()) + n.SnapshotPath = newSnapshotPath + }) + d.DockerComposeUp(true) + d.DumpContainerLog(d.Node(alias).ContainerName, "reset1") + d.WaitUntilSync() +} + func (d *DockerTestFramework) Clients(names ...string) map[string]mock.Client { d.nodesLock.RLock() defer d.nodesLock.RUnlock() @@ -785,15 +820,25 @@ func (d *DockerTestFramework) DumpContainerLogsToFiles() { continue } - filePath := fmt.Sprintf("%s/%s.log", d.logDirectoryPath, name) - // dump logs to file if the file does not exist, which means the container is just started. - // logs should exist for the already running containers. - _, err := os.Stat(filePath) - if os.IsNotExist(err) { - logCmd := fmt.Sprintf("docker logs -f %s > %s 2>&1 &", name, filePath) - err := exec.Command("bash", "-c", logCmd).Run() - require.NoError(d.Testing, err) - } + d.DumpContainerLog(name) + } +} + +func (d *DockerTestFramework) DumpContainerLog(name string, optLogNameExtension ...string) { + var filePath string + if len(optLogNameExtension) > 0 { + filePath = fmt.Sprintf("%s/%s-%s.log", d.logDirectoryPath, name, optLogNameExtension[0]) + } else { + filePath = fmt.Sprintf("%s/%s.log", d.logDirectoryPath, name) + } + + // dump logs to file if the file does not exist, which means the container is just started. + // logs should exist for the already running containers. + _, err := os.Stat(filePath) + if os.IsNotExist(err) { + logCmd := fmt.Sprintf("docker logs -f %s > %s 2>&1 &", name, filePath) + err := exec.Command("bash", "-c", logCmd).Run() + require.NoError(d.Testing, err) } } diff --git a/tools/docker-network/tests/run_tests.sh b/tools/docker-network/tests/run_tests.sh index edc5b5401..5abf2b638 100755 --- a/tools/docker-network/tests/run_tests.sh +++ b/tools/docker-network/tests/run_tests.sh @@ -16,6 +16,10 @@ export COMPOSE_DOCKER_CLI_BUILD=1 export DOCKER_BUILD_CONTEXT="../../" export DOCKERFILE_PATH="./Dockerfile.dev" +mkdir -p docker-network-snapshots/ +# Allow 'others' to write, so a snapshot can be created via the management API from within docker containers. +chmod o+w docker-network-snapshots/ + # Allow docker compose to build and cache an image docker compose build --build-arg DOCKER_BUILD_CONTEXT=${DOCKER_BUILD_CONTEXT} --build-arg DOCKERFILE_PATH=${DOCKERFILE_PATH} diff --git a/tools/docker-network/tests/sync_snapshot_test.go b/tools/docker-network/tests/sync_snapshot_test.go new file mode 100644 index 000000000..88f92cfcd --- /dev/null +++ b/tools/docker-network/tests/sync_snapshot_test.go @@ -0,0 +1,51 @@ +//go:build dockertests + +package tests + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + iotago "github.com/iotaledger/iota.go/v4" +) + +func Test_SyncFromSnapshot(t *testing.T) { + d := NewDockerTestFramework(t, + WithProtocolParametersOptions( + iotago.WithTimeProviderOptions(0, time.Now().Unix(), 4, 4), + iotago.WithLivenessOptions(3, 4, 2, 4, 5), + iotago.WithCongestionControlOptions(1, 1, 1, 400_000, 250_000, 50_000_000, 1000, 100), + iotago.WithRewardsOptions(8, 10, 2, 384), + iotago.WithTargetCommitteeSize(4), + )) + defer d.Stop() + + d.AddValidatorNode("V1", "docker-network-inx-validator-1-1", "http://localhost:8050", "rms1pzg8cqhfxqhq7pt37y8cs4v5u4kcc48lquy2k73ehsdhf5ukhya3y5rx2w6") + d.AddValidatorNode("V2", "docker-network-inx-validator-2-1", "http://localhost:8060", "rms1pqm4xk8e9ny5w5rxjkvtp249tfhlwvcshyr3pc0665jvp7g3hc875k538hl") + d.AddValidatorNode("V3", "docker-network-inx-validator-3-1", "http://localhost:8070", "rms1pp4wuuz0y42caz48vv876qfpmffswsvg40zz8v79sy8cp0jfxm4kunflcgt") + d.AddValidatorNode("V4", "docker-network-inx-validator-4-1", "http://localhost:8040", "rms1pr8cxs3dzu9xh4cduff4dd4cxdthpjkpwmz2244f75m0urslrsvtsshrrjw") + d.AddNode("node5", "docker-network-node-5-1", "http://localhost:8090") + + err := d.Run() + require.NoError(t, err) + + d.WaitUntilNetworkReady() + + d.AwaitNextEpoch() + d.AwaitNextEpoch() + + nodeClientV1 := d.Client("V1") + managementClient, err := nodeClientV1.Management(getContextWithTimeout(5 * time.Second)) + require.NoError(t, err) + + response, err := managementClient.CreateSnapshot(getContextWithTimeout(5 * time.Second)) + require.NoError(t, err) + + // Deletes the database of node5 and restarts it with the just created snapshot. + d.ResetNode("node5", response.FilePath) + + d.AwaitNextEpoch() + d.AwaitNextEpoch() +} diff --git a/tools/docker-network/tests/utils.go b/tools/docker-network/tests/utils.go index 2e81ac8c8..ab4bd38eb 100644 --- a/tools/docker-network/tests/utils.go +++ b/tools/docker-network/tests/utils.go @@ -273,6 +273,19 @@ func (d *DockerTestFramework) AwaitFinalization(targetSlot iotago.SlotIndex) { }) } +func (d *DockerTestFramework) AwaitNextEpoch() { + //nolint:lostcancel + ctx, _ := context.WithTimeout(context.Background(), 5*time.Second) + + info, err := d.defaultWallet.Client.Info(ctx) + require.NoError(d.Testing, err) + + currentEpoch := d.defaultWallet.Client.CommittedAPI().TimeProvider().EpochFromSlot(info.Status.LatestFinalizedSlot) + + // await the start slot of the next epoch + d.AwaitFinalization(d.defaultWallet.Client.CommittedAPI().TimeProvider().EpochStart(currentEpoch + 1)) +} + func (d *DockerTestFramework) AwaitAddressUnspentOutputAccepted(ctx context.Context, wallet *mock.Wallet, addr iotago.Address) (outputID iotago.OutputID, output iotago.Output, err error) { indexerClt, err := wallet.Client.Indexer(ctx) require.NoError(d.Testing, err)