diff --git a/Dockerfile.combined b/Dockerfile.combined new file mode 100644 index 0000000000..5ab867c89c --- /dev/null +++ b/Dockerfile.combined @@ -0,0 +1,53 @@ +# syntax=docker/dockerfile:1 + +# Dockerfile.combined defines a docker image using different provider/consumer versions +# originated from other docker images. +# This image is used to test different versions of provider and consumer together. +# +# Use docker's build argument --build-arg to specify the consumer/provider image to be used +# e.g. docker build --build-arg CONSUMER_IMAGE=v3.1.0 --build-arg PROVIDER_IMAGE=v3.1.0 + +ARG PROVIDER_IMAGE +ARG CONSUMER_IMAGE + +# The image from where the consumer implementation will be used +# Defaults to +FROM --platform=linux/amd64 ${PROVIDER_IMAGE} AS provider + + +# The image from where the consumer implementation will be used +# Defaults to +FROM --platform=linux/amd64 ${CONSUMER_IMAGE} AS consumer + +# Get Hermes build +FROM --platform=linux/amd64 otacrew/hermes-ics:evidence-cmd AS hermes-builder + + +# Get GoRelayer +FROM ghcr.io/informalsystems/relayer-no-gas-sim:v2.3.0-rc4-no-gas-sim AS gorelayer-builder + + +FROM --platform=linux/amd64 fedora:39 +RUN dnf update -y +RUN dnf install -y which iproute iputils procps-ng vim-minimal tmux net-tools htop jq +USER root + +COPY --from=hermes-builder /usr/bin/hermes /usr/local/bin/ +COPY --from=gorelayer-builder /bin/rly /usr/local/bin/ + +# Copy consumer from specified image +COPY --from=consumer /usr/local/bin/interchain-security-cd /usr/local/bin/interchain-security-cd +COPY --from=consumer /usr/local/bin/interchain-security-cdd /usr/local/bin/interchain-security-cdd +COPY --from=consumer /usr/local/bin/interchain-security-sd /usr/local/bin/interchain-security-sd +COPY --from=consumer /testnet-scripts /consumer/testnet-scripts + + +# Copy provider from specified image +COPY --from=provider /usr/local/bin/interchain-security-pd /usr/local/bin/interchain-security-pd +COPY --from=provider /testnet-scripts /provider/testnet-scripts + +#Copy cometmock from provider image +COPY --from=provider /usr/local/bin/cometmock /usr/local/bin + +# Copy in the hermes config +ADD ./tests/e2e/testnet-scripts/hermes-config.toml /root/.hermes/config.toml diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 963494a2ee..cc74e57834 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -17,7 +17,7 @@ End-to-end tests can still be useful, and we need them, but when possible, we should prefer more local tests. At a high-level, every test case consists of the following steps. -* The test starts a docker container, see [the startup script](testnet-scripts/start-docker.sh) +* The test starts a docker container, see [setupEnvironment](test_runner.go) * We run a defined sequence of actions and expected states, see as an example the steps for [testing the democracy module](steps_democracy.go) * Actions are any event that might meaningfully modify the system state, such as submitting transactions to a node, making nodes double-sign, starting a relayer, starting a new chain, etc. * Expected states define the state we expect after the action was taken. @@ -35,7 +35,7 @@ At a high-level, every test case consists of the following steps. If you just want to run the end-to-end tests, see the commands in the Makefile in the repo root. -If you want to run the tests with a bit more control, see the help by running +If you want to run the tests with a bit more control, see the help by running ```go run ./tests/e2e/... --help``` in the repo root to see how to do that. @@ -102,7 +102,7 @@ the actions necessary to start a provider and multiple consumer chains are already "packaged together" and available as `stepsStartChains` in [steps_start_chains.go](steps_start_chains.go). -**Note:** The parts of the state that are *not* defined are just *not checked*. +**Note:** The parts of the state that are *not* defined are just *not checked*. For example, if the balance of a validator is not listed in a state, it means we do not care about the balance of that validator in this particular state. @@ -131,7 +131,7 @@ You can see the basic template for how to do this by looking at the actions in The basic principle is to use `exec.Command` to execute a command inside the docker container. The pattern for this looks something like this: ``` -cmd := exec.Command("docker", "exec", +cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[action.Chain].BinaryName, "the command to execute, for example tx", @@ -145,7 +145,7 @@ if err != nil { // potentially check something in the output, or log something, ... ``` -Don't forget to wire your action into [main.go](main.go):runStep, where +Don't forget to wire your action into [test_driver.go](test_driver.go):runAction, where the action structs and functions to run for each of them are wired together. **Note:** Actions don't need to check that the state was modified correctly, @@ -163,7 +163,7 @@ work and sometimes fail due to gas, as can happen with `--gas=auto` and no `--ga Essentially, sometimes the gas estimation will underestimate gas, but not always - it seems to be non-deterministic, and probably depends on subtle things like block times, or heights, which are not finely controlled in the end-to-end tests -and do not perfectly match each time. +and do not perfectly match each time. To be sure we don't introduce nondeterminism like this, we need to use a sufficient adjustment to make sure there is enough gas for transactions to pass. diff --git a/tests/e2e/actions.go b/tests/e2e/actions.go index 84c3020b91..81bc3c8b76 100644 --- a/tests/e2e/actions.go +++ b/tests/e2e/actions.go @@ -6,6 +6,7 @@ import ( "fmt" "log" "math" + "os" "os/exec" "strconv" "strings" @@ -31,6 +32,7 @@ const done = "done!!!!!!!!" func (tr TestConfig) sendTokens( action SendTokensAction, + target ExecutionTarget, verbose bool, ) { fromValCfg := tr.validatorConfigs[action.From] @@ -54,8 +56,7 @@ func (tr TestConfig) sendTokens( } binaryName := tr.chainConfigs[action.Chain].BinaryName - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, binaryName, + cmd := target.ExecCommand(binaryName, "tx", "bank", "send", fromAddress, @@ -96,6 +97,7 @@ type StartChainValidator struct { func (tr *TestConfig) startChain( action StartChainAction, + target ExecutionTarget, verbose bool, ) { chainConfig := tr.chainConfigs[action.Chain] @@ -151,9 +153,9 @@ func (tr *TestConfig) startChain( cometmockArg = "false" } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "/bin/bash", - "/testnet-scripts/start-chain.sh", chainConfig.BinaryName, string(vals), + startChainScript := target.GetTestScriptPath(action.IsConsumer, "start-chain.sh") + cmd := target.ExecCommand("/bin/bash", + startChainScript, chainConfig.BinaryName, string(vals), string(chainConfig.ChainId), chainConfig.IpPrefix, genesisChanges, fmt.Sprint(action.IsConsumer), // override config/config.toml for each node on chain @@ -193,7 +195,7 @@ func (tr *TestConfig) startChain( Chain: action.Chain, Validator: action.Validators[0].Id, IsConsumer: action.IsConsumer, - }, verbose) + }, target, verbose) // store the fact that we started the chain tr.runningChains[action.Chain] = true @@ -214,11 +216,12 @@ type SubmitTextProposalAction struct { func (tr TestConfig) submitTextProposal( action SubmitTextProposalAction, + target ExecutionTarget, verbose bool, ) { // TEXT PROPOSAL - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - bz, err := exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[action.Chain].BinaryName, + bz, err := target.ExecCommand( + tr.chainConfigs[action.Chain].BinaryName, "tx", "gov", "submit-legacy-proposal", `--title`, action.Title, `--description`, action.Description, @@ -251,6 +254,7 @@ type SubmitConsumerAdditionProposalAction struct { func (tr TestConfig) submitConsumerAdditionProposal( action SubmitConsumerAdditionProposalAction, + target ExecutionTarget, verbose bool, ) { spawnTime := tr.containerConfig.Now.Add(time.Duration(action.SpawnTime) * time.Millisecond) @@ -283,17 +287,18 @@ func (tr TestConfig) submitConsumerAdditionProposal( log.Fatal("prop json contains single quote") } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - bz, err = exec.Command("docker", "exec", tr.containerConfig.InstanceName, - "/bin/bash", "-c", fmt.Sprintf(`echo '%s' > %s`, jsonStr, "/temp-proposal.json")).CombinedOutput() + //#nosec G204 -- bypass unsafe quoting warning (no production code) + bz, err = target.ExecCommand( + "/bin/bash", "-c", fmt.Sprintf(`echo '%s' > %s`, jsonStr, "/temp-proposal.json"), + ).CombinedOutput() if err != nil { log.Fatal(err, "\n", string(bz)) } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. // CONSUMER ADDITION PROPOSAL - bz, err = exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[action.Chain].BinaryName, + bz, err = target.ExecCommand( + tr.chainConfigs[action.Chain].BinaryName, "tx", "gov", "submit-legacy-proposal", "consumer-addition", "/temp-proposal.json", `--from`, `validator`+fmt.Sprint(action.From), `--chain-id`, string(tr.chainConfigs[action.Chain].ChainId), @@ -322,6 +327,7 @@ type SubmitConsumerRemovalProposalAction struct { func (tr TestConfig) submitConsumerRemovalProposal( action SubmitConsumerRemovalProposalAction, + target ExecutionTarget, verbose bool, ) { stopTime := tr.containerConfig.Now.Add(action.StopTimeOffset) @@ -343,17 +349,15 @@ func (tr TestConfig) submitConsumerRemovalProposal( log.Fatal("prop json contains single quote") } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - bz, err = exec.Command("docker", "exec", tr.containerConfig.InstanceName, + bz, err = target.ExecCommand( "/bin/bash", "-c", fmt.Sprintf(`echo '%s' > %s`, jsonStr, "/temp-proposal.json")).CombinedOutput() if err != nil { log.Fatal(err, "\n", string(bz)) } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - bz, err = exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[action.Chain].BinaryName, - + bz, err = target.ExecCommand( + tr.chainConfigs[action.Chain].BinaryName, "tx", "gov", "submit-legacy-proposal", "consumer-removal", "/temp-proposal.json", `--from`, `validator`+fmt.Sprint(action.From), @@ -398,6 +402,7 @@ type paramChangeJSON struct { func (tr TestConfig) submitParamChangeProposal( action SubmitParamChangeLegacyProposalAction, + target ExecutionTarget, verbose bool, ) { prop := paramChangeProposalJSON{ @@ -418,17 +423,17 @@ func (tr TestConfig) submitParamChangeProposal( log.Fatal("prop json contains single quote") } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - bz, err = exec.Command("docker", "exec", tr.containerConfig.InstanceName, - "/bin/bash", "-c", fmt.Sprintf(`echo '%s' > %s`, jsonStr, "/params-proposal.json")).CombinedOutput() + //#nosec G204 -- bypass unsafe quoting warning (no production code) + bz, err = target.ExecCommand( + "/bin/bash", "-c", fmt.Sprintf(`echo '%s' > %s`, jsonStr, "/params-proposal.json"), + ).CombinedOutput() if err != nil { log.Fatal(err, "\n", string(bz)) } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - // PARAM CHANGE PROPOSAL - cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[action.Chain].BinaryName, + cmd := target.ExecCommand( + tr.chainConfigs[action.Chain].BinaryName, "tx", "gov", "submit-legacy-proposal", "param-change", "/params-proposal.json", @@ -459,6 +464,7 @@ type VoteGovProposalAction struct { func (tr *TestConfig) voteGovProposal( action VoteGovProposalAction, + target ExecutionTarget, verbose bool, ) { var wg sync.WaitGroup @@ -467,8 +473,8 @@ func (tr *TestConfig) voteGovProposal( vote := action.Vote[i] go func(val ValidatorID, vote string) { defer wg.Done() - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - bz, err := exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[action.Chain].BinaryName, + bz, err := target.ExecCommand( + tr.chainConfigs[action.Chain].BinaryName, "tx", "gov", "vote", fmt.Sprint(action.PropNumber), vote, @@ -502,39 +508,88 @@ type StartConsumerChainAction struct { func (tr *TestConfig) startConsumerChain( action StartConsumerChainAction, + target ExecutionTarget, verbose bool, ) { - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[action.ProviderChain].BinaryName, + fmt.Println("Starting consumer chain ", action.ConsumerChain) + consumerGenesis := ".app_state.ccvconsumer = " + tr.getConsumerGenesis(action.ProviderChain, action.ConsumerChain, target) + consumerGenesisChanges := tr.chainConfigs[action.ConsumerChain].GenesisChanges + if consumerGenesisChanges != "" { + consumerGenesis = consumerGenesis + " | " + consumerGenesisChanges + " | " + action.GenesisChanges + } + + tr.startChain(StartChainAction{ + Chain: action.ConsumerChain, + Validators: action.Validators, + GenesisChanges: consumerGenesis, + IsConsumer: true, + }, target, verbose) +} + +// Get consumer genesis from provider +func (tr *TestConfig) getConsumerGenesis(providerChain, consumerChain ChainID, target ExecutionTarget) string { + fmt.Println("Exporting consumer genesis from provider") + providerBinaryName := tr.chainConfigs[providerChain].BinaryName + + cmd := target.ExecCommand( + providerBinaryName, "query", "provider", "consumer-genesis", - string(tr.chainConfigs[action.ConsumerChain].ChainId), + string(tr.chainConfigs[consumerChain].ChainId), - `--node`, tr.getQueryNode(action.ProviderChain), + `--node`, tr.getQueryNode(providerChain), `-o`, `json`, ) - if verbose { - log.Println("startConsumerChain cmd: ", cmd.String()) - } - bz, err := cmd.CombinedOutput() if err != nil { log.Fatal(err, "\n", string(bz)) } - consumerGenesis := ".app_state.ccvconsumer = " + string(bz) - consumerGenesisChanges := tr.chainConfigs[action.ConsumerChain].GenesisChanges - if consumerGenesisChanges != "" { - consumerGenesis = consumerGenesis + " | " + consumerGenesisChanges + " | " + action.GenesisChanges + // only needed when consumer is running v3.3.x and later + if tr.transformGenesis { + return string(tr.transformConsumerGenesis(consumerChain, bz, target)) } + return string(bz) +} - tr.startChain(StartChainAction{ - Chain: action.ConsumerChain, - Validators: action.Validators, - GenesisChanges: consumerGenesis, - IsConsumer: true, - }, verbose) +// Transform consumer genesis content from older version +func (tr *TestConfig) transformConsumerGenesis(consumerChain ChainID, genesis []byte, target ExecutionTarget) []byte { + fmt.Println("Transforming consumer genesis") + fmt.Printf("Original ccv genesis: %s\n", string(genesis)) + + fileName := "consumer_genesis.json" + file, err := os.CreateTemp("", fileName) + if err != nil { + panic(fmt.Sprintf("failed writing ccv consumer file : %v", err)) + } + defer file.Close() + err = os.WriteFile(file.Name(), genesis, 0600) + if err != nil { + log.Fatalf("Failed writing consumer genesis to file: %v", err) + } + + containerInstance := tr.containerConfig.InstanceName + targetFile := fmt.Sprintf("/tmp/%s", fileName) + sourceFile := file.Name() + //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. + cmd := exec.Command("docker", "cp", sourceFile, + fmt.Sprintf("%s:%s", containerInstance, targetFile)) + genesis, err = cmd.CombinedOutput() + if err != nil { + log.Fatal(err, "\n", string(genesis)) + } + + consumerBinaryName := tr.chainConfigs[consumerChain].BinaryName + cmd = target.ExecCommand( + consumerBinaryName, + "genesis", "transform", targetFile) + result, err := cmd.CombinedOutput() + if err != nil { + log.Fatal(err, "CCV consumer genesis transformation failed: %s", string(result)) + } + fmt.Printf("Transformed genesis is: %s\n", string(result)) + return result } type ChangeoverChainAction struct { @@ -546,12 +601,13 @@ type ChangeoverChainAction struct { func (tr TestConfig) changeoverChain( action ChangeoverChainAction, + target ExecutionTarget, verbose bool, ) { // sleep until the consumer chain genesis is ready on consumer time.Sleep(5 * time.Second) - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[action.ProviderChain].BinaryName, + cmd := target.ExecCommand( + tr.chainConfigs[action.ProviderChain].BinaryName, "query", "provider", "consumer-genesis", string(tr.chainConfigs[action.SovereignChain].ChainId), @@ -578,11 +634,12 @@ func (tr TestConfig) changeoverChain( tr.startChangeover(ChangeoverChainAction{ Validators: action.Validators, GenesisChanges: consumerGenesis, - }, verbose) + }, target, verbose) } func (tr TestConfig) startChangeover( action ChangeoverChainAction, + target ExecutionTarget, verbose bool, ) { chainConfig := tr.chainConfigs[ChainID("sover")] @@ -631,9 +688,11 @@ func (tr TestConfig) startChangeover( genesisChanges = chainConfig.GenesisChanges } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "/bin/bash", - "/testnet-scripts/start-changeover.sh", chainConfig.UpgradeBinary, string(vals), + isConsumer := true + changeoverScript := target.GetTestScriptPath(isConsumer, "start-changeover.sh") + cmd := target.ExecCommand( + "/bin/bash", + changeoverScript, chainConfig.UpgradeBinary, string(vals), "sover", chainConfig.IpPrefix, genesisChanges, tr.tendermintConfigOverride, ) @@ -720,17 +779,19 @@ const gorelayerChainConfigTemplate = ` func (tr TestConfig) addChainToRelayer( action AddChainToRelayerAction, + target ExecutionTarget, verbose bool, ) { if !tr.useGorelayer { - tr.addChainToHermes(action, verbose) + tr.addChainToHermes(action, target, verbose) } else { - tr.addChainToGorelayer(action, verbose) + tr.addChainToGorelayer(action, target, verbose) } } func (tr TestConfig) addChainToGorelayer( action AddChainToRelayerAction, + target ExecutionTarget, verbose bool, ) { queryNodeIP := tr.getQueryNodeIP(action.Chain) @@ -743,8 +804,8 @@ func (tr TestConfig) addChainToGorelayer( tr.chainConfigs[action.Chain].AccountPrefix, ) - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - bz, err := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "rly", "config", "init").CombinedOutput() + bz, err := target.ExecCommand( + "rly", "config", "init").CombinedOutput() if err != nil && !strings.Contains(string(bz), "config already exists") { log.Fatal(err, "\n", string(bz)) } @@ -752,24 +813,22 @@ func (tr TestConfig) addChainToGorelayer( chainConfigFileName := fmt.Sprintf("/root/%s_config.json", ChainId) bashCommand := fmt.Sprintf(`echo '%s' >> %s`, chainConfig, chainConfigFileName) - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - bz, err = exec.Command("docker", "exec", tr.containerConfig.InstanceName, "bash", "-c", + bz, err = target.ExecCommand("bash", "-c", bashCommand).CombinedOutput() if err != nil { log.Fatal(err, "\n", string(bz)) } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - addChainCommand := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "rly", "chains", "add", "--file", chainConfigFileName, string(ChainId)) + addChainCommand := target.ExecCommand("rly", "chains", "add", "--file", chainConfigFileName, string(ChainId)) executeCommand(addChainCommand, "add chain") - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - keyRestoreCommand := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "rly", "keys", "restore", string(ChainId), "default", tr.validatorConfigs[action.Validator].Mnemonic) + keyRestoreCommand := target.ExecCommand("rly", "keys", "restore", string(ChainId), "default", tr.validatorConfigs[action.Validator].Mnemonic) executeCommand(keyRestoreCommand, "restore keys") } func (tr TestConfig) addChainToHermes( action AddChainToRelayerAction, + target ExecutionTarget, verbose bool, ) { queryNodeIP := tr.getQueryNodeIP(action.Chain) @@ -791,10 +850,7 @@ func (tr TestConfig) addChainToHermes( bashCommand := fmt.Sprintf(`echo '%s' >> %s`, chainConfig, "/root/.hermes/config.toml") - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - bz, err := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "bash", "-c", - bashCommand, - ).CombinedOutput() + bz, err := target.ExecCommand("bash", "-c", bashCommand).CombinedOutput() if err != nil { log.Fatal(err, "\n", string(bz)) } @@ -809,16 +865,12 @@ func (tr TestConfig) addChainToHermes( saveMnemonicCommand := fmt.Sprintf(`echo '%s' > %s`, mnemonic, "/root/.hermes/mnemonic.txt") fmt.Println("Add to hermes", action.Validator) - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - bz, err = exec.Command("docker", "exec", tr.containerConfig.InstanceName, "bash", "-c", - saveMnemonicCommand, - ).CombinedOutput() + bz, err = target.ExecCommand("bash", "-c", saveMnemonicCommand).CombinedOutput() if err != nil { log.Fatal(err, "\n", string(bz)) } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - bz, err = exec.Command("docker", "exec", tr.containerConfig.InstanceName, "hermes", + bz, err = target.ExecCommand("hermes", "keys", "add", "--chain", string(tr.chainConfigs[action.Chain].ChainId), "--mnemonic-file", "/root/.hermes/mnemonic.txt", @@ -858,10 +910,11 @@ type AddIbcConnectionAction struct { func (tr TestConfig) addIbcConnection( action AddIbcConnectionAction, + target ExecutionTarget, verbose bool, ) { if !tr.useGorelayer { - tr.addIbcConnectionHermes(action, verbose) + tr.addIbcConnectionHermes(action, target, verbose) } else { tr.addIbcConnectionGorelayer(action, verbose) } @@ -927,10 +980,10 @@ type CreateIbcClientsAction struct { // otherwise, it would use client provided as CLI argument (-a-client) func (tr TestConfig) createIbcClientsHermes( action CreateIbcClientsAction, + target ExecutionTarget, verbose bool, ) { - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "hermes", + cmd := target.ExecCommand("hermes", "create", "connection", "--a-chain", string(tr.chainConfigs[action.ChainA].ChainId), "--b-chain", string(tr.chainConfigs[action.ChainB].ChainId), @@ -964,10 +1017,10 @@ func (tr TestConfig) createIbcClientsHermes( func (tr TestConfig) addIbcConnectionHermes( action AddIbcConnectionAction, + target ExecutionTarget, verbose bool, ) { - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "hermes", + cmd := target.ExecCommand("hermes", "create", "connection", "--a-chain", string(tr.chainConfigs[action.ChainA].ChainId), "--a-client", "07-tendermint-"+fmt.Sprint(action.ClientA), @@ -1014,24 +1067,23 @@ type StartRelayerAction struct{} func (tr TestConfig) startRelayer( action StartRelayerAction, + target ExecutionTarget, verbose bool, ) { if tr.useGorelayer { - tr.startGorelayer(action, verbose) + tr.startGorelayer(action, target, verbose) } else { - tr.startHermes(action, verbose) + tr.startHermes(action, target, verbose) } } func (tr TestConfig) startGorelayer( action StartRelayerAction, + target ExecutionTarget, verbose bool, ) { // gorelayer start is running in detached mode - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", "-d", tr.containerConfig.InstanceName, "rly", - "start", - ) + cmd := target.ExecDetachedCommand("rly", "start") if err := cmd.Start(); err != nil { log.Fatal(err) @@ -1044,13 +1096,11 @@ func (tr TestConfig) startGorelayer( func (tr TestConfig) startHermes( action StartRelayerAction, + target ExecutionTarget, verbose bool, ) { // hermes start is running in detached mode - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", "-d", tr.containerConfig.InstanceName, "hermes", - "start", - ) + cmd := target.ExecDetachedCommand("hermes", "start") if err := cmd.Start(); err != nil { log.Fatal(err) @@ -1063,22 +1113,23 @@ func (tr TestConfig) startHermes( func (tr TestConfig) addIbcChannel( action AddIbcChannelAction, + target ExecutionTarget, verbose bool, ) { if tr.useGorelayer { - tr.addIbcChannelGorelayer(action, verbose) + tr.addIbcChannelGorelayer(action, target, verbose) } else { - tr.addIbcChannelHermes(action, verbose) + tr.addIbcChannelHermes(action, target, verbose) } } func (tr TestConfig) addIbcChannelGorelayer( action AddIbcChannelAction, + target ExecutionTarget, verbose bool, ) { pathName := tr.GetPathNameForGorelayer(action.ChainA, action.ChainB) - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "rly", + cmd := target.ExecCommand("rly", "transact", "channel", pathName, "--src-port", action.PortA, @@ -1092,6 +1143,7 @@ func (tr TestConfig) addIbcChannelGorelayer( func (tr TestConfig) addIbcChannelHermes( action AddIbcChannelAction, + target ExecutionTarget, verbose bool, ) { // if version is not specified, use the default version when creating ccv connections @@ -1101,8 +1153,7 @@ func (tr TestConfig) addIbcChannelHermes( chanVersion = tr.containerConfig.CcvVersion } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "hermes", + cmd := target.ExecCommand("hermes", "create", "channel", "--a-chain", string(tr.chainConfigs[action.ChainA].ChainId), "--a-connection", "connection-"+fmt.Sprint(action.ConnectionA), @@ -1155,14 +1206,14 @@ type TransferChannelCompleteAction struct { func (tr TestConfig) transferChannelComplete( action TransferChannelCompleteAction, + target ExecutionTarget, verbose bool, ) { if tr.useGorelayer { log.Fatal("transferChannelComplete is not implemented for rly") } - //#nosec G204 -- Bypass linter warning for spawning subprocess with chanOpenTryCmd arguments. - chanOpenTryCmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "hermes", + chanOpenTryCmd := target.ExecCommand("hermes", "tx", "chan-open-try", "--dst-chain", string(tr.chainConfigs[action.ChainB].ChainId), "--src-chain", string(tr.chainConfigs[action.ChainA].ChainId), @@ -1173,8 +1224,7 @@ func (tr TestConfig) transferChannelComplete( ) executeCommand(chanOpenTryCmd, "transferChanOpenTry") - //#nosec G204 -- Bypass linter warning for spawning subprocess with chanOpenAckCmd arguments. - chanOpenAckCmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "hermes", + chanOpenAckCmd := target.ExecCommand("hermes", "tx", "chan-open-ack", "--dst-chain", string(tr.chainConfigs[action.ChainA].ChainId), "--src-chain", string(tr.chainConfigs[action.ChainB].ChainId), @@ -1187,8 +1237,7 @@ func (tr TestConfig) transferChannelComplete( executeCommand(chanOpenAckCmd, "transferChanOpenAck") - //#nosec G204 -- Bypass linter warning for spawning subprocess with chanOpenConfirmCmd arguments. - chanOpenConfirmCmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "hermes", + chanOpenConfirmCmd := target.ExecCommand("hermes", "tx", "chan-open-confirm", "--dst-chain", string(tr.chainConfigs[action.ChainB].ChainId), "--src-chain", string(tr.chainConfigs[action.ChainA].ChainId), @@ -1243,24 +1292,25 @@ type RelayPacketsAction struct { func (tr TestConfig) relayPackets( action RelayPacketsAction, + target ExecutionTarget, verbose bool, ) { if tr.useGorelayer { - tr.relayPacketsGorelayer(action, verbose) + tr.relayPacketsGorelayer(action, target, verbose) } else { - tr.relayPacketsHermes(action, verbose) + tr.relayPacketsHermes(action, target, verbose) } } func (tr TestConfig) relayPacketsGorelayer( action RelayPacketsAction, + target ExecutionTarget, verbose bool, ) { pathName := tr.GetPathNameForGorelayer(action.ChainA, action.ChainB) // rly transact relay-packets [path-name] --channel [channel-id] - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "rly", "transact", "flush", + cmd := target.ExecCommand("rly", "transact", "flush", pathName, "channel-"+fmt.Sprint(action.Channel), ) @@ -1278,11 +1328,11 @@ func (tr TestConfig) relayPacketsGorelayer( func (tr TestConfig) relayPacketsHermes( action RelayPacketsAction, + target ExecutionTarget, verbose bool, ) { // hermes clear packets ibc0 transfer channel-13 - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "hermes", "clear", "packets", + cmd := target.ExecCommand("hermes", "clear", "packets", "--chain", string(tr.chainConfigs[action.ChainA].ChainId), "--port", action.Port, "--channel", "channel-"+fmt.Sprint(action.Channel), @@ -1309,6 +1359,7 @@ type RelayRewardPacketsToProviderAction struct { func (tr TestConfig) relayRewardPacketsToProvider( action RelayRewardPacketsToProviderAction, + target ExecutionTarget, verbose bool, ) { blockPerDistribution, _ := strconv.ParseUint(strings.Trim(tr.getParam(action.ConsumerChain, Param{Subspace: "ccvconsumer", Key: "BlocksPerDistributionTransmission"}), "\""), 10, 64) @@ -1317,7 +1368,7 @@ func (tr TestConfig) relayRewardPacketsToProvider( tr.waitBlocks(action.ConsumerChain, uint(blockPerDistribution-currentBlock+1), 60*time.Second) } - tr.relayPackets(RelayPacketsAction{ChainA: action.ConsumerChain, ChainB: action.ProviderChain, Port: action.Port, Channel: action.Channel}, verbose) + tr.relayPackets(RelayPacketsAction{ChainA: action.ConsumerChain, ChainB: action.ProviderChain, Port: action.Port, Channel: action.Channel}, target, verbose) tr.waitBlocks(action.ProviderChain, 1, 10*time.Second) } @@ -1330,6 +1381,7 @@ type DelegateTokensAction struct { func (tr TestConfig) delegateTokens( action DelegateTokensAction, + target ExecutionTarget, verbose bool, ) { toValCfg := tr.validatorConfigs[action.To] @@ -1344,8 +1396,7 @@ func (tr TestConfig) delegateTokens( } } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[action.Chain].BinaryName, + cmd := target.ExecCommand(tr.chainConfigs[action.Chain].BinaryName, "tx", "staking", "delegate", validatorAddress, @@ -1381,6 +1432,7 @@ type UnbondTokensAction struct { func (tr TestConfig) unbondTokens( action UnbondTokensAction, + target ExecutionTarget, verbose bool, ) { unbondFromValCfg := tr.validatorConfigs[action.UnbondFrom] @@ -1395,8 +1447,7 @@ func (tr TestConfig) unbondTokens( } } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[action.Chain].BinaryName, + cmd := target.ExecCommand(tr.chainConfigs[action.Chain].BinaryName, "tx", "staking", "unbond", validatorAddress, @@ -1433,6 +1484,7 @@ type CancelUnbondTokensAction struct { func (tr TestConfig) cancelUnbondTokens( action CancelUnbondTokensAction, + target ExecutionTarget, verbose bool, ) { valCfg := tr.validatorConfigs[action.Validator] @@ -1456,8 +1508,7 @@ func (tr TestConfig) cancelUnbondTokens( } // get creation-height from state - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[action.Chain].BinaryName, + cmd := target.ExecCommand(tr.chainConfigs[action.Chain].BinaryName, "q", "staking", "unbonding-delegation", delegatorAddress, validatorAddress, @@ -1478,8 +1529,7 @@ func (tr TestConfig) cancelUnbondTokens( log.Fatal("invalid creation height") } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd = exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[action.Chain].BinaryName, + cmd = target.ExecCommand(tr.chainConfigs[action.Chain].BinaryName, "tx", "staking", "cancel-unbond", validatorAddress, fmt.Sprint(action.Amount)+`stake`, @@ -1515,7 +1565,7 @@ type RedelegateTokensAction struct { Amount uint } -func (tr TestConfig) redelegateTokens(action RedelegateTokensAction, verbose bool) { +func (tr TestConfig) redelegateTokens(action RedelegateTokensAction, target ExecutionTarget, verbose bool) { srcCfg := tr.validatorConfigs[action.Src] dstCfg := tr.validatorConfigs[action.Dst] redelegateSrc := srcCfg.ValoperAddress @@ -1536,9 +1586,7 @@ func (tr TestConfig) redelegateTokens(action RedelegateTokensAction, verbose boo } } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", - tr.containerConfig.InstanceName, + cmd := target.ExecCommand( tr.chainConfigs[action.Chain].BinaryName, "tx", "staking", "redelegate", @@ -1587,17 +1635,17 @@ func (tr TestConfig) getValidatorKeyAddressFromString(keystring string) string { return key.Address } -func (tr TestConfig) invokeDowntimeSlash(action DowntimeSlashAction, verbose bool) { +func (tr TestConfig) invokeDowntimeSlash(action DowntimeSlashAction, target ExecutionTarget, verbose bool) { // Bring validator down - tr.setValidatorDowntime(action.Chain, action.Validator, true, verbose) + tr.setValidatorDowntime(action.Chain, action.Validator, true, target, verbose) // Wait appropriate amount of blocks for validator to be slashed tr.waitBlocks(action.Chain, 10, 3*time.Minute) // Bring validator back up - tr.setValidatorDowntime(action.Chain, action.Validator, false, verbose) + tr.setValidatorDowntime(action.Chain, action.Validator, false, target, verbose) } // Sets validator downtime by setting the virtual ethernet interface of a node to "up" or "down" -func (tr TestConfig) setValidatorDowntime(chain ChainID, validator ValidatorID, down, verbose bool) { +func (tr TestConfig) setValidatorDowntime(chain ChainID, validator ValidatorID, down bool, target ExecutionTarget, verbose bool) { var lastArg string if down { lastArg = "down" @@ -1618,11 +1666,7 @@ func (tr TestConfig) setValidatorDowntime(chain ChainID, validator ValidatorID, return } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command( - "docker", - "exec", - tr.containerConfig.InstanceName, + cmd := target.ExecCommand( "ip", "link", "set", @@ -1662,13 +1706,11 @@ type UnjailValidatorAction struct { } // Sends an unjail transaction to the provider chain -func (tr TestConfig) unjailValidator(action UnjailValidatorAction, verbose bool) { +func (tr TestConfig) unjailValidator(action UnjailValidatorAction, target ExecutionTarget, verbose bool) { // wait until downtime_jail_duration has elapsed, to make sure the validator can be unjailed tr.WaitTime(61 * time.Second) - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", - tr.containerConfig.InstanceName, + cmd := target.ExecCommand( tr.chainConfigs[action.Provider].BinaryName, "tx", "slashing", "unjail", // Validator is sender here @@ -1703,6 +1745,7 @@ type RegisterRepresentativeAction struct { func (tr TestConfig) registerRepresentative( action RegisterRepresentativeAction, + target ExecutionTarget, verbose bool, ) { var wg sync.WaitGroup @@ -1712,8 +1755,7 @@ func (tr TestConfig) registerRepresentative( go func(val ValidatorID, stake uint) { defer wg.Done() - //#nosec G204 -- Bypass linter warning for spawning subprocess with pubKeycmd arguments. - pubKeycmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[action.Chain].BinaryName, + pubKeycmd := target.ExecCommand(tr.chainConfigs[action.Chain].BinaryName, "tendermint", "show-validator", `--home`, tr.getValidatorHome(action.Chain, val), ) @@ -1723,8 +1765,7 @@ func (tr TestConfig) registerRepresentative( log.Fatal(err, "\n", string(bzPubKey)) } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - bz, err := exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[action.Chain].BinaryName, + bz, err := target.ExecCommand(tr.chainConfigs[action.Chain].BinaryName, "tx", "staking", "create-validator", `--amount`, fmt.Sprint(stake)+"stake", `--pubkey`, string(bzPubKey), @@ -1758,7 +1799,7 @@ type SubmitChangeRewardDenomsProposalAction struct { From ValidatorID } -func (tr TestConfig) submitChangeRewardDenomsProposal(action SubmitChangeRewardDenomsProposalAction, verbose bool) { +func (tr TestConfig) submitChangeRewardDenomsProposal(action SubmitChangeRewardDenomsProposalAction, target ExecutionTarget, verbose bool) { providerChain := tr.chainConfigs[ChainID("provi")] prop := client.ChangeRewardDenomsProposalJSON{ @@ -1782,17 +1823,15 @@ func (tr TestConfig) submitChangeRewardDenomsProposal(action SubmitChangeRewardD log.Fatal("prop json contains single quote") } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - bz, err = exec.Command("docker", "exec", tr.containerConfig.InstanceName, + bz, err = target.ExecCommand( "/bin/bash", "-c", fmt.Sprintf(`echo '%s' > %s`, jsonStr, "/change-reward-denoms-proposal.json")).CombinedOutput() if err != nil { log.Fatal(err, "\n", string(bz)) } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. // CHANGE REWARDS DENOM PROPOSAL - bz, err = exec.Command("docker", "exec", tr.containerConfig.InstanceName, providerChain.BinaryName, + bz, err = target.ExecCommand(providerChain.BinaryName, "tx", "gov", "submit-legacy-proposal", "change-reward-denoms", "/change-reward-denoms-proposal.json", `--from`, `validator`+fmt.Sprint(action.From), `--chain-id`, string(providerChain.ChainId), @@ -1829,13 +1868,14 @@ type DoublesignSlashAction struct { func (tr TestConfig) invokeDoublesignSlash( action DoublesignSlashAction, + target ExecutionTarget, verbose bool, ) { if !tr.useCometmock { chainConfig := tr.chainConfigs[action.Chain] - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - bz, err := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "/bin/bash", - "/testnet-scripts/cause-doublesign.sh", chainConfig.BinaryName, string(action.Validator), + doubleSignScript := target.GetTestScriptPath(false, "cause-doublesign.sh") + bz, err := target.ExecCommand("/bin/bash", + doubleSignScript, chainConfig.BinaryName, string(action.Validator), string(chainConfig.ChainId), chainConfig.IpPrefix).CombinedOutput() if err != nil { log.Fatal(err, "\n", string(bz)) @@ -1941,7 +1981,7 @@ type AssignConsumerPubKeyAction struct { ExpectedError string } -func (tr TestConfig) assignConsumerPubKey(action AssignConsumerPubKeyAction, verbose bool) { +func (tr TestConfig) assignConsumerPubKey(action AssignConsumerPubKeyAction, target ExecutionTarget, verbose bool) { valCfg := tr.validatorConfigs[action.Validator] // Note: to get error response reported back from this command '--gas auto' needs to be set. @@ -1962,9 +2002,7 @@ func (tr TestConfig) assignConsumerPubKey(action AssignConsumerPubKeyAction, ver gas, ) - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", - tr.containerConfig.InstanceName, + cmd := target.ExecCommand( "/bin/bash", "-c", assignKey, ) @@ -1991,9 +2029,10 @@ func (tr TestConfig) assignConsumerPubKey(action AssignConsumerPubKeyAction, ver // node was started with provider key // we swap the nodes's keys for consumer keys and restart it if action.ReconfigureNode { - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - configureNodeCmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "/bin/bash", - "/testnet-scripts/reconfigure-node.sh", tr.chainConfigs[action.Chain].BinaryName, + isConsumer := tr.chainConfigs[action.Chain].BinaryName != "interchain-security-pd" + reconfigureScript := target.GetTestScriptPath(isConsumer, "reconfigure-node.sh") + configureNodeCmd := target.ExecCommand("/bin/bash", + reconfigureScript, tr.chainConfigs[action.Chain].BinaryName, string(action.Validator), string(action.Chain), tr.chainConfigs[action.Chain].IpPrefix, valCfg.IpSuffix, valCfg.ConsumerMnemonic, valCfg.ConsumerPrivValidatorKey, @@ -2113,12 +2152,12 @@ type StartConsumerEvidenceDetectorAction struct { func (tc TestConfig) startConsumerEvidenceDetector( action StartConsumerEvidenceDetectorAction, + target ExecutionTarget, verbose bool, ) { chainConfig := tc.chainConfigs[action.Chain] // run in detached mode so it will keep running in the background - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - bz, err := exec.Command("docker", "exec", "-d", tc.containerConfig.InstanceName, + bz, err := target.ExecDetachedCommand( "hermes", "evidence", "--chain", string(chainConfig.ChainId)).CombinedOutput() if err != nil { log.Fatal(err, "\n", string(bz)) diff --git a/tests/e2e/actions_sovereign_chain.go b/tests/e2e/actions_sovereign_chain.go index 9a5bb0f182..187c9036cc 100644 --- a/tests/e2e/actions_sovereign_chain.go +++ b/tests/e2e/actions_sovereign_chain.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "log" - "os/exec" "time" ) @@ -20,6 +19,7 @@ type StartSovereignChainAction struct { // upgrades are simpler with a single validator node since only one node needs to be upgraded func (tr TestConfig) startSovereignChain( action StartSovereignChainAction, + target ExecutionTarget, verbose bool, ) { chainConfig := tr.chainConfigs["sover"] @@ -68,12 +68,11 @@ func (tr TestConfig) startSovereignChain( genesisChanges = chainConfig.GenesisChanges } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "/bin/bash", - "/testnet-scripts/start-sovereign.sh", chainConfig.BinaryName, string(vals), + isConsumer := chainConfig.BinaryName != "interchain-security-pd" + testScriptPath := target.GetTestScriptPath(isConsumer, "start-sovereign.sh") + cmd := target.ExecCommand("/bin/bash", testScriptPath, string(vals), string(chainConfig.ChainId), chainConfig.IpPrefix, genesisChanges, - tr.tendermintConfigOverride, - ) + tr.tendermintConfigOverride) cmdReader, err := cmd.StdoutPipe() if err != nil { @@ -102,7 +101,7 @@ func (tr TestConfig) startSovereignChain( tr.addChainToRelayer(AddChainToRelayerAction{ Chain: action.Chain, Validator: action.Validators[0].Id, - }, verbose) + }, target, verbose) } type LegacyUpgradeProposalAction struct { @@ -112,7 +111,7 @@ type LegacyUpgradeProposalAction struct { UpgradeHeight uint64 } -func (tr *TestConfig) submitLegacyUpgradeProposal(action LegacyUpgradeProposalAction, verbose bool) { +func (tr *TestConfig) submitLegacyUpgradeProposal(action LegacyUpgradeProposalAction, target ExecutionTarget, verbose bool) { submit := fmt.Sprintf( `%s tx gov submit-legacy-proposal software-upgrade %s \ --title %s \ @@ -137,12 +136,7 @@ func (tr *TestConfig) submitLegacyUpgradeProposal(action LegacyUpgradeProposalAc tr.getValidatorHome(ChainID("sover"), action.Proposer), tr.getValidatorNode(ChainID("sover"), action.Proposer), ) - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", - tr.containerConfig.InstanceName, - "/bin/bash", "-c", - submit, - ) + cmd := target.ExecCommand("/bin/bash", "-c", submit) if verbose { fmt.Println("submitUpgradeProposal cmd:", cmd.String()) diff --git a/tests/e2e/builder.go b/tests/e2e/builder.go new file mode 100644 index 0000000000..63e9961661 --- /dev/null +++ b/tests/e2e/builder.go @@ -0,0 +1,128 @@ +package main + +import ( + "bytes" + "fmt" + "log" + "os" + "os/exec" + "strings" +) + +// setupWorkSpace checks out given revision in a tmp directory +// and returns the path where the workspace is located +func setupWorkSpace(revision string) (string, error) { + workSpace, err := os.MkdirTemp(os.TempDir(), "e2eWorkTree_") + if err != nil { + return "", fmt.Errorf("error creating temp directory %v", err) + } + + fmt.Printf("Setting up worktree in '%s'\n", workSpace) + + //#nosec G204 -- Bypass linter warning for spawning subprocess with variable + cmd := exec.Command("git", "worktree", "add", + "--force", "--checkout", workSpace, revision) + out, err := cmd.CombinedOutput() + fmt.Println("Running: ", cmd.String()) + if err != nil { + log.Printf("Error creating worktree (%v): %s", err, string(out)) + return "", err + } + return workSpace, nil +} + +// cleanupWorkSpace removes the created worktree +func cleanupWorkSpace(workSpacePath string) error { + cmd := exec.Command("git", "worktree", "remove", workSpacePath) + cmd.Stderr = cmd.Stdout + if err := cmd.Start(); err != nil { + log.Printf("Failed removing git worktree '%s' used for docker image creation", workSpacePath) + return err + } + return cmd.Wait() +} + +// Check if docker is running +func dockerIsUp() bool { + cmd := exec.Command("docker", "info") + var errbuf bytes.Buffer + cmd.Stderr = &errbuf + if err := cmd.Run(); err != nil { + log.Printf("Docker engine is not running (%v): %s", err, errbuf.String()) + return false + } + return true +} + +// Build docker image of ICS for a given revision +func buildDockerImage(imageName, revision string, targetCfg TargetConfig, noCache bool) error { + fmt.Printf("Building ICS %s image %s\n", revision, imageName) + if !dockerIsUp() { + return fmt.Errorf("docker engine is not running") + } + + workSpace := "./" + var err error = nil + // if a revision is provided the related version will be staged + // on a separate worktree (note: revision should _not_ be checked out already elsewhere). + // which will be deleted after image creation + if revision != "" { + workSpace, err = setupWorkSpace(revision) + if err != nil { + return err + } + defer cleanupWorkSpace(workSpace) + } + + // Check if workspace exists + _, err = os.Stat(workSpace) + if err != nil { + log.Fatalf("image build failed. invalid workspace: %v", err) + } + + // Use custom SDK setup if required + sdkPath := strings.Join([]string{workSpace, "cosmos-sdk"}, string(os.PathSeparator)) + err = os.RemoveAll(sdkPath) //delete old SDK directory + if err != nil { + return fmt.Errorf("error deleting SDK directory from workspace: %v", err) + } + if targetCfg.localSdkPath != "" { + fmt.Printf("Using local SDK version from %s\n", targetCfg.localSdkPath) + //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. + cmd := exec.Command("cp", "-n", "-r", targetCfg.localSdkPath, sdkPath) + out, err := cmd.CombinedOutput() + if err != nil { + log.Printf("Error running command %v: %s", cmd, string(out)) + return fmt.Errorf("error setting up local SDK: %v, %s", err, string(out)) + } + } else { + fmt.Println("Using default SDK version") + } + + dockerFile := "Dockerfile" + args := []string{"build", "-t", imageName} + if noCache { + args = append(args, "--no-cache") + } + + if targetCfg.useGaia && targetCfg.gaiaTag != "" { + dockerFile = "Dockerfile.gaia" + args = append(args, "--build-arg", fmt.Sprintf("USE_GAIA_TAG=%s", targetCfg.gaiaTag)) + } + args = append(args, "-f", dockerFile, "./") + + //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. + cmd := exec.Command("docker", args...) + cmd.Dir = workSpace + out, err := cmd.CombinedOutput() + if err != nil && !noCache { + // Retry image creation from pristine state by enforcing --no-cache + log.Printf("Image creation failed '%v'. Re-trying without cache!", err) + return buildDockerImage(imageName, revision, targetCfg, true) + } + if err != nil { + return fmt.Errorf("building docker image failed running '%v' (%v): %s", cmd, err, out) + } + + return err +} diff --git a/tests/e2e/config.go b/tests/e2e/config.go index c2079af0c3..307f61d62c 100644 --- a/tests/e2e/config.go +++ b/tests/e2e/config.go @@ -101,7 +101,14 @@ type ContainerConfig struct { Now time.Time } -// TODO: Split out TestConfig and system wide config like localsdkpath +type TargetConfig struct { + gaiaTag string + localSdkPath string + useGaia bool + providerVersion string + consumerVersion string +} + type TestConfig struct { // These are the non altered values during a typical test run, where multiple test runs can exist // to validate different action sequences and corresponding state checks. @@ -113,17 +120,14 @@ type TestConfig struct { // having shorter timeout_commit reduces the test runtime because blocks are produced faster // lengthening the timeout_commit increases the test runtime because blocks are produced slower but the test is more reliable tendermintConfigOverride string - localSdkPath string - useGaia bool useCometmock bool // if false, nodes run CometBFT useGorelayer bool // if false, Hermes is used as the relayer - gaiaTag string // chains which are running, i.e. producing blocks, at the moment runningChains map[ChainID]bool // Used with CometMock. The time by which chains have been advanced. Used to keep chains in sync: when a new chain is started, advance its time by this value to keep chains in sync. - timeOffset time.Duration - - name string + timeOffset time.Duration + transformGenesis bool + name string } // Initialize initializes the TestConfig instance by setting the runningChains field to an empty map. @@ -575,10 +579,6 @@ func (s *TestConfig) SetDockerConfig(localSdkPath string, useGaia bool, gaiaTag if useGaia { fmt.Println("USING GAIA INSTEAD OF ICS provider app", gaiaTag) } - - s.useGaia = useGaia - s.gaiaTag = gaiaTag - s.localSdkPath = localSdkPath } func (s *TestConfig) SetCometMockConfig(useCometmock bool) { diff --git a/tests/e2e/main.go b/tests/e2e/main.go index 5f3ea71e74..03ce45180f 100644 --- a/tests/e2e/main.go +++ b/tests/e2e/main.go @@ -1,18 +1,12 @@ package main import ( - "bufio" "flag" "fmt" "log" - "os/exec" - "reflect" - "strconv" "strings" "sync" "time" - - "github.com/kylelemons/godebug/pretty" ) // The list of test cases to be executed @@ -33,6 +27,21 @@ func (t *TestSet) String() string { return fmt.Sprint(*t) } +type VersionSet map[string]bool + +func (vs *VersionSet) Set(value string) error { + (*vs)[value] = true + return nil +} + +func (vs *VersionSet) String() string { + keys := []string{} + for k, _ := range *vs { + keys = append(keys, k) + } + return fmt.Sprint(keys) +} + var ( verbose = flag.Bool("verbose", false, "turn verbose logging on/off") includeMultiConsumer = flag.Bool("include-multi-consumer", false, "include multiconsumer tests in run") @@ -48,6 +57,12 @@ var ( gaiaTag = flag.String("gaia-tag", "", "gaia tag to use - default is latest") ) +var ( + consumerVersions VersionSet = VersionSet{} + providerVersions VersionSet = VersionSet{} + transformGenesis = flag.Bool("transform-genesis", false, "enforces a consumer app to perform genesis transformation of exported ccv genesis data. For details see compatibility notes (RELEASES.md) of used versions") +) + var ( selectedTests TestSet @@ -137,31 +152,6 @@ var stepChoices = map[string]StepChoice{ }, } -func executeTests(tests []testStepsWithConfig) (err error) { - if parallel != nil && *parallel { - fmt.Println("=============== running all tests in parallel ===============") - } - - var wg sync.WaitGroup - for _, testCase := range tests { - if parallel != nil && *parallel { - wg.Add(1) - go func(run testStepsWithConfig) { - defer wg.Done() - run.testRun.Run(run.steps, *localSdkPath, *useGaia, *gaiaTag) - }(testCase) - } else { - log.Printf("=============== running %s ===============\n", testCase.testRun.name) - testCase.testRun.Run(testCase.steps, *localSdkPath, *useGaia, *gaiaTag) - } - } - - if parallel != nil && *parallel { - wg.Wait() - } - return -} - func getTestCaseUsageString() string { var builder strings.Builder @@ -215,6 +205,10 @@ func parseArguments() (err error) { flag.Var(&selectedTestfiles, "test-file", getTestFileUsageString()) + + flag.Var(&consumerVersions, "cv", "Version (git tag, revison, branch) of the consumer to be tested. Tests will be run against combinations of all defined provider versions (-pv) with this consumer version. Default: consumer implementation of local workspace") + flag.Var(&providerVersions, "pv", "Version (git tag, revison, branch) of the provider to be tested. Tests will be run against combinations of all defined consumer versions (-cv) with this provider version. Default: provider implementation of local workspace") + flag.Parse() // Enforce go-relayer in case of cometmock as hermes is not yet supported @@ -295,226 +289,128 @@ func getTestCases(selectedPredefinedTests, selectedTestFiles TestSet) (tests []t return tests } -// runs E2E tests -// all docker containers are built sequentially to avoid race conditions when using local cosmos-sdk -// after building docker containers, all tests are run in parallel using their respective docker containers -func main() { - if err := parseArguments(); err != nil { - flag.Usage() - log.Fatalf("Error parsing command arguments %s\n", err) - } - - testCases := getTestCases(selectedTests, selectedTestfiles) - - start := time.Now() - err := executeTests(testCases) - if err != nil { - log.Fatalf("Test execution failed '%s'", err) +// delete all test targets +func deleteTargets(targets []ExecutionTarget) { + for _, target := range targets { + if err := target.Delete(); err != nil { + log.Println("error deleting target: ", err) + } } - log.Printf("TOTAL TIME ELAPSED: %v\n", time.Since(start)) } -// Run sets up docker container and executes the steps in the test run. -// Docker containers are torn down after the test run is complete. -func (tr *TestConfig) Run(steps []Step, localSdkPath string, useGaia bool, gaiaTag string) { - tr.SetDockerConfig(localSdkPath, useGaia, gaiaTag) - tr.SetCometMockConfig(*useCometmock) - tr.SetRelayerConfig(*useGorelayer) - - tr.validateStringLiterals() - tr.startDocker() - tr.executeSteps(steps) - tr.teardownDocker() -} +// Create targets where test cases should be executed on +// For each combination of provider & consumer versions an ExecutionTarget +// is created. +func createTargets(providerVersions, consumerVersions VersionSet) ([]ExecutionTarget, error) { + targetCfg := TargetConfig{useGaia: *useGaia, localSdkPath: *localSdkPath, gaiaTag: *gaiaTag} + var targets []ExecutionTarget -type StepChoice struct { - name string - steps []Step - description string - testConfig TestConfig -} - -func (tr *TestConfig) runStep(step Step, verbose bool) { - switch action := step.Action.(type) { - case StartChainAction: - tr.startChain(action, verbose) - case StartSovereignChainAction: - tr.startSovereignChain(action, verbose) - case LegacyUpgradeProposalAction: - tr.submitLegacyUpgradeProposal(action, verbose) - case WaitUntilBlockAction: - tr.waitUntilBlockOnChain(action) - case ChangeoverChainAction: - tr.changeoverChain(action, verbose) - case SendTokensAction: - tr.sendTokens(action, verbose) - case SubmitTextProposalAction: - tr.submitTextProposal(action, verbose) - case SubmitConsumerAdditionProposalAction: - tr.submitConsumerAdditionProposal(action, verbose) - case SubmitConsumerRemovalProposalAction: - tr.submitConsumerRemovalProposal(action, verbose) - case SubmitParamChangeLegacyProposalAction: - tr.submitParamChangeProposal(action, verbose) - case VoteGovProposalAction: - tr.voteGovProposal(action, verbose) - case StartConsumerChainAction: - tr.startConsumerChain(action, verbose) - case AddChainToRelayerAction: - tr.addChainToRelayer(action, verbose) - case CreateIbcClientsAction: - tr.createIbcClientsHermes(action, verbose) - case AddIbcConnectionAction: - tr.addIbcConnection(action, verbose) - case AddIbcChannelAction: - tr.addIbcChannel(action, verbose) - case TransferChannelCompleteAction: - tr.transferChannelComplete(action, verbose) - case RelayPacketsAction: - tr.relayPackets(action, verbose) - case RelayRewardPacketsToProviderAction: - tr.relayRewardPacketsToProvider(action, verbose) - case DelegateTokensAction: - tr.delegateTokens(action, verbose) - case UnbondTokensAction: - tr.unbondTokens(action, verbose) - case CancelUnbondTokensAction: - tr.cancelUnbondTokens(action, verbose) - case RedelegateTokensAction: - tr.redelegateTokens(action, verbose) - case DowntimeSlashAction: - tr.invokeDowntimeSlash(action, verbose) - case UnjailValidatorAction: - tr.unjailValidator(action, verbose) - case DoublesignSlashAction: - tr.invokeDoublesignSlash(action, verbose) - case LightClientAmnesiaAttackAction: - tr.lightClientAmnesiaAttack(action, verbose) - case LightClientEquivocationAttackAction: - tr.lightClientEquivocationAttack(action, verbose) - case LightClientLunaticAttackAction: - tr.lightClientLunaticAttack(action, verbose) - case RegisterRepresentativeAction: - tr.registerRepresentative(action, verbose) - case AssignConsumerPubKeyAction: - tr.assignConsumerPubKey(action, verbose) - case SlashMeterReplenishmentAction: - tr.waitForSlashMeterReplenishment(action, verbose) - case WaitTimeAction: - tr.waitForTime(action, verbose) - case StartRelayerAction: - tr.startRelayer(action, verbose) - case ForkConsumerChainAction: - tr.forkConsumerChain(action, verbose) - case UpdateLightClientAction: - tr.updateLightClient(action, verbose) - case StartConsumerEvidenceDetectorAction: - tr.startConsumerEvidenceDetector(action, verbose) - case SubmitChangeRewardDenomsProposalAction: - tr.submitChangeRewardDenomsProposal(action, verbose) - default: - log.Fatalf("unknown action in testRun %s: %#v", tr.name, action) + if len(consumerVersions) == 0 { + consumerVersions[""] = true } - - modelState := step.State - actualState := tr.getState(step.State) - - // Check state - if !reflect.DeepEqual(actualState, modelState) { - fmt.Printf("=============== %s FAILED ===============\n", tr.name) - fmt.Println("FAILED action", reflect.TypeOf(step.Action).Name()) - pretty.Print("actual state", actualState) - pretty.Print("model state", modelState) - log.Fatal(`actual state (-) not equal to model state (+): ` + pretty.Compare(actualState, modelState)) + if len(providerVersions) == 0 { + providerVersions[""] = true } -} - -// executeSteps sequentially runs steps. -func (tr *TestConfig) executeSteps(steps []Step) { - fmt.Printf("=============== started %s tests ===============\n", tr.name) - start := time.Now() - for i, step := range steps { - // print something the show the test is alive - fmt.Printf("running %s: step %d == %s \n", - tr.name, i+1, reflect.TypeOf(step.Action).Name()) - tr.runStep(step, *verbose) + for provider, _ := range providerVersions { + for consumer, _ := range consumerVersions { + targetCfg.consumerVersion = consumer + targetCfg.providerVersion = provider + target := DockerContainer{targetConfig: targetCfg} + err := target.Build() + if err != nil { + log.Println("@@@ failed creating target") + deleteTargets(targets) + return nil, err + } + targets = append(targets, &target) + } } - - fmt.Printf("=============== finished %s tests in %v ===============\n", tr.name, time.Since(start)) + return targets, nil } -func (tr *TestConfig) startDocker() { - fmt.Printf("=============== building %s testRun ===============\n", tr.name) - localSdk := tr.localSdkPath - if localSdk == "" { - localSdk = "default" - } - useGaia := "false" - gaiaTag := "" - if tr.useGaia { - useGaia = "true" - if tr.gaiaTag != "" { - majVersion, err := strconv.Atoi(tr.gaiaTag[1:strings.Index(tr.gaiaTag, ".")]) - if err != nil { - panic(fmt.Sprintf("invalid gaia version %s", tr.gaiaTag)) +func createTestRunners(targets []ExecutionTarget, testCases []testStepsWithConfig) []TestRunner { + runners := []TestRunner{} + for _, target := range targets { + for _, tc := range testCases { + tr := TestRunner{ + config: tc.testRun, + steps: tc.steps, + target: target, + verbose: *verbose, } - if majVersion < 9 { - panic(fmt.Sprintf("gaia version %s is not supported - supporting only v9.x.x and newer", tr.gaiaTag)) - } - gaiaTag = tr.gaiaTag + //TODO: refactor this target specific setting + tr.target.(*DockerContainer).containerCfg = tc.testRun.containerConfig + tr.config.transformGenesis = *transformGenesis + tr.config.SetCometMockConfig(*useCometmock) + tr.config.SetRelayerConfig(*useGorelayer) + runners = append(runners, tr) } } - scriptStr := fmt.Sprintf( - "tests/e2e/testnet-scripts/start-docker.sh %s %s %s %s %s", - tr.containerConfig.ContainerName, - tr.containerConfig.InstanceName, - localSdk, - useGaia, - gaiaTag, - ) - - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("/bin/bash", "-c", scriptStr) - - cmdReader, err := cmd.StdoutPipe() - if err != nil { - log.Fatal(err) - } - cmd.Stderr = cmd.Stdout + return runners +} - if err := cmd.Start(); err != nil { - log.Fatal(err) +func executeTests(runners []TestRunner) error { + if parallel != nil && *parallel { + fmt.Println("=============== running all tests in parallel ===============") } - scanner := bufio.NewScanner(cmdReader) + var wg sync.WaitGroup + var err error = nil - for scanner.Scan() { - out := scanner.Text() - if verbose != nil && *verbose { - fmt.Println("startDocker: " + out) - } - if out == "beacon!!!!!!!!!!" { - return + for _, runner := range runners { + if parallel != nil && *parallel { + wg.Add(1) + go func(runner TestRunner) { + defer wg.Done() + result := runner.Run() + if result != nil { + log.Printf("Test '%s' failed", runner.config.name) + err = result + } + }(runner) + } else { + fmt.Printf("=============== running %s ===============\n", runner.config.name) + err = runner.Run() } } - if err := scanner.Err(); err != nil { - log.Fatal(err) + + if parallel != nil && *parallel { + wg.Wait() } - err = cmd.Wait() - log.Fatalf("StartDocker exited with error: %v", err) + return err } -// remove docker container to reduce resource usage -// otherwise the chain will keep running in the background -func (tr *TestConfig) teardownDocker() { - fmt.Printf("=============== tearing down %s testRun ===============\n", tr.name) - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "kill", tr.containerConfig.InstanceName) +// runs E2E tests +// all docker containers are built sequentially to avoid race conditions when using local cosmos-sdk +// after building docker containers, all tests are run in parallel using their respective docker containers +func main() { + if err := parseArguments(); err != nil { + flag.Usage() + log.Fatalf("Error parsing command arguments %s\n", err) + } + + testCases := getTestCases(selectedTests, selectedTestfiles) + targets, err := createTargets(providerVersions, consumerVersions) + if err != nil { + log.Fatal("failed creating test targets: ", err) + } + defer func() { deleteTargets(targets) }() - bz, err := cmd.CombinedOutput() + testRunners := createTestRunners(targets, testCases) + start := time.Now() + err = executeTests(testRunners) if err != nil { - log.Fatal(err, "\n", string(bz)) + log.Fatalf("Test execution failed '%s'", err) } + log.Printf("TOTAL TIME ELAPSED: %v\n", time.Since(start)) + +} + +type StepChoice struct { + name string + steps []Step + description string + testConfig TestConfig } diff --git a/tests/e2e/state.go b/tests/e2e/state.go index 0b2c2d1835..27d63187ba 100644 --- a/tests/e2e/state.go +++ b/tests/e2e/state.go @@ -101,10 +101,12 @@ type Param struct { Value string } -func (tr TestConfig) getState(modelState State) State { +func (tr TestConfig) getState(modelState State, verbose bool) State { systemState := State{} for k, modelState := range modelState { - log.Println("Getting model state for chain: ", k) + if verbose { + fmt.Println("Getting model state for chain: ", k) + } systemState[k] = tr.getChainState(k, modelState) } diff --git a/tests/e2e/test_driver.go b/tests/e2e/test_driver.go new file mode 100644 index 0000000000..26f44d9600 --- /dev/null +++ b/tests/e2e/test_driver.go @@ -0,0 +1,150 @@ +package main + +import ( + "fmt" + "log" + "reflect" + "time" + + "github.com/kylelemons/godebug/pretty" +) + +// TestCaseDriver knows how different TC can be executed +type TestCaseDriver interface { + Run(steps []Step, target ExecutionTarget, verbose bool) error +} + +func GetTestCaseDriver(testCfg TestConfig) TestCaseDriver { + return &DefaultDriver{testCfg: testCfg} +} + +type DefaultDriver struct { + testCfg TestConfig + target ExecutionTarget + verbose bool +} + +// Execute tests +func (td *DefaultDriver) Run(steps []Step, target ExecutionTarget, verbose bool) error { + td.target = target + td.verbose = verbose + + fmt.Printf("=============== started %s tests ===============\n", td.testCfg.name) + + start := time.Now() + for i, step := range steps { + fmt.Printf("running %s: step %d/%d == %s \n", + td.testCfg.name, i+1, len(steps), reflect.TypeOf(step.Action).Name()) + + err := td.runStep(step) + if err != nil { + return err + } + } + fmt.Printf("=============== finished %s tests in %v ===============\n", td.testCfg.name, time.Since(start)) + return nil +} + +// runStep executes an action and evaluates the result against expected state +func (td *DefaultDriver) runStep(step Step) error { + err := td.runAction(step.Action) + if err != nil { + return err + } + modelState := step.State + actualState := td.testCfg.getState(modelState, td.verbose) + + // Check state + if !reflect.DeepEqual(actualState, modelState) { + fmt.Printf("=============== %s FAILED ===============\n", td.testCfg.name) + fmt.Println("FAILED action", reflect.TypeOf(step.Action).Name()) + pretty.Print("actual state", actualState) + pretty.Print("model state", modelState) + log.Fatal(`actual state (-) not equal to model state (+): ` + pretty.Compare(actualState, modelState)) + } + return nil +} + +func (td *DefaultDriver) runAction(action interface{}) error { + switch action := action.(type) { + case StartChainAction: + td.testCfg.startChain(action, td.target, td.verbose) + case StartSovereignChainAction: + td.testCfg.startSovereignChain(action, td.target, td.verbose) + case LegacyUpgradeProposalAction: + td.testCfg.submitLegacyUpgradeProposal(action, td.target, td.verbose) + case WaitUntilBlockAction: + td.testCfg.waitUntilBlockOnChain(action) + case ChangeoverChainAction: + td.testCfg.changeoverChain(action, td.target, td.verbose) + case SendTokensAction: + td.testCfg.sendTokens(action, td.target, td.verbose) + case SubmitTextProposalAction: + td.testCfg.submitTextProposal(action, td.target, td.verbose) + case SubmitConsumerAdditionProposalAction: + td.testCfg.submitConsumerAdditionProposal(action, td.target, td.verbose) + case SubmitConsumerRemovalProposalAction: + td.testCfg.submitConsumerRemovalProposal(action, td.target, td.verbose) + case SubmitParamChangeLegacyProposalAction: + td.testCfg.submitParamChangeProposal(action, td.target, td.verbose) + case VoteGovProposalAction: + td.testCfg.voteGovProposal(action, td.target, td.verbose) + case StartConsumerChainAction: + td.testCfg.startConsumerChain(action, td.target, td.verbose) + case AddChainToRelayerAction: + td.testCfg.addChainToRelayer(action, td.target, td.verbose) + case CreateIbcClientsAction: + td.testCfg.createIbcClientsHermes(action, td.target, td.verbose) + case AddIbcConnectionAction: + td.testCfg.addIbcConnection(action, td.target, td.verbose) + case AddIbcChannelAction: + td.testCfg.addIbcChannel(action, td.target, td.verbose) + case TransferChannelCompleteAction: + td.testCfg.transferChannelComplete(action, td.target, td.verbose) + case RelayPacketsAction: + td.testCfg.relayPackets(action, td.target, td.verbose) + case RelayRewardPacketsToProviderAction: + td.testCfg.relayRewardPacketsToProvider(action, td.target, td.verbose) + case DelegateTokensAction: + td.testCfg.delegateTokens(action, td.target, td.verbose) + case UnbondTokensAction: + td.testCfg.unbondTokens(action, td.target, td.verbose) + case CancelUnbondTokensAction: + td.testCfg.cancelUnbondTokens(action, td.target, td.verbose) + case RedelegateTokensAction: + td.testCfg.redelegateTokens(action, td.target, td.verbose) + case DowntimeSlashAction: + td.testCfg.invokeDowntimeSlash(action, td.target, td.verbose) + case UnjailValidatorAction: + td.testCfg.unjailValidator(action, td.target, td.verbose) + case DoublesignSlashAction: + td.testCfg.invokeDoublesignSlash(action, td.target, td.verbose) + case LightClientAmnesiaAttackAction: + td.testCfg.lightClientAmnesiaAttack(action, td.verbose) + case LightClientEquivocationAttackAction: + td.testCfg.lightClientEquivocationAttack(action, td.verbose) + case LightClientLunaticAttackAction: + td.testCfg.lightClientLunaticAttack(action, td.verbose) + case RegisterRepresentativeAction: + td.testCfg.registerRepresentative(action, td.target, td.verbose) + case AssignConsumerPubKeyAction: + td.testCfg.assignConsumerPubKey(action, td.target, td.verbose) + case SlashMeterReplenishmentAction: + td.testCfg.waitForSlashMeterReplenishment(action, td.verbose) + case WaitTimeAction: + td.testCfg.waitForTime(action, td.verbose) + case StartRelayerAction: + td.testCfg.startRelayer(action, td.target, td.verbose) + case ForkConsumerChainAction: + td.testCfg.forkConsumerChain(action, td.verbose) + case UpdateLightClientAction: + td.testCfg.updateLightClient(action, td.verbose) + case StartConsumerEvidenceDetectorAction: + td.testCfg.startConsumerEvidenceDetector(action, td.target, td.verbose) + case SubmitChangeRewardDenomsProposalAction: + td.testCfg.submitChangeRewardDenomsProposal(action, td.target, td.verbose) + default: + log.Fatalf("unknown action in testRun %s: %#v", td.testCfg.name, action) + } + return nil +} diff --git a/tests/e2e/test_runner.go b/tests/e2e/test_runner.go new file mode 100644 index 0000000000..adcf31d592 --- /dev/null +++ b/tests/e2e/test_runner.go @@ -0,0 +1,52 @@ +package main + +import "fmt" + +// A test runner drives the execution of test cases +// It sets up the test environment and the test driver to run the tests +type TestRunner struct { + config TestConfig + steps []Step + testDriver TestCaseDriver + target ExecutionTarget + verbose bool +} + +// Run will set up the test environment and executes the tests +func (tr *TestRunner) Run() error { + err := tr.checkConfig() + if err != nil { + return err + } + + err = tr.setupEnvironment(tr.target) + if err != nil { + return fmt.Errorf("error setting up test environment: %v", err) + } + + tr.testDriver = GetTestCaseDriver(tr.config) + err = tr.testDriver.Run(tr.steps, tr.target, tr.verbose) + if err != nil { + // not tearing down environment for troubleshooting reasons on container + return fmt.Errorf("test run '%s' failed: %v", tr.config.name, err) + } + return tr.teardownEnvironment() +} + +func (tr *TestRunner) checkConfig() error { + tr.config.validateStringLiterals() + return nil +} +func (tr *TestRunner) setupEnvironment(target ExecutionTarget) error { + tr.target = target + return target.Start() +} + +func (tr *TestRunner) teardownEnvironment() error { + return tr.target.Stop() +} + +func (tr *TestRunner) Setup(testCfg TestConfig) error { + tr.config = testCfg + return nil +} diff --git a/tests/e2e/test_target.go b/tests/e2e/test_target.go new file mode 100644 index 0000000000..23de16127f --- /dev/null +++ b/tests/e2e/test_target.go @@ -0,0 +1,228 @@ +package main + +import ( + "bufio" + "fmt" + "log" + "os" + "os/exec" + "strings" +) + +type ExecutionTarget interface { + GetTargetType() string + GetTestScriptPath(isConsumer bool, script string) string + // ExecCommand: when executed the command will run and return after completion + ExecCommand(name string, arg ...string) *exec.Cmd + // ExecDetachedCommand: when executed the command will be run in the background and call will return immediately + ExecDetachedCommand(name string, args ...string) *exec.Cmd + Start() error + Stop() error + Build() error + Delete() error +} + +type DockerContainer struct { + targetConfig TargetConfig + containerCfg ContainerConfig + images []string //images needed to build the target container + ImageName string +} + +func (dc *DockerContainer) GetTargetType() string { + return "docker" +} + +func generateImageName(version string, cfg TargetConfig) (string, error) { + // identify a tag name + tagName := "" + if version == "" { + tagName = "local" // this refers to build from local workspace + } else { + // use git hash of rev as docker image tag + //cmd := exec.Command("git", "rev-parse", "--verify", "--short", version) + cmd := exec.Command("git", "log", "--pretty=format:%h", "-n", "1", version) + out, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("error getting hash for revision: %v, '%s'", err, out) + } + tagName = strings.TrimSpace(string(out)) + } + + imageName := "cosmos-ics" + if cfg.useGaia { + imageName += "_gaia" + tagName += "-" + cfg.gaiaTag + } + if cfg.localSdkPath != "" { + imageName += "_sdk" + } + + return fmt.Sprintf("%s:%s", imageName, tagName), nil +} + +// Build the docker image for the target container +func (dc *DockerContainer) Build() error { + consumerVersion := dc.targetConfig.consumerVersion + providerVersion := dc.targetConfig.providerVersion + + consumerImageName, err := generateImageName(consumerVersion, dc.targetConfig) + if err != nil { + return fmt.Errorf("failed building docker image: %v", err) + } + err = buildDockerImage(consumerImageName, consumerVersion, dc.targetConfig, false) + if err != nil { + return err + } + dc.images = append(dc.images, consumerImageName) + + if consumerVersion == "" && consumerVersion == providerVersion { + dc.ImageName = consumerImageName + return nil + } + + // build image for provider as it's a different version to be run + providerImageName, err := generateImageName(providerVersion, dc.targetConfig) + if err != nil { + return fmt.Errorf("failed building docker image: %v", err) + } + err = buildDockerImage(providerImageName, providerVersion, dc.targetConfig, false) + if err != nil { + return err + } + dc.images = append(dc.images, providerImageName) + + // build combined image using provider/consumer versions from images built above + combinedImageName := fmt.Sprintf("cosmos-ics-combined:%s_%s", + strings.Split(providerImageName, ":")[1], + strings.Split(consumerImageName, ":")[1]) + //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. + cmd := exec.Command("docker", "build", "-f", "Dockerfile.combined", + "--build-arg", fmt.Sprintf("PROVIDER_IMAGE=%s", providerImageName), + "--build-arg", fmt.Sprintf("CONSUMER_IMAGE=%s", consumerImageName), + "-t", combinedImageName, + ".") + out, err := cmd.CombinedOutput() + if err != nil { + log.Printf("Failed running: %v", cmd) + return fmt.Errorf("error building combined docker image: %v, %s", err, string(out)) + } + dc.images = append(dc.images, combinedImageName) + dc.ImageName = combinedImageName + return err +} + +func (dc *DockerContainer) Delete() error { + for _, img := range dc.images { + //#nosec G204 -- Bypass linter warning for spawning subprocess with variable + cmd := exec.Command("docker", "image", "rm", img) + out, err := cmd.CombinedOutput() + //TODO: ignore errors related to non-existing images + if err != nil { + log.Printf("failed deleting image '%v' (%v): %v", cmd, err, string(out)) + } + } + return nil +} + +// ExecCommand returns the command struct to execute the named program with +// given arguments on the current target (docker container) +func (dc *DockerContainer) ExecCommand(name string, arg ...string) *exec.Cmd { + args := []string{"exec", dc.containerCfg.InstanceName, name} + args = append(args, arg...) + //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. + return exec.Command("docker", args...) +} + +// ExecDetachedCommand returns the command struct to execute the named program with +// given arguments on the current target (docker container) in _detached_ mode +func (dc *DockerContainer) ExecDetachedCommand(name string, arg ...string) *exec.Cmd { + args := []string{"exec", "-d", dc.containerCfg.InstanceName, name} + args = append(args, arg...) + //#nosec G204 -- Bypass linter warning for spawning subprocess with variable + return exec.Command("docker", args...) +} + +// Get path to testnet-script on target for a specific chain type +// Needed for different consumer/provider versions staged in one container +func (dc *DockerContainer) GetTestScriptPath(isConsumer bool, script string) string { + path := "/testnet-scripts" + // in case the provider and consumer version differ the test-scripts are in dedicated directories on the target + // for each of them (see Docker.combined) + if dc.targetConfig.providerVersion != dc.targetConfig.consumerVersion { + if !isConsumer { + fmt.Printf("Using script path for provider version '%s'\n", dc.targetConfig.providerVersion) + path = "/provider/testnet-scripts" + } else { + fmt.Printf("Using script path for consumer version '%s'\n", dc.targetConfig.consumerVersion) + path = "/consumer/testnet-scripts" + } + } + // no combined image (see Dockerfile) + return strings.Join([]string{path, script}, string(os.PathSeparator)) +} + +// Startup the container +func (dc *DockerContainer) Start() error { + fmt.Println("Starting container: ", dc.containerCfg.InstanceName) + // Remove existing containers from previous runs + if err := dc.Stop(); err != nil { + return err + } + + // Run new test container instance with extended privileges. + // Extended privileges are granted to the container here to allow for network namespace manipulation (bringing a node up/down) + // See: https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities + beaconScript := dc.GetTestScriptPath(false, "beacon.sh") + //#nosec G204 -- subprocess launched with potential tainted input (no production code) + cmd := exec.Command("docker", "run", "--name", dc.containerCfg.InstanceName, + "--cap-add=NET_ADMIN", "--privileged", dc.ImageName, + "/bin/bash", beaconScript) + + cmdReader, err := cmd.StdoutPipe() + if err != nil { + log.Fatal(err) + } + cmd.Stderr = cmd.Stdout + + if err := cmd.Start(); err != nil { + log.Fatal(err) + } + scanner := bufio.NewScanner(cmdReader) + + // Wait until container is up + for scanner.Scan() { + out := scanner.Text() + if verbose != nil && *verbose { + fmt.Println("startDocker: " + out) + } + if out == "beacon!!!!!!!!!!" { + return nil + } + } + if err := scanner.Err(); err != nil { + return fmt.Errorf("error bringing up container: %v", err) + } + + err = cmd.Wait() + return fmt.Errorf("starting container exited with error: %v", err) +} + +// Stop will stop the container and remove it +func (dc *DockerContainer) Stop() error { + fmt.Println("Stopping existing containers: ", dc.containerCfg.InstanceName) + //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. + cmd := exec.Command("docker", "stop", dc.containerCfg.InstanceName) + bz, err := cmd.CombinedOutput() + if err != nil && !strings.Contains(string(bz), "No such container") { + return fmt.Errorf("error stopping docker container: %v, %s", err, string(bz)) + } + + //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. + cmd = exec.Command("docker", "rm", dc.containerCfg.InstanceName) + bz, err = cmd.CombinedOutput() + if err != nil && !strings.Contains(string(bz), "No such container") { + return fmt.Errorf("error removing docker container: %v, %s", err, string(bz)) + } + return nil +} diff --git a/tests/e2e/testnet-scripts/start-docker.sh b/tests/e2e/testnet-scripts/start-docker.sh deleted file mode 100755 index a1d46e1f4d..0000000000 --- a/tests/e2e/testnet-scripts/start-docker.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -# If -e is not set then if the build fails, it will use the old container, resulting in a very confusing debugging situation -# Setting -e makes it error out if the build fails -set -eux - -CONTAINER_NAME=$1 -INSTANCE_NAME=$2 -LOCAL_SDK_PATH=${3:-"default"} # Sets this var to default if null or unset -USE_GAIA_PROVIDER=${4:-"false"} # if true, use gaia as provider; if false, use ICS app -USE_GAIA_TAG=${5:-""} # gaia tag to use if using gaia as provider; by default the latest tag is used - -# Remove existing container instance -set +e -docker rm -f "$INSTANCE_NAME" -set -e - -# Delete old sdk directory if it exists -if [ -d "./cosmos-sdk" ]; then - rm -rf ./cosmos-sdk/ -fi - -# Copy sdk directory to working dir if path was specified -if [[ "$LOCAL_SDK_PATH" != "default" ]] -then - cp -n -r "$LOCAL_SDK_PATH" ./cosmos-sdk - printf "\n\nUsing local sdk version from %s\n\n\n" "$LOCAL_SDK_PATH" -else - printf "\n\nUsing default sdk version\n\n\n" -fi - -# Build the Docker container -if [[ "$USE_GAIA_PROVIDER" = "true" ]] -then - printf "\n\nUsing gaia as provider\n\n" - printf "\n\nUsing gaia tag %s\n\n" "$USE_GAIA_TAG" - docker build -f Dockerfile.gaia -t "$CONTAINER_NAME" --build-arg USE_GAIA_TAG="$USE_GAIA_TAG" . -else - printf "\n\nUsing ICS provider app as provider\n\n\n" - docker build -f Dockerfile -t "$CONTAINER_NAME" . -fi - -# Remove copied sdk directory -rm -rf ./cosmos-sdk/ - -# Run new test container instance with extended privileges. -# Extended privileges are granted to the container here to allow for network namespace manipulation (bringing a node up/down) -# See: https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities -docker run --name "$INSTANCE_NAME" --cap-add=NET_ADMIN --privileged "$CONTAINER_NAME" /bin/bash /testnet-scripts/beacon.sh &