From caf22148d03d8a2368a21e377ffe9cebf9a44218 Mon Sep 17 00:00:00 2001 From: Bernd Mueller Date: Mon, 20 Nov 2023 11:21:20 +0100 Subject: [PATCH 01/16] Infra for compatibility tests --- Dockerfile-Combined | 54 +++++++++ Dockerfile-Consumer | 72 ++++++++++++ Dockerfile-Provider | 70 +++++++++++ Makefile | 5 + tests/e2e/actions.go | 37 +++++- tests/e2e/builder.go | 85 ++++++++++++++ tests/e2e/config.go | 8 +- tests/e2e/main.go | 136 ++++++++++++++++++++-- tests/e2e/steps.go | 12 ++ tests/e2e/testnet-scripts/start-docker.sh | 63 ++++++++-- 10 files changed, 517 insertions(+), 25 deletions(-) create mode 100644 Dockerfile-Combined create mode 100644 Dockerfile-Consumer create mode 100644 Dockerfile-Provider create mode 100644 tests/e2e/builder.go diff --git a/Dockerfile-Combined b/Dockerfile-Combined new file mode 100644 index 0000000000..b1eb1c8492 --- /dev/null +++ b/Dockerfile-Combined @@ -0,0 +1,54 @@ +# syntax=docker/dockerfile:1 + +# Consumer image to be used for compatibility tests with provider from workspace +# use docker's build argument --build-arg to overwrite the defaults +# e.g. docker build --build-arg CONSUMER_TAG=v3.1.0 +ARG PROVIDER_VERSION +ARG PROVIDER_IMAGE="cosmos-ics" + +ARG CONSUMER_VERSION +ARG CONSUMER_IMAGE="cosmos-ics" + + +# The image from where the consumer implementation will be used +# Defaults to +FROM --platform=linux/amd64 ${PROVIDER_IMAGE}:${PROVIDER_VERSION} AS provider + + +# The image from where the consumer implementation will be used +# Defaults to +FROM --platform=linux/amd64 ${CONSUMER_IMAGE}:${CONSUMER_VERSION} AS consumer + +# Get Hermes build +FROM otacrew/hermes-ics:evidence-cmd AS hermes-builder + +# Get CometMock +FROM ghcr.io/informalsystems/cometmock:v0.37.x as cometmock-builder + +# Get GoRelayer +FROM ghcr.io/informalsystems/relayer-no-gas-sim:v2.3.0-rc4-no-gas-sim AS gorelayer-builder + +FROM --platform=linux/amd64 fedora:39 +RUN dnf update -y +RUN dnf install -y which iproute iputils procps-ng vim-minimal tmux net-tools htop jq +USER root + +COPY --from=hermes-builder /usr/bin/hermes /usr/local/bin/ +COPY --from=cometmock-builder /usr/local/bin/cometmock /usr/local/bin/cometmock +COPY --from=gorelayer-builder /bin/rly /usr/local/bin/ + +# Copy consumer from specified image +COPY --from=consumer /usr/local/bin/interchain-security-cd /usr/local/bin/interchain-security-cd +COPY --from=consumer /usr/local/bin/interchain-security-cdd /usr/local/bin/interchain-security-cdd +COPY --from=consumer /usr/local/bin/interchain-security-sd /usr/local/bin/interchain-security-sd +COPY --from=consumer /testnet-scripts /consumer/testnet-scripts + +# Copy provider from specified image +COPY --from=provider /usr/local/bin/interchain-security-pd /usr/local/bin/interchain-security-pd +COPY --from=provider /testnet-scripts /provider/testnet-scripts + +# Copy in the shell scripts that run the testnet +ADD ./tests/e2e/testnet-scripts /testnet-scripts + +# Copy in the hermes config +ADD ./tests/e2e/testnet-scripts/hermes-config.toml /root/.hermes/config.toml diff --git a/Dockerfile-Consumer b/Dockerfile-Consumer new file mode 100644 index 0000000000..666db5f893 --- /dev/null +++ b/Dockerfile-Consumer @@ -0,0 +1,72 @@ +# syntax=docker/dockerfile:1 + +# Consumer image to be used for compatibility tests with provider from workspace +# use docker's build argument --build-arg to overwrite the defaults +# e.g. docker build --build-arg CONSUMER_TAG=v3.1.0 +ARG CONSUMER_VERSION +ARG CONSUMER_IMAGE="cosmos-ics" + + +FROM golang:1.20-alpine AS is-builder + +ENV PACKAGES curl make git libc-dev bash gcc linux-headers +RUN apk add --no-cache $PACKAGES + +ENV CGO_ENABLED=0 +ENV GOOS=linux +ENV GOFLAGS="-buildvcs=false" + +# cache go modules - done before the files are copied to allow docker to better cache +COPY go.mod /go.mod +COPY go.sum /go.sum +RUN go mod download + + +# Copy in the repo under test +ADD . /interchain-security +WORKDIR /interchain-security + +# Do not specify version here. It leads to odd replacement behavior +RUN if [ -d "./cosmos-sdk" ]; then go mod edit -replace github.com/cosmos/cosmos-sdk=./cosmos-sdk; fi +RUN go mod tidy + +# Install interchain security binary +RUN make install + + +# The image from where the consumer implementation will be used +# Defaults to +FROM --platform=linux/amd64 ${CONSUMER_IMAGE}:${CONSUMER_VERSION} AS consumer + +# Get Hermes build +FROM otacrew/hermes-ics:evidence-cmd AS hermes-builder + +# Get CometMock +FROM ghcr.io/informalsystems/cometmock:v0.37.x as cometmock-builder + +# Get GoRelayer +FROM ghcr.io/informalsystems/relayer-no-gas-sim:v2.3.0-rc4-no-gas-sim AS gorelayer-builder + +FROM --platform=linux/amd64 fedora:39 +RUN dnf update -y +RUN dnf install -y which iproute iputils procps-ng vim-minimal tmux net-tools htop jq +USER root + +COPY --from=hermes-builder /usr/bin/hermes /usr/local/bin/ +COPY --from=cometmock-builder /usr/local/bin/cometmock /usr/local/bin/cometmock +COPY --from=gorelayer-builder /bin/rly /usr/local/bin/ + +# Copy consumer from specified image +COPY --from=consumer /usr/local/bin/interchain-security-cd /usr/local/bin/interchain-security-cd +COPY --from=consumer /usr/local/bin/interchain-security-cdd /usr/local/bin/interchain-security-cdd +COPY --from=consumer /usr/local/bin/interchain-security-sd /usr/local/bin/interchain-security-sd +COPY --from=consumer /testnet-scripts /consumer/testnet-scripts + +# Copy provider from local build +COPY --from=is-builder /go/bin/interchain-security-pd /usr/local/bin/interchain-security-pd + +# Copy in the shell scripts that run the testnet +ADD ./tests/e2e/testnet-scripts /testnet-scripts + +# Copy in the hermes config +ADD ./tests/e2e/testnet-scripts/hermes-config.toml /root/.hermes/config.toml diff --git a/Dockerfile-Provider b/Dockerfile-Provider new file mode 100644 index 0000000000..962c0bdd0f --- /dev/null +++ b/Dockerfile-Provider @@ -0,0 +1,70 @@ +# syntax=docker/dockerfile:1 + +# Provider image to be used for compatibility tests with consumer from workspace. +# use docker's build argument --build-arg to overwrite the defaults +# e.g. docker build --build-arg CONSUMER_TAG=v3.1.0 +ARG PROVIDER_VERSION="latest" +ARG PROVIDER_IMAGE="cosmos-ics" + +FROM golang:1.20-alpine AS is-builder + +ENV PACKAGES curl make git libc-dev bash gcc linux-headers +RUN apk add --no-cache $PACKAGES + +ENV CGO_ENABLED=0 +ENV GOOS=linux +ENV GOFLAGS="-buildvcs=false" + +# cache go modules - done before the files are copied to allow docker to better cache +COPY go.mod /go.mod +COPY go.sum /go.sum +RUN go mod download + +# Copy in the repo under test +ADD . /interchain-security +WORKDIR /interchain-security + +# Do not specify version here. It leads to odd replacement behavior +RUN if [ -d "./cosmos-sdk" ]; then go mod edit -replace github.com/cosmos/cosmos-sdk=./cosmos-sdk; fi +RUN go mod tidy + +# Install interchain security binary +RUN make install + + +# The image from where the consumer implementation will be used +# Defaults to +FROM --platform=linux/amd64 ${PROVIDER_IMAGE}:${PROVIDER_VERSION} AS provider + +# Get Hermes build +FROM otacrew/hermes-ics:evidence-cmd AS hermes-builder + +# Get CometMock +FROM ghcr.io/informalsystems/cometmock:v0.37.x as cometmock-builder + +# Get GoRelayer +FROM ghcr.io/informalsystems/relayer-no-gas-sim:v2.3.0-rc4-no-gas-sim AS gorelayer-builder + +FROM --platform=linux/amd64 fedora:39 +RUN dnf update -y +RUN dnf install -y which iproute iputils procps-ng vim-minimal tmux net-tools htop jq +USER root + +COPY --from=hermes-builder /usr/bin/hermes /usr/local/bin/ +COPY --from=cometmock-builder /usr/local/bin/cometmock /usr/local/bin/cometmock +COPY --from=gorelayer-builder /bin/rly /usr/local/bin/ + +# Copy provider from specified image +COPY --from=provider /usr/local/bin/interchain-security-pd /usr/local/bin/interchain-security-pd +COPY --from=provider /testnet-scripts /provider/testnet-scripts + +# Copy provider from local build +COPY --from=is-builder /go/bin/interchain-security-cd /usr/local/bin/interchain-security-cd +COPY --from=is-builder /go/bin/interchain-security-cdd /usr/local/bin/interchain-security-cdd +COPY --from=is-builder /go/bin/interchain-security-sd /usr/local/bin/interchain-security-sd + +# Copy in the shell scripts that run the testnet +ADD ./tests/e2e/testnet-scripts /testnet-scripts + +# Copy in the hermes config +ADD ./tests/e2e/testnet-scripts/hermes-config.toml /root/.hermes/config.toml diff --git a/Makefile b/Makefile index dfa42b3e82..1d439e8b32 100644 --- a/Makefile +++ b/Makefile @@ -70,6 +70,11 @@ test-e2e-multi-consumer: test-e2e-parallel: go run ./tests/e2e/... --include-multi-consumer --parallel +# run E2E compatibility tests for v3.x +# Note: e54799020 is the rev for v3.1.x candidate as build is broken for v3.1.0 +test-e2e-compatibility-v3: + go run ./tests/e2e/... -parallel -tc compatibility -pv v3.2.0-rc1 -pv e54799020 -pv v3.0.0 -cv v3.2.0-rc1 -cv e54799020 -cv v3.0.0 + # run full E2E tests in sequence (including multiconsumer) using latest tagged gaia test-gaia-e2e: go run ./tests/e2e/... --include-multi-consumer --use-gaia diff --git a/tests/e2e/actions.go b/tests/e2e/actions.go index 2940eaab8d..0c592bec19 100644 --- a/tests/e2e/actions.go +++ b/tests/e2e/actions.go @@ -6,6 +6,7 @@ import ( "fmt" "log" "math" + "os" "os/exec" "strconv" "strings" @@ -131,9 +132,19 @@ func (tr *TestConfig) startChain( cometmockArg = "false" } + startChainScript := "/testnet-scripts/start-chain.sh" + if tr.providerVersion != "" && chainConfig.BinaryName == "interchain-security-pd" { + log.Printf("Using start-chain script for provider version '%s'", tr.providerVersion) + startChainScript = "/provider/testnet-scripts/start-chain.sh" + } + if tr.consumerVersion != "" && chainConfig.BinaryName != "interchain-security-pd" { + log.Printf("Using start-chain script for consumer version '%s'", tr.consumerVersion) + startChainScript = "/consumer/testnet-scripts/start-chain.sh" + } + //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "/bin/bash", - "/testnet-scripts/start-chain.sh", chainConfig.BinaryName, string(vals), + startChainScript, chainConfig.BinaryName, string(vals), string(chainConfig.ChainId), chainConfig.IpPrefix, genesisChanges, fmt.Sprint(action.IsConsumer), // override config/config.toml for each node on chain @@ -503,6 +514,30 @@ func (tr *TestConfig) startConsumerChain( log.Fatal(err, "\n", string(bz)) } + // only needed when consumer is running v3.2.x and later + if tr.transformGenesis { + log.Printf("@@@@ Transforming consumer genesis for a newer version: %s\n", tr.consumerVersion) + log.Printf("Original ccv genesis: %s\n", string(bz)) + + file, err := os.Create("consumer_genesis.json") + if err != nil { + panic(fmt.Sprintf("failed writing ccv consumer file : %v", err)) + } + os.WriteFile(file.Name(), bz, 0644) + cmd := exec.Command("docker", "cp", file.Name(), fmt.Sprintf("%s:/tmp/%s", tr.containerConfig.InstanceName, file.Name())) + bz, err = cmd.CombinedOutput() + if err != nil { + log.Fatal(err, "\n", string(bz)) + } + cmd = exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[action.ConsumerChain].BinaryName, + "genesis", "transform", fmt.Sprintf("/tmp/%s", file.Name())) + bz, err = cmd.CombinedOutput() + if err != nil { + log.Fatal(err, "CCV consumer genesis transformation failed: %s", string(bz)) + } + log.Printf("Transformed genesis is: %s", string(bz)) + } + consumerGenesis := ".app_state.ccvconsumer = " + string(bz) consumerGenesisChanges := tr.chainConfigs[action.ConsumerChain].GenesisChanges if consumerGenesisChanges != "" { diff --git a/tests/e2e/builder.go b/tests/e2e/builder.go new file mode 100644 index 0000000000..32de3d735c --- /dev/null +++ b/tests/e2e/builder.go @@ -0,0 +1,85 @@ +package main + +import ( + "bytes" + "fmt" + "log" + "os" + "os/exec" + "path" +) + +func setupWorkspace(revision string, tmpDir string) error { + log.Printf("Setting up worktree in '%s'", tmpDir) + cmd := exec.Command("git", "worktree", "add", + "--checkout", tmpDir, revision) + var errbuf bytes.Buffer + cmd.Stderr = &errbuf + log.Printf("Running: %s", cmd.String()) + if err := cmd.Start(); err != nil { + return err + } + if err := cmd.Wait(); err != nil { + log.Printf("Error creating worktree (%v): %s", err, errbuf.String()) + return err + } + return nil +} + +func cleanupWorkspace(workSpacePath string) error { + cmd := exec.Command("git", "worktree", "remove", workSpacePath) + cmd.Stderr = cmd.Stdout + if err := cmd.Start(); err != nil { + log.Printf("Failed removing git worktree '%s' used for docker image creation", workSpacePath) + return err + } + return cmd.Wait() +} + +// Check if docker is running +func dockerIsUp() bool { + cmd := exec.Command("docker", "info") + var errbuf bytes.Buffer + cmd.Stderr = &errbuf + if err := cmd.Run(); err != nil { + log.Printf("Docker engine is not running (%v): %s", err, errbuf.String()) + return false + } + return true +} + +// Build docker image of ICS for a given revision +func buildDockerImage(imageName string, revision string, tmpDir string) error { + log.Printf("Building ICS image for version %s", revision) + + if !dockerIsUp() { + return fmt.Errorf("docker engine is not running") + } + workSpace := path.Join(tmpDir, revision) + if err := setupWorkspace(revision, workSpace); err != nil { + return err + } + defer cleanupWorkspace(workSpace) + + _, err := os.Stat(workSpace) + if err != nil { + log.Fatalf("Worktree creation for image build failed: %v", err) + } + + log.Printf("Building docker image") + // TODO: TBD if we should use option "--no-cache" here + cmd := exec.Command("docker", "build", "-t", + fmt.Sprintf("cosmos-ics:%s", revision), "-f", "./Dockerfile", "./") + cmd.Dir = workSpace + + if err := cmd.Start(); err != nil { + log.Printf("Failed building docker image '%s': %v", revision, err) + return err + } + if err := cmd.Wait(); err != nil { + out, _ := cmd.CombinedOutput() + log.Printf("Error building image (%v): %s", err, out) + return err + } + return nil +} diff --git a/tests/e2e/config.go b/tests/e2e/config.go index 311ee67c89..aa9c98f8f8 100644 --- a/tests/e2e/config.go +++ b/tests/e2e/config.go @@ -81,7 +81,11 @@ type TestConfig struct { // Used with CometMock. The time by which chains have been advanced. Used to keep chains in sync: when a new chain is started, advance its time by this value to keep chains in sync. timeOffset time.Duration - name string + // consumer version the provider should be tested against + consumerVersion string + providerVersion string + transformGenesis bool + name string } // Initialize initializes the TestConfig instance by setting the runningChains field to an empty map. @@ -483,7 +487,7 @@ func ConsumerMisbehaviourTestConfig() TestConfig { return tc } -func (s *TestConfig) SetDockerConfig(localSdkPath string, useGaia bool, gaiaTag string) { +func (s *TestConfig) SetDockerConfig(localSdkPath string, useGaia bool, gaiaTag string, consumerVersion string, providerVersion string) { if localSdkPath != "" { fmt.Println("USING LOCAL SDK", localSdkPath) } diff --git a/tests/e2e/main.go b/tests/e2e/main.go index 13e3fae1e7..2c2fc1c86b 100644 --- a/tests/e2e/main.go +++ b/tests/e2e/main.go @@ -5,6 +5,7 @@ import ( "flag" "fmt" "log" + "os" "os/exec" "reflect" "strconv" @@ -33,6 +34,23 @@ func (t *TestSet) String() string { return fmt.Sprint(*t) } +type VersionSet []string + +func (c *VersionSet) Set(value string) error { + // Check and skip duplicates + for _, v := range *c { + if v == value { + return nil + } + } + *c = append(*c, value) + return nil +} + +func (t *VersionSet) String() string { + return fmt.Sprint(*t) +} + var ( verbose = flag.Bool("verbose", false, "turn verbose logging on/off") includeMultiConsumer = flag.Bool("include-multi-consumer", false, "include multiconsumer tests in run") @@ -48,6 +66,14 @@ var ( gaiaTag = flag.String("gaia-tag", "", "gaia tag to use - default is latest") ) +var ( + //useConsumerVersion = flag.String("consumer-version", "", "ICS tag to specify the consumer version to test the provider against") + //useProviderVersion = flag.String("provider-version", "", "ICS tag to specify the provider version to test the consumer against") + consumerVersions VersionSet + providerVersions VersionSet + transformGenesis = flag.Bool("transform-genesis", false, "do consumer genesis transformation for newer clients. Needed when provider chain is on an older version") +) + var ( selectedTests TestSet @@ -68,6 +94,12 @@ var ( var selectedTestfiles TestSet var stepChoices = map[string]StepChoice{ + "compatibility": { + name: "compatibility", + steps: compatibilitySteps, + description: `Minimal set of test steps to perform compatibility tests`, + testConfig: DefaultTestConfig(), + }, "happy-path-short": { name: "happy-path-short", steps: shortHappyPathSteps, @@ -208,6 +240,10 @@ func parseArguments() (err error) { flag.Var(&selectedTestfiles, "test-file", getTestFileUsageString()) + + flag.Var(&consumerVersions, "cv", "Consumer version") + flag.Var(&providerVersions, "pv", "Provider version") + flag.Parse() // Enforce go-relayer in case of cometmock as hermes is not yet supported @@ -287,6 +323,50 @@ func getTestCases(selectedPredefinedTests, selectedTestFiles TestSet) (tests []t return tests } +// Create multiversion test cases +// Based on the original set of test cases a multitude of test cases +// for the provided consumer and provider versions are generated. +func multiVersionConfig(testCases []testStepsWithConfig, providerVersions, consumerVerions VersionSet) []testStepsWithConfig { + var newTCs []testStepsWithConfig + if len(consumerVerions) == 0 && len(providerVersions) == 0 { + return testCases + } + + if len(consumerVersions) == 0 { + consumerVerions = append(consumerVerions, "") + } + if len(providerVersions) == 0 { + providerVersions = append(providerVersions, "") + } + + for _, provider := range providerVersions { + for _, consumer := range consumerVerions { + if consumer == provider { + log.Printf("Skipping same version '%s' for provider & consumer", consumer) + continue + } + log.Printf("Configuring test cases for: provider version %s, consumer version %s", provider, consumer) + for _, tc := range testCases { + tc.testRun.consumerVersion = consumer + tc.testRun.providerVersion = provider + tc.testRun.transformGenesis = *transformGenesis + suffix := "" + if consumer != "" { + suffix = fmt.Sprintf("_cons-%s", consumer) + } + if provider != "" { + suffix += fmt.Sprintf("_prov-%s", provider) + } + tc.testRun.containerConfig.ContainerName += suffix + tc.testRun.containerConfig.InstanceName += suffix + tc.testRun.name += suffix + newTCs = append(newTCs, tc) + } + } + } + return newTCs +} + // runs E2E tests // all docker containers are built sequentially to avoid race conditions when using local cosmos-sdk // after building docker containers, all tests are run in parallel using their respective docker containers @@ -297,6 +377,7 @@ func main() { } testCases := getTestCases(selectedTests, selectedTestfiles) + testCases = multiVersionConfig(testCases, providerVersions, consumerVersions) start := time.Now() err := executeTests(testCases) @@ -309,7 +390,8 @@ func main() { // Run sets up docker container and executes the steps in the test run. // Docker containers are torn down after the test run is complete. func (tr *TestConfig) Run(steps []Step, localSdkPath string, useGaia bool, gaiaTag string) { - tr.SetDockerConfig(localSdkPath, useGaia, gaiaTag) + + tr.SetDockerConfig(localSdkPath, useGaia, gaiaTag, tr.consumerVersion, tr.providerVersion) tr.SetCometMockConfig(*useCometmock) tr.SetRelayerConfig(*useGorelayer) @@ -436,16 +518,50 @@ func (tr *TestConfig) executeSteps(steps []Step) { fmt.Printf("=============== finished %s tests in %v ===============\n", tr.name, time.Since(start)) } +func (tr *TestConfig) buildDockerImages() { + fmt.Printf("=============== building %s images ===============\n", tr.name) + tmpDir, err := os.MkdirTemp(os.TempDir(), "e2eWorkTree") + if err != nil { + log.Fatalf("Error createing temp directory for docker creation") + } + + // Build ICS image of a given version + icsVersions := []string{} + if tr.consumerVersion != "" { + icsVersions = append(icsVersions, tr.consumerVersion) + } + if tr.providerVersion != "" && tr.consumerVersion != tr.providerVersion { + icsVersions = append(icsVersions, tr.providerVersion) + } + for _, icsVersion := range icsVersions { + imageName := fmt.Sprintf("cosmos-ics:%s", icsVersion) + err := buildDockerImage(imageName, icsVersion, tmpDir) + if err != nil { + log.Fatalf("Error building docker image '%s':%v", icsVersion, err) + } + } +} + func (tr *TestConfig) startDocker() { + tr.buildDockerImages() fmt.Printf("=============== building %s testRun ===============\n", tr.name) + + options := []string{} + localSdk := tr.localSdkPath - if localSdk == "" { - localSdk = "default" + if localSdk != "" { + options = append(options, fmt.Sprintf("-s %s", tr.localSdkPath)) } - useGaia := "false" - gaiaTag := "" + + if tr.consumerVersion != "" { + options = append(options, fmt.Sprintf("-c %s", tr.consumerVersion)) + } + + if tr.providerVersion != "" { + options = append(options, fmt.Sprintf("-p %s", tr.providerVersion)) + } + if tr.useGaia { - useGaia = "true" if tr.gaiaTag != "" { majVersion, err := strconv.Atoi(tr.gaiaTag[1:strings.Index(tr.gaiaTag, ".")]) if err != nil { @@ -454,16 +570,14 @@ func (tr *TestConfig) startDocker() { if majVersion < 9 { panic(fmt.Sprintf("gaia version %s is not supported - supporting only v9.x.x and newer", tr.gaiaTag)) } - gaiaTag = tr.gaiaTag + options = append(options, fmt.Sprintf("-g %s", tr.gaiaTag)) } } scriptStr := fmt.Sprintf( - "tests/e2e/testnet-scripts/start-docker.sh %s %s %s %s %s", + "tests/e2e/testnet-scripts/start-docker.sh %s %s %s", + strings.Join(options, " "), tr.containerConfig.ContainerName, tr.containerConfig.InstanceName, - localSdk, - useGaia, - gaiaTag, ) //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. diff --git a/tests/e2e/steps.go b/tests/e2e/steps.go index 783d976a3a..10266e5568 100644 --- a/tests/e2e/steps.go +++ b/tests/e2e/steps.go @@ -13,6 +13,18 @@ func concatSteps(steps ...[]Step) []Step { return concat } +// Limited amount of sanity tests to use for compatibility testing +var compatibilitySteps = concatSteps( + stepsStartChains([]string{"consu"}, false), + stepsDelegate("consu"), + stepsUnbond("consu"), + stepsRedelegateShort("consu"), + stepsDowntime("consu"), + stepsStartRelayer(), + stepsConsumerRemovalPropNotPassing("consu", 2), // submit removal prop but vote no on it - chain should stay + stepsStopChain("consu", 3), // stop chain +) + var happyPathSteps = concatSteps( stepsStartChains([]string{"consu"}, false), stepsDelegate("consu"), diff --git a/tests/e2e/testnet-scripts/start-docker.sh b/tests/e2e/testnet-scripts/start-docker.sh index a1d46e1f4d..4ed0f0a70c 100755 --- a/tests/e2e/testnet-scripts/start-docker.sh +++ b/tests/e2e/testnet-scripts/start-docker.sh @@ -2,26 +2,57 @@ # If -e is not set then if the build fails, it will use the old container, resulting in a very confusing debugging situation # Setting -e makes it error out if the build fails -set -eux +set -eux +usage() { + echo """ +Usage: $0 [-c ] [-g ] [-s ] [-h] container-name instance-name + [-c consumer-version] : the consumer version to be used in the container. + Mutial exclusive to -p + [-p provider-version] : the provider version to be used in the container. + Mutial exclusive to -c + [-g gaia-tag] : use gaia as provider with specified version + [-s SDK-path] : use custom SDK + [-h] : print this help +""" +} + +## Process the optional arguments if any +while getopts ":c:p:g:s:h" flag + do + case "${flag}" in + c) CONSUMER_VERSION=${OPTARG};; + p) PROVIDER_VERSION=${OPTARG};; + g) USE_GAIA_TAG=${OPTARG};; + s) LOCAL_SDK_PATH=${OPTARG};; + h) SHOW_HELP=1;; + *) usage;; + esac + done + +if [ ${SHOW_HELP+x} ]; then + usage + exit 0 +fi + +shift $((OPTIND - 1)) + +# Set positional arguments CONTAINER_NAME=$1 INSTANCE_NAME=$2 -LOCAL_SDK_PATH=${3:-"default"} # Sets this var to default if null or unset -USE_GAIA_PROVIDER=${4:-"false"} # if true, use gaia as provider; if false, use ICS app -USE_GAIA_TAG=${5:-""} # gaia tag to use if using gaia as provider; by default the latest tag is used # Remove existing container instance set +e docker rm -f "$INSTANCE_NAME" set -e -# Delete old sdk directory if it exists +# Delete old sdk directory if it exists if [ -d "./cosmos-sdk" ]; then rm -rf ./cosmos-sdk/ -fi +fi # Copy sdk directory to working dir if path was specified -if [[ "$LOCAL_SDK_PATH" != "default" ]] +if [[ ${LOCAL_SDK_PATH+x} ]] then cp -n -r "$LOCAL_SDK_PATH" ./cosmos-sdk printf "\n\nUsing local sdk version from %s\n\n\n" "$LOCAL_SDK_PATH" @@ -30,20 +61,30 @@ else fi # Build the Docker container -if [[ "$USE_GAIA_PROVIDER" = "true" ]] +if [[ ${USE_GAIA_TAG+x} ]] then printf "\n\nUsing gaia as provider\n\n" printf "\n\nUsing gaia tag %s\n\n" "$USE_GAIA_TAG" - docker build -f Dockerfile.gaia -t "$CONTAINER_NAME" --build-arg USE_GAIA_TAG="$USE_GAIA_TAG" . + docker build -f Dockerfile.gaia -t "$CONTAINER_NAME" --build-arg USE_GAIA_TAG="$USE_GAIA_TAG" ./ +elif [ ${CONSUMER_VERSION+x} ] && [ ${PROVIDER_VERSION+x} ]; then + printf "\n\nUsing ICS consumer app from image version ${CONSUMER_VERSION} and provider from image version ${PROVIDER_VERSION}" + docker build -f Dockerfile-Combined --build-arg PROVIDER_VERSION="${PROVIDER_VERSION}" --build-arg CONSUMER_VERSION="${CONSUMER_VERSION}" -t "$CONTAINER_NAME" ./ + +elif [ ${CONSUMER_VERSION+x} ]; then + printf "\n\nUsing ICS consumer app from image version ${CONSUMER_VERSION}" + docker build -f Dockerfile-Consumer --build-arg CONSUMER_VERSION="${CONSUMER_VERSION}" -t "$CONTAINER_NAME" ./ +elif [ ${PROVIDER_VERSION+x} ]; then + printf "\n\nUsing ICS provider app from image version ${PROVIDER_VERSION}" + docker build -f Dockerfile-Provider --build-arg PROVIDER_VERSION="${PROVIDER_VERSION}" -t "$CONTAINER_NAME" ./ else printf "\n\nUsing ICS provider app as provider\n\n\n" - docker build -f Dockerfile -t "$CONTAINER_NAME" . + docker build -f Dockerfile -t "$CONTAINER_NAME" ./ fi # Remove copied sdk directory rm -rf ./cosmos-sdk/ # Run new test container instance with extended privileges. -# Extended privileges are granted to the container here to allow for network namespace manipulation (bringing a node up/down) +# Extended privileges are granted to the container here to allow for network namespace manipulation (bringing a node up/down) # See: https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities docker run --name "$INSTANCE_NAME" --cap-add=NET_ADMIN --privileged "$CONTAINER_NAME" /bin/bash /testnet-scripts/beacon.sh & From 5ad3b22aa28f19484e3d49edb562b556bbe08d31 Mon Sep 17 00:00:00 2001 From: Bernd Mueller Date: Tue, 5 Dec 2023 18:33:48 +0100 Subject: [PATCH 02/16] Dockerfile fix for hermes --- Dockerfile | 2 +- Dockerfile-Combined | 2 +- Dockerfile-Consumer | 2 +- Dockerfile-Provider | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 665aa3d84f..12974fa634 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,7 +29,7 @@ RUN go mod tidy RUN make install # Get Hermes build -FROM otacrew/hermes-ics:evidence-cmd AS hermes-builder +FROM --platform=linux/amd64 otacrew/hermes-ics:evidence-cmd AS hermes-builder # Get CometMock FROM ghcr.io/informalsystems/cometmock:v0.37.x as cometmock-builder diff --git a/Dockerfile-Combined b/Dockerfile-Combined index b1eb1c8492..360b12401c 100644 --- a/Dockerfile-Combined +++ b/Dockerfile-Combined @@ -20,7 +20,7 @@ FROM --platform=linux/amd64 ${PROVIDER_IMAGE}:${PROVIDER_VERSION} AS provider FROM --platform=linux/amd64 ${CONSUMER_IMAGE}:${CONSUMER_VERSION} AS consumer # Get Hermes build -FROM otacrew/hermes-ics:evidence-cmd AS hermes-builder +FROM --platform=linux/amd64 otacrew/hermes-ics:evidence-cmd AS hermes-builder # Get CometMock FROM ghcr.io/informalsystems/cometmock:v0.37.x as cometmock-builder diff --git a/Dockerfile-Consumer b/Dockerfile-Consumer index 666db5f893..19de7106d1 100644 --- a/Dockerfile-Consumer +++ b/Dockerfile-Consumer @@ -39,7 +39,7 @@ RUN make install FROM --platform=linux/amd64 ${CONSUMER_IMAGE}:${CONSUMER_VERSION} AS consumer # Get Hermes build -FROM otacrew/hermes-ics:evidence-cmd AS hermes-builder +FROM --platform=linux/amd64 otacrew/hermes-ics:evidence-cmd AS hermes-builder # Get CometMock FROM ghcr.io/informalsystems/cometmock:v0.37.x as cometmock-builder diff --git a/Dockerfile-Provider b/Dockerfile-Provider index 962c0bdd0f..ba231f35dc 100644 --- a/Dockerfile-Provider +++ b/Dockerfile-Provider @@ -37,7 +37,7 @@ RUN make install FROM --platform=linux/amd64 ${PROVIDER_IMAGE}:${PROVIDER_VERSION} AS provider # Get Hermes build -FROM otacrew/hermes-ics:evidence-cmd AS hermes-builder +FROM --platform=linux/amd64 otacrew/hermes-ics:evidence-cmd AS hermes-builder # Get CometMock FROM ghcr.io/informalsystems/cometmock:v0.37.x as cometmock-builder From 7b9bbb892352253d86a7237d10214f81a33bbe1a Mon Sep 17 00:00:00 2001 From: Bernd Mueller Date: Tue, 5 Dec 2023 19:07:36 +0100 Subject: [PATCH 03/16] cleanup fix gosec + refactor --- Dockerfile-Combined | 14 +- Makefile | 3 +- tests/e2e/actions.go | 111 ++++--- tests/e2e/builder.go | 96 ++++-- tests/e2e/compatibility_tests.go | 20 ++ tests/e2e/config.go | 24 +- tests/e2e/main.go | 377 +++++----------------- tests/e2e/steps.go | 12 - tests/e2e/test_driver.go | 161 +++++++++ tests/e2e/test_runner.go | 52 +++ tests/e2e/test_target.go | 210 ++++++++++++ tests/e2e/testnet-scripts/start-docker.sh | 10 +- 12 files changed, 673 insertions(+), 417 deletions(-) create mode 100644 tests/e2e/compatibility_tests.go create mode 100644 tests/e2e/test_driver.go create mode 100644 tests/e2e/test_runner.go create mode 100644 tests/e2e/test_target.go diff --git a/Dockerfile-Combined b/Dockerfile-Combined index 360b12401c..c129b86cfc 100644 --- a/Dockerfile-Combined +++ b/Dockerfile-Combined @@ -3,21 +3,17 @@ # Consumer image to be used for compatibility tests with provider from workspace # use docker's build argument --build-arg to overwrite the defaults # e.g. docker build --build-arg CONSUMER_TAG=v3.1.0 -ARG PROVIDER_VERSION -ARG PROVIDER_IMAGE="cosmos-ics" - -ARG CONSUMER_VERSION -ARG CONSUMER_IMAGE="cosmos-ics" - +ARG PROVIDER_IMAGE +ARG CONSUMER_IMAGE # The image from where the consumer implementation will be used # Defaults to -FROM --platform=linux/amd64 ${PROVIDER_IMAGE}:${PROVIDER_VERSION} AS provider +FROM --platform=linux/amd64 ${PROVIDER_IMAGE} AS provider # The image from where the consumer implementation will be used # Defaults to -FROM --platform=linux/amd64 ${CONSUMER_IMAGE}:${CONSUMER_VERSION} AS consumer +FROM --platform=linux/amd64 ${CONSUMER_IMAGE} AS consumer # Get Hermes build FROM --platform=linux/amd64 otacrew/hermes-ics:evidence-cmd AS hermes-builder @@ -28,6 +24,7 @@ FROM ghcr.io/informalsystems/cometmock:v0.37.x as cometmock-builder # Get GoRelayer FROM ghcr.io/informalsystems/relayer-no-gas-sim:v2.3.0-rc4-no-gas-sim AS gorelayer-builder + FROM --platform=linux/amd64 fedora:39 RUN dnf update -y RUN dnf install -y which iproute iputils procps-ng vim-minimal tmux net-tools htop jq @@ -43,6 +40,7 @@ COPY --from=consumer /usr/local/bin/interchain-security-cdd /usr/local/bin/inter COPY --from=consumer /usr/local/bin/interchain-security-sd /usr/local/bin/interchain-security-sd COPY --from=consumer /testnet-scripts /consumer/testnet-scripts + # Copy provider from specified image COPY --from=provider /usr/local/bin/interchain-security-pd /usr/local/bin/interchain-security-pd COPY --from=provider /testnet-scripts /provider/testnet-scripts diff --git a/Makefile b/Makefile index 1d439e8b32..fc460454ce 100644 --- a/Makefile +++ b/Makefile @@ -71,9 +71,8 @@ test-e2e-parallel: go run ./tests/e2e/... --include-multi-consumer --parallel # run E2E compatibility tests for v3.x -# Note: e54799020 is the rev for v3.1.x candidate as build is broken for v3.1.0 test-e2e-compatibility-v3: - go run ./tests/e2e/... -parallel -tc compatibility -pv v3.2.0-rc1 -pv e54799020 -pv v3.0.0 -cv v3.2.0-rc1 -cv e54799020 -cv v3.0.0 + go run ./tests/e2e/... -parallel -tc compatibility -pv v3.2.0-rc1 -pv v3.1.0 -pv v3.0.0 -cv v3.2.0-rc1 -cv v3.1.0 -cv v3.0.0 # run full E2E tests in sequence (including multiconsumer) using latest tagged gaia test-gaia-e2e: diff --git a/tests/e2e/actions.go b/tests/e2e/actions.go index 0c592bec19..009419053d 100644 --- a/tests/e2e/actions.go +++ b/tests/e2e/actions.go @@ -77,6 +77,7 @@ type StartChainValidator struct { func (tr *TestConfig) startChain( action StartChainAction, + target ExecutionTarget, verbose bool, ) { chainConfig := tr.chainConfigs[action.Chain] @@ -132,18 +133,8 @@ func (tr *TestConfig) startChain( cometmockArg = "false" } - startChainScript := "/testnet-scripts/start-chain.sh" - if tr.providerVersion != "" && chainConfig.BinaryName == "interchain-security-pd" { - log.Printf("Using start-chain script for provider version '%s'", tr.providerVersion) - startChainScript = "/provider/testnet-scripts/start-chain.sh" - } - if tr.consumerVersion != "" && chainConfig.BinaryName != "interchain-security-pd" { - log.Printf("Using start-chain script for consumer version '%s'", tr.consumerVersion) - startChainScript = "/consumer/testnet-scripts/start-chain.sh" - } - - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "/bin/bash", + startChainScript := target.GetStartChainScript(action.IsConsumer) + cmd := target.ExecCommand("/bin/bash", startChainScript, chainConfig.BinaryName, string(vals), string(chainConfig.ChainId), chainConfig.IpPrefix, genesisChanges, fmt.Sprint(action.IsConsumer), @@ -208,7 +199,7 @@ func (tr TestConfig) submitTextProposal( verbose bool, ) { // TEXT PROPOSAL - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. + //#nosec G204 -- Bypass linter warning for spa wning subprocess with cmd arguments. bz, err := exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[action.Chain].BinaryName, "tx", "gov", "submit-legacy-proposal", `--title`, action.Title, @@ -491,54 +482,80 @@ type StartConsumerChainAction struct { GenesisChanges string } -func (tr *TestConfig) startConsumerChain( - action StartConsumerChainAction, - verbose bool, -) { +// Transform consumer genesis content from older version +func (tr *TestConfig) transformConsumerGenesis(consumerChain ChainID, genesis []byte) []byte { + log.Print("Transforming consumer genesis") + log.Printf("Original ccv genesis: %s\n", string(genesis)) + + fileName := "consumer_genesis.json" + file, err := os.CreateTemp("", fileName) + if err != nil { + panic(fmt.Sprintf("failed writing ccv consumer file : %v", err)) + } + defer file.Close() + err = os.WriteFile(file.Name(), genesis, 0644) + if err != nil { + log.Fatalf("Failed writing consumer genesis to file: %v", err) + } + + containerInstance := tr.containerConfig.InstanceName + targetFile := fmt.Sprintf("/tmp/%s", fileName) + sourceFile := file.Name() //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[action.ProviderChain].BinaryName, + cmd := exec.Command("docker", "cp", sourceFile, + fmt.Sprintf("%s:%s", containerInstance, targetFile)) + genesis, err = cmd.CombinedOutput() + if err != nil { + log.Fatal(err, "\n", string(genesis)) + } + + consumerBinaryName := tr.chainConfigs[consumerChain].BinaryName + //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. + cmd = exec.Command("docker", "exec", containerInstance, consumerBinaryName, + "genesis", "transform", targetFile) + result, err := cmd.CombinedOutput() + if err != nil { + log.Fatal(err, "CCV consumer genesis transformation failed: %s", string(result)) + } + log.Printf("Transformed genesis is: %s", string(result)) + return result +} + +// Get consumer genesis from provider +func (tr *TestConfig) getConsumerGenesis(providerChain, consumerChain ChainID) string { + log.Print("Exporting consumer genesis from provider") + containerInstance := tr.containerConfig.InstanceName + providerBinaryName := tr.chainConfigs[providerChain].BinaryName + + //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. + cmd := exec.Command("docker", "exec", containerInstance, providerBinaryName, "query", "provider", "consumer-genesis", - string(tr.chainConfigs[action.ConsumerChain].ChainId), + string(tr.chainConfigs[consumerChain].ChainId), - `--node`, tr.getQueryNode(action.ProviderChain), + `--node`, tr.getQueryNode(providerChain), `-o`, `json`, ) - if verbose { - log.Println("startConsumerChain cmd: ", cmd.String()) - } - bz, err := cmd.CombinedOutput() if err != nil { log.Fatal(err, "\n", string(bz)) } - // only needed when consumer is running v3.2.x and later + // only needed when consumer is running v3.3.x and later if tr.transformGenesis { - log.Printf("@@@@ Transforming consumer genesis for a newer version: %s\n", tr.consumerVersion) - log.Printf("Original ccv genesis: %s\n", string(bz)) - - file, err := os.Create("consumer_genesis.json") - if err != nil { - panic(fmt.Sprintf("failed writing ccv consumer file : %v", err)) - } - os.WriteFile(file.Name(), bz, 0644) - cmd := exec.Command("docker", "cp", file.Name(), fmt.Sprintf("%s:/tmp/%s", tr.containerConfig.InstanceName, file.Name())) - bz, err = cmd.CombinedOutput() - if err != nil { - log.Fatal(err, "\n", string(bz)) - } - cmd = exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[action.ConsumerChain].BinaryName, - "genesis", "transform", fmt.Sprintf("/tmp/%s", file.Name())) - bz, err = cmd.CombinedOutput() - if err != nil { - log.Fatal(err, "CCV consumer genesis transformation failed: %s", string(bz)) - } - log.Printf("Transformed genesis is: %s", string(bz)) + return string(tr.transformConsumerGenesis(consumerChain, bz)) } + return string(bz) +} - consumerGenesis := ".app_state.ccvconsumer = " + string(bz) +func (tr *TestConfig) startConsumerChain( + action StartConsumerChainAction, + target ExecutionTarget, + verbose bool, +) { + log.Printf("Starting consumer chain %s", action.ConsumerChain) + consumerGenesis := ".app_state.ccvconsumer = " + tr.getConsumerGenesis(action.ProviderChain, action.ConsumerChain) consumerGenesisChanges := tr.chainConfigs[action.ConsumerChain].GenesisChanges if consumerGenesisChanges != "" { consumerGenesis = consumerGenesis + " | " + consumerGenesisChanges + " | " + action.GenesisChanges @@ -549,7 +566,7 @@ func (tr *TestConfig) startConsumerChain( Validators: action.Validators, GenesisChanges: consumerGenesis, IsConsumer: true, - }, verbose) + }, target, verbose) } type ChangeoverChainAction struct { diff --git a/tests/e2e/builder.go b/tests/e2e/builder.go index 32de3d735c..3f0162b189 100644 --- a/tests/e2e/builder.go +++ b/tests/e2e/builder.go @@ -6,27 +6,36 @@ import ( "log" "os" "os/exec" - "path" + "strings" ) -func setupWorkspace(revision string, tmpDir string) error { - log.Printf("Setting up worktree in '%s'", tmpDir) +// setupWorkSpace checks out given revision in a tmp directory +// and returns path where the workspace is located +func setupWorkSpace(revision string) (string, error) { + workSpace, err := os.MkdirTemp(os.TempDir(), "e2eWorkTree_") + if err != nil { + return "", fmt.Errorf("error creating temp directory %v", err) + } + + log.Printf("Setting up worktree in '%s'", workSpace) + cmd := exec.Command("git", "worktree", "add", - "--checkout", tmpDir, revision) + "--checkout", workSpace, revision) var errbuf bytes.Buffer cmd.Stderr = &errbuf log.Printf("Running: %s", cmd.String()) if err := cmd.Start(); err != nil { - return err + return "", err } if err := cmd.Wait(); err != nil { log.Printf("Error creating worktree (%v): %s", err, errbuf.String()) - return err + return "", err } - return nil + return workSpace, nil } -func cleanupWorkspace(workSpacePath string) error { +// cleanupWorkSpace removes the created worktree +func cleanupWorkSpace(workSpacePath string) error { cmd := exec.Command("git", "worktree", "remove", workSpacePath) cmd.Stderr = cmd.Stdout if err := cmd.Start(); err != nil { @@ -49,37 +58,64 @@ func dockerIsUp() bool { } // Build docker image of ICS for a given revision -func buildDockerImage(imageName string, revision string, tmpDir string) error { - log.Printf("Building ICS image for version %s", revision) - +func buildDockerImage(imageName, revision string, targetCfg TargetConfig) error { + log.Printf("Building ICS %s image %s", revision, imageName) if !dockerIsUp() { return fmt.Errorf("docker engine is not running") } - workSpace := path.Join(tmpDir, revision) - if err := setupWorkspace(revision, workSpace); err != nil { - return err + + workSpace := "./" + var err error = nil + // if a revision is provided the related version will be staged + // on a separate worktree (note: revision should _not_ be checked out already elsewhere). + // which will be deleted after image creation + if revision != "" { + workSpace, err = setupWorkSpace(revision) + if err != nil { + return err + } + defer cleanupWorkSpace(workSpace) } - defer cleanupWorkspace(workSpace) - _, err := os.Stat(workSpace) + // Check if workspace exists + _, err = os.Stat(workSpace) if err != nil { - log.Fatalf("Worktree creation for image build failed: %v", err) + log.Fatalf("image build failed. invalid workspace: %v", err) } - log.Printf("Building docker image") - // TODO: TBD if we should use option "--no-cache" here - cmd := exec.Command("docker", "build", "-t", - fmt.Sprintf("cosmos-ics:%s", revision), "-f", "./Dockerfile", "./") - cmd.Dir = workSpace + // Use custom SDK setup if required + sdkPath := strings.Join([]string{workSpace, "cosmos-sdk"}, string(os.PathSeparator)) + err = os.RemoveAll(sdkPath) //delete old SDK directory + if err != nil { + return fmt.Errorf("error deleting SDK directory from workspace: %v", err) + } + if targetCfg.localSdkPath != "" { + fmt.Printf("Using local SDK version from %s\n", targetCfg.localSdkPath) + cmd := exec.Command("cp", "-n", "-r", targetCfg.localSdkPath, sdkPath) + out, err := cmd.CombinedOutput() + if err != nil { + log.Printf("Error running command %v: %s", cmd, string(out)) + return fmt.Errorf("error setting up local SDK: %v, %s", err, string(out)) + } + } else { + fmt.Println("Using default SDK version") + } - if err := cmd.Start(); err != nil { - log.Printf("Failed building docker image '%s': %v", revision, err) - return err + dockerFile := "Dockerfile" + args := []string{"build", "-t", imageName} + if targetCfg.useGaia && targetCfg.gaiaTag != "" { + dockerFile = "Dockerfile.gaia" + args = append(args, "--build-arg", fmt.Sprintf("USE_GAIA_TAG=%s", targetCfg.gaiaTag)) } - if err := cmd.Wait(); err != nil { - out, _ := cmd.CombinedOutput() - log.Printf("Error building image (%v): %s", err, out) - return err + args = append(args, "-f", dockerFile, "./") + + //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. + cmd := exec.Command("docker", args...) + cmd.Dir = workSpace + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("building docker image failed running '%v' (%v): %s", cmd, err, out) } - return nil + + return err } diff --git a/tests/e2e/compatibility_tests.go b/tests/e2e/compatibility_tests.go new file mode 100644 index 0000000000..14bd48d7c7 --- /dev/null +++ b/tests/e2e/compatibility_tests.go @@ -0,0 +1,20 @@ +package main + +// Get compatible provider versions for this consumer +// +// Note: This is a hardcoded list of tags which has to be updated manually +// and serves as base information of different versions to be tested +// against this consumer implementation +func GetCompatibleProviderVersions() []string { + return []string{"v3.0.x"} +} + +// Get compatible consumer versions for this provider +// +// Note: This is a hardcoded list of tags which has to be updated manually +// and serves as base information of different versions to be tested +// against this provider implementation +func GetCompatibleConsumerVersions() []string { + // For now it's the same as for provider + return GetCompatibleProviderVersions() +} diff --git a/tests/e2e/config.go b/tests/e2e/config.go index aa9c98f8f8..78c3158808 100644 --- a/tests/e2e/config.go +++ b/tests/e2e/config.go @@ -59,7 +59,14 @@ type ContainerConfig struct { Now time.Time } -// TODO: Split out TestConfig and system wide config like localsdkpath +type TargetConfig struct { + gaiaTag string + localSdkPath string + useGaia bool + providerVersion string + consumerVersion string +} + type TestConfig struct { // These are the non altered values during a typical test run, where multiple test runs can exist // to validate different action sequences and corresponding state checks. @@ -71,19 +78,12 @@ type TestConfig struct { // having shorter timeout_commit reduces the test runtime because blocks are produced faster // lengthening the timeout_commit increases the test runtime because blocks are produced slower but the test is more reliable tendermintConfigOverride string - localSdkPath string - useGaia bool useCometmock bool // if false, nodes run CometBFT useGorelayer bool // if false, Hermes is used as the relayer - gaiaTag string // chains which are running, i.e. producing blocks, at the moment runningChains map[ChainID]bool // Used with CometMock. The time by which chains have been advanced. Used to keep chains in sync: when a new chain is started, advance its time by this value to keep chains in sync. - timeOffset time.Duration - - // consumer version the provider should be tested against - consumerVersion string - providerVersion string + timeOffset time.Duration transformGenesis bool name string } @@ -487,17 +487,13 @@ func ConsumerMisbehaviourTestConfig() TestConfig { return tc } -func (s *TestConfig) SetDockerConfig(localSdkPath string, useGaia bool, gaiaTag string, consumerVersion string, providerVersion string) { +func (s *TestConfig) SetDockerConfig(localSdkPath string, useGaia bool, gaiaTag string) { if localSdkPath != "" { fmt.Println("USING LOCAL SDK", localSdkPath) } if useGaia { fmt.Println("USING GAIA INSTEAD OF ICS provider app", gaiaTag) } - - s.useGaia = useGaia - s.gaiaTag = gaiaTag - s.localSdkPath = localSdkPath } func (s *TestConfig) SetCometMockConfig(useCometmock bool) { diff --git a/tests/e2e/main.go b/tests/e2e/main.go index 2c2fc1c86b..75ef0cf87b 100644 --- a/tests/e2e/main.go +++ b/tests/e2e/main.go @@ -1,19 +1,12 @@ package main import ( - "bufio" "flag" "fmt" "log" - "os" - "os/exec" - "reflect" - "strconv" "strings" "sync" "time" - - "github.com/kylelemons/godebug/pretty" ) // The list of test cases to be executed @@ -94,12 +87,6 @@ var ( var selectedTestfiles TestSet var stepChoices = map[string]StepChoice{ - "compatibility": { - name: "compatibility", - steps: compatibilitySteps, - description: `Minimal set of test steps to perform compatibility tests`, - testConfig: DefaultTestConfig(), - }, "happy-path-short": { name: "happy-path-short", steps: shortHappyPathSteps, @@ -162,31 +149,6 @@ var stepChoices = map[string]StepChoice{ }, } -func executeTests(tests []testStepsWithConfig) (err error) { - if parallel != nil && *parallel { - fmt.Println("=============== running all tests in parallel ===============") - } - - var wg sync.WaitGroup - for _, testCase := range tests { - if parallel != nil && *parallel { - wg.Add(1) - go func(run testStepsWithConfig) { - defer wg.Done() - run.testRun.Run(run.steps, *localSdkPath, *useGaia, *gaiaTag) - }(testCase) - } else { - log.Printf("=============== running %s ===============\n", testCase.testRun.name) - testCase.testRun.Run(testCase.steps, *localSdkPath, *useGaia, *gaiaTag) - } - } - - if parallel != nil && *parallel { - wg.Wait() - } - return -} - func getTestCaseUsageString() string { var builder strings.Builder @@ -323,48 +285,90 @@ func getTestCases(selectedPredefinedTests, selectedTestFiles TestSet) (tests []t return tests } -// Create multiversion test cases -// Based on the original set of test cases a multitude of test cases -// for the provided consumer and provider versions are generated. -func multiVersionConfig(testCases []testStepsWithConfig, providerVersions, consumerVerions VersionSet) []testStepsWithConfig { - var newTCs []testStepsWithConfig - if len(consumerVerions) == 0 && len(providerVersions) == 0 { - return testCases +func deleteTargets(targets []ExecutionTarget) error { + for _, target := range targets { + target.Delete() } + return nil +} + +// Create targets where test cases should be executed on +// For each combination of provider & consumer versions an ExecutionTarget +// is created. +func createTargets(providerVersions, consumerVersions VersionSet) ([]ExecutionTarget, error) { + targetCfg := TargetConfig{useGaia: *useGaia, localSdkPath: *localSdkPath, gaiaTag: *gaiaTag} + var targets []ExecutionTarget if len(consumerVersions) == 0 { - consumerVerions = append(consumerVerions, "") + consumerVersions = append(consumerVersions, "") } if len(providerVersions) == 0 { providerVersions = append(providerVersions, "") } for _, provider := range providerVersions { - for _, consumer := range consumerVerions { - if consumer == provider { - log.Printf("Skipping same version '%s' for provider & consumer", consumer) - continue + for _, consumer := range consumerVersions { + targetCfg.consumerVersion = consumer + targetCfg.providerVersion = provider + target := DockerContainer{targetConfig: targetCfg} + err := target.Build() + if err != nil { + deleteTargets(targets) + return nil, err } - log.Printf("Configuring test cases for: provider version %s, consumer version %s", provider, consumer) - for _, tc := range testCases { - tc.testRun.consumerVersion = consumer - tc.testRun.providerVersion = provider - tc.testRun.transformGenesis = *transformGenesis - suffix := "" - if consumer != "" { - suffix = fmt.Sprintf("_cons-%s", consumer) - } - if provider != "" { - suffix += fmt.Sprintf("_prov-%s", provider) - } - tc.testRun.containerConfig.ContainerName += suffix - tc.testRun.containerConfig.InstanceName += suffix - tc.testRun.name += suffix - newTCs = append(newTCs, tc) + targets = append(targets, &target) + } + } + return targets, nil +} + +func createTestRunners(targets []ExecutionTarget, testCases []testStepsWithConfig) []TestRunner { + runners := []TestRunner{} + for _, target := range targets { + for _, tc := range testCases { + tr := TestRunner{ + config: tc.testRun, + steps: tc.steps, + target: target, + verbose: *verbose, } + //TODO: refactor this target specific setting + tr.target.(*DockerContainer).containerCfg = tc.testRun.containerConfig + tr.config.transformGenesis = *transformGenesis + tr.config.SetCometMockConfig(*useCometmock) + tr.config.SetRelayerConfig(*useGorelayer) + runners = append(runners, tr) } } - return newTCs + return runners +} + +func executeTests(runners []TestRunner) error { + if parallel != nil && *parallel { + fmt.Println("=============== running all tests in parallel ===============") + } + + var wg sync.WaitGroup + var err error = nil + + for _, runner := range runners { + if parallel != nil && *parallel { + wg.Add(1) + go func(runner TestRunner) { + defer wg.Done() + runner.Run() + }(runner) + } else { + log.Printf("=============== running %s ===============\n", runner.config.name) + err = runner.Run() + } + } + + if parallel != nil && *parallel { + wg.Wait() + } + + return err } // runs E2E tests @@ -377,28 +381,20 @@ func main() { } testCases := getTestCases(selectedTests, selectedTestfiles) - testCases = multiVersionConfig(testCases, providerVersions, consumerVersions) + targets, err := createTargets(providerVersions, consumerVersions) + if err != nil { + log.Fatal("failed creating test targets: ", err) + } + defer func() { deleteTargets(targets) }() + testRunners := createTestRunners(targets, testCases) start := time.Now() - err := executeTests(testCases) + err = executeTests(testRunners) if err != nil { log.Fatalf("Test execution failed '%s'", err) } log.Printf("TOTAL TIME ELAPSED: %v\n", time.Since(start)) -} - -// Run sets up docker container and executes the steps in the test run. -// Docker containers are torn down after the test run is complete. -func (tr *TestConfig) Run(steps []Step, localSdkPath string, useGaia bool, gaiaTag string) { - tr.SetDockerConfig(localSdkPath, useGaia, gaiaTag, tr.consumerVersion, tr.providerVersion) - tr.SetCometMockConfig(*useCometmock) - tr.SetRelayerConfig(*useGorelayer) - - tr.validateStringLiterals() - tr.startDocker() - tr.executeSteps(steps) - tr.teardownDocker() } type StepChoice struct { @@ -407,220 +403,3 @@ type StepChoice struct { description string testConfig TestConfig } - -func (tr *TestConfig) runStep(step Step, verbose bool) { - switch action := step.Action.(type) { - case StartChainAction: - tr.startChain(action, verbose) - case StartSovereignChainAction: - tr.startSovereignChain(action, verbose) - case LegacyUpgradeProposalAction: - tr.submitLegacyUpgradeProposal(action, verbose) - case WaitUntilBlockAction: - tr.waitUntilBlockOnChain(action) - case ChangeoverChainAction: - tr.changeoverChain(action, verbose) - case SendTokensAction: - tr.sendTokens(action, verbose) - case SubmitTextProposalAction: - tr.submitTextProposal(action, verbose) - case SubmitConsumerAdditionProposalAction: - tr.submitConsumerAdditionProposal(action, verbose) - case SubmitConsumerRemovalProposalAction: - tr.submitConsumerRemovalProposal(action, verbose) - case SubmitParamChangeLegacyProposalAction: - tr.submitParamChangeProposal(action, verbose) - case VoteGovProposalAction: - tr.voteGovProposal(action, verbose) - case StartConsumerChainAction: - tr.startConsumerChain(action, verbose) - case AddChainToRelayerAction: - tr.addChainToRelayer(action, verbose) - case CreateIbcClientsAction: - tr.createIbcClientsHermes(action, verbose) - case AddIbcConnectionAction: - tr.addIbcConnection(action, verbose) - case AddIbcChannelAction: - tr.addIbcChannel(action, verbose) - case TransferChannelCompleteAction: - tr.transferChannelComplete(action, verbose) - case RelayPacketsAction: - tr.relayPackets(action, verbose) - case RelayRewardPacketsToProviderAction: - tr.relayRewardPacketsToProvider(action, verbose) - case DelegateTokensAction: - tr.delegateTokens(action, verbose) - case UnbondTokensAction: - tr.unbondTokens(action, verbose) - case CancelUnbondTokensAction: - tr.cancelUnbondTokens(action, verbose) - case RedelegateTokensAction: - tr.redelegateTokens(action, verbose) - case DowntimeSlashAction: - tr.invokeDowntimeSlash(action, verbose) - case UnjailValidatorAction: - tr.unjailValidator(action, verbose) - case DoublesignSlashAction: - tr.invokeDoublesignSlash(action, verbose) - case LightClientAmnesiaAttackAction: - tr.lightClientAmnesiaAttack(action, verbose) - case LightClientEquivocationAttackAction: - tr.lightClientEquivocationAttack(action, verbose) - case LightClientLunaticAttackAction: - tr.lightClientLunaticAttack(action, verbose) - case RegisterRepresentativeAction: - tr.registerRepresentative(action, verbose) - case AssignConsumerPubKeyAction: - tr.assignConsumerPubKey(action, verbose) - case SlashMeterReplenishmentAction: - tr.waitForSlashMeterReplenishment(action, verbose) - case WaitTimeAction: - tr.waitForTime(action, verbose) - case StartRelayerAction: - tr.startRelayer(action, verbose) - case ForkConsumerChainAction: - tr.forkConsumerChain(action, verbose) - case UpdateLightClientAction: - tr.updateLightClient(action, verbose) - case StartConsumerEvidenceDetectorAction: - tr.startConsumerEvidenceDetector(action, verbose) - case SubmitChangeRewardDenomsProposalAction: - tr.submitChangeRewardDenomsProposal(action, verbose) - default: - log.Fatalf("unknown action in testRun %s: %#v", tr.name, action) - } - - modelState := step.State - actualState := tr.getState(step.State) - - // Check state - if !reflect.DeepEqual(actualState, modelState) { - fmt.Printf("=============== %s FAILED ===============\n", tr.name) - fmt.Println("FAILED action", reflect.TypeOf(step.Action).Name()) - pretty.Print("actual state", actualState) - pretty.Print("model state", modelState) - log.Fatal(`actual state (-) not equal to model state (+): ` + pretty.Compare(actualState, modelState)) - } -} - -// executeSteps sequentially runs steps. -func (tr *TestConfig) executeSteps(steps []Step) { - fmt.Printf("=============== started %s tests ===============\n", tr.name) - - start := time.Now() - for i, step := range steps { - // print something the show the test is alive - fmt.Printf("running %s: step %d == %s \n", - tr.name, i+1, reflect.TypeOf(step.Action).Name()) - tr.runStep(step, *verbose) - } - - fmt.Printf("=============== finished %s tests in %v ===============\n", tr.name, time.Since(start)) -} - -func (tr *TestConfig) buildDockerImages() { - fmt.Printf("=============== building %s images ===============\n", tr.name) - tmpDir, err := os.MkdirTemp(os.TempDir(), "e2eWorkTree") - if err != nil { - log.Fatalf("Error createing temp directory for docker creation") - } - - // Build ICS image of a given version - icsVersions := []string{} - if tr.consumerVersion != "" { - icsVersions = append(icsVersions, tr.consumerVersion) - } - if tr.providerVersion != "" && tr.consumerVersion != tr.providerVersion { - icsVersions = append(icsVersions, tr.providerVersion) - } - for _, icsVersion := range icsVersions { - imageName := fmt.Sprintf("cosmos-ics:%s", icsVersion) - err := buildDockerImage(imageName, icsVersion, tmpDir) - if err != nil { - log.Fatalf("Error building docker image '%s':%v", icsVersion, err) - } - } -} - -func (tr *TestConfig) startDocker() { - tr.buildDockerImages() - fmt.Printf("=============== building %s testRun ===============\n", tr.name) - - options := []string{} - - localSdk := tr.localSdkPath - if localSdk != "" { - options = append(options, fmt.Sprintf("-s %s", tr.localSdkPath)) - } - - if tr.consumerVersion != "" { - options = append(options, fmt.Sprintf("-c %s", tr.consumerVersion)) - } - - if tr.providerVersion != "" { - options = append(options, fmt.Sprintf("-p %s", tr.providerVersion)) - } - - if tr.useGaia { - if tr.gaiaTag != "" { - majVersion, err := strconv.Atoi(tr.gaiaTag[1:strings.Index(tr.gaiaTag, ".")]) - if err != nil { - panic(fmt.Sprintf("invalid gaia version %s", tr.gaiaTag)) - } - if majVersion < 9 { - panic(fmt.Sprintf("gaia version %s is not supported - supporting only v9.x.x and newer", tr.gaiaTag)) - } - options = append(options, fmt.Sprintf("-g %s", tr.gaiaTag)) - } - } - scriptStr := fmt.Sprintf( - "tests/e2e/testnet-scripts/start-docker.sh %s %s %s", - strings.Join(options, " "), - tr.containerConfig.ContainerName, - tr.containerConfig.InstanceName, - ) - - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("/bin/bash", "-c", scriptStr) - - cmdReader, err := cmd.StdoutPipe() - if err != nil { - log.Fatal(err) - } - cmd.Stderr = cmd.Stdout - - if err := cmd.Start(); err != nil { - log.Fatal(err) - } - - scanner := bufio.NewScanner(cmdReader) - - for scanner.Scan() { - out := scanner.Text() - if verbose != nil && *verbose { - fmt.Println("startDocker: " + out) - } - if out == "beacon!!!!!!!!!!" { - return - } - } - if err := scanner.Err(); err != nil { - log.Fatal(err) - } - - err = cmd.Wait() - log.Fatalf("StartDocker exited with error: %v", err) -} - -// remove docker container to reduce resource usage -// otherwise the chain will keep running in the background -func (tr *TestConfig) teardownDocker() { - fmt.Printf("=============== tearing down %s testRun ===============\n", tr.name) - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "kill", tr.containerConfig.InstanceName) - - bz, err := cmd.CombinedOutput() - if err != nil { - log.Fatal(err, "\n", string(bz)) - } -} diff --git a/tests/e2e/steps.go b/tests/e2e/steps.go index 10266e5568..783d976a3a 100644 --- a/tests/e2e/steps.go +++ b/tests/e2e/steps.go @@ -13,18 +13,6 @@ func concatSteps(steps ...[]Step) []Step { return concat } -// Limited amount of sanity tests to use for compatibility testing -var compatibilitySteps = concatSteps( - stepsStartChains([]string{"consu"}, false), - stepsDelegate("consu"), - stepsUnbond("consu"), - stepsRedelegateShort("consu"), - stepsDowntime("consu"), - stepsStartRelayer(), - stepsConsumerRemovalPropNotPassing("consu", 2), // submit removal prop but vote no on it - chain should stay - stepsStopChain("consu", 3), // stop chain -) - var happyPathSteps = concatSteps( stepsStartChains([]string{"consu"}, false), stepsDelegate("consu"), diff --git a/tests/e2e/test_driver.go b/tests/e2e/test_driver.go new file mode 100644 index 0000000000..98430bb12c --- /dev/null +++ b/tests/e2e/test_driver.go @@ -0,0 +1,161 @@ +package main + +import ( + "fmt" + "log" + "reflect" + "time" + + "github.com/kylelemons/godebug/pretty" +) + +// TestCaseDriver knows how different TC can be executed +type TestCaseDriver interface { + Run(steps []Step, target ExecutionTarget, verbose bool) error +} + +func GetTestCaseDriver(testCfg TestConfig) TestCaseDriver { + return &DefaultDriver{testCfg: testCfg} +} + +type DefaultDriver struct { + testCfg TestConfig + target ExecutionTarget + verbose bool +} + +// Execute tests +func (pr *DefaultDriver) Run(steps []Step, target ExecutionTarget, verbose bool) error { + pr.target = target + pr.verbose = verbose + + fmt.Printf("=============== started %s tests ===============\n", pr.testCfg.name) + + start := time.Now() + for i, step := range steps { + fmt.Printf("running %s: step %d/%d == %s \n", + pr.testCfg.name, i+1, len(steps), reflect.TypeOf(step.Action).Name()) + + err := pr.runStep(step) + if err != nil { + return err + } + } + fmt.Printf("=============== finished %s tests in %v ===============\n", pr.testCfg.name, time.Since(start)) + return nil +} + +func (pr *DefaultDriver) runStep(step Step) error { + err := pr.runAction(step.Action) + if err != nil { + return err + } + modelState := step.State + actualState, err := pr.getState(step.State) + if err != nil { + return err + } + + // Check state + if !reflect.DeepEqual(actualState, modelState) { + fmt.Printf("=============== %s FAILED ===============\n", pr.testCfg.name) + fmt.Println("FAILED action", reflect.TypeOf(step.Action).Name()) + pretty.Print("actual state", actualState) + pretty.Print("model state", modelState) + log.Fatal(`actual state (-) not equal to model state (+): ` + pretty.Compare(actualState, modelState)) + } + return nil +} + +func (pr *DefaultDriver) GetState(modelState State) { + pr.testCfg.getState(modelState) +} + +func (pr *DefaultDriver) runAction(action interface{}) error { + switch action := action.(type) { + case StartChainAction: + pr.testCfg.startChain(action, pr.target, pr.verbose) + case StartSovereignChainAction: + pr.testCfg.startSovereignChain(action, pr.verbose) + case LegacyUpgradeProposalAction: + pr.testCfg.submitLegacyUpgradeProposal(action, pr.verbose) + case WaitUntilBlockAction: + pr.testCfg.waitUntilBlockOnChain(action) + case ChangeoverChainAction: + pr.testCfg.changeoverChain(action, pr.verbose) + case SendTokensAction: + pr.testCfg.sendTokens(action, pr.verbose) + case SubmitTextProposalAction: + pr.testCfg.submitTextProposal(action, pr.verbose) + case SubmitConsumerAdditionProposalAction: + pr.testCfg.submitConsumerAdditionProposal(action, pr.verbose) + case SubmitConsumerRemovalProposalAction: + pr.testCfg.submitConsumerRemovalProposal(action, pr.verbose) + case SubmitParamChangeLegacyProposalAction: + pr.testCfg.submitParamChangeProposal(action, pr.verbose) + case VoteGovProposalAction: + pr.testCfg.voteGovProposal(action, pr.verbose) + case StartConsumerChainAction: + pr.testCfg.startConsumerChain(action, pr.target, pr.verbose) + case AddChainToRelayerAction: + pr.testCfg.addChainToRelayer(action, pr.verbose) + case CreateIbcClientsAction: + pr.testCfg.createIbcClientsHermes(action, pr.verbose) + case AddIbcConnectionAction: + pr.testCfg.addIbcConnection(action, pr.verbose) + case AddIbcChannelAction: + pr.testCfg.addIbcChannel(action, pr.verbose) + case TransferChannelCompleteAction: + pr.testCfg.transferChannelComplete(action, pr.verbose) + case RelayPacketsAction: + pr.testCfg.relayPackets(action, pr.verbose) + case RelayRewardPacketsToProviderAction: + pr.testCfg.relayRewardPacketsToProvider(action, pr.verbose) + case DelegateTokensAction: + pr.testCfg.delegateTokens(action, pr.verbose) + case UnbondTokensAction: + pr.testCfg.unbondTokens(action, pr.verbose) + case CancelUnbondTokensAction: + pr.testCfg.cancelUnbondTokens(action, pr.verbose) + case RedelegateTokensAction: + pr.testCfg.redelegateTokens(action, pr.verbose) + case DowntimeSlashAction: + pr.testCfg.invokeDowntimeSlash(action, pr.verbose) + case UnjailValidatorAction: + pr.testCfg.unjailValidator(action, pr.verbose) + case DoublesignSlashAction: + pr.testCfg.invokeDoublesignSlash(action, pr.verbose) + case LightClientAmnesiaAttackAction: + pr.testCfg.lightClientAmnesiaAttack(action, pr.verbose) + case LightClientEquivocationAttackAction: + pr.testCfg.lightClientEquivocationAttack(action, pr.verbose) + case LightClientLunaticAttackAction: + pr.testCfg.lightClientLunaticAttack(action, pr.verbose) + case RegisterRepresentativeAction: + pr.testCfg.registerRepresentative(action, pr.verbose) + case AssignConsumerPubKeyAction: + pr.testCfg.assignConsumerPubKey(action, pr.verbose) + case SlashMeterReplenishmentAction: + pr.testCfg.waitForSlashMeterReplenishment(action, pr.verbose) + case WaitTimeAction: + pr.testCfg.waitForTime(action, pr.verbose) + case StartRelayerAction: + pr.testCfg.startRelayer(action, pr.verbose) + case ForkConsumerChainAction: + pr.testCfg.forkConsumerChain(action, pr.verbose) + case UpdateLightClientAction: + pr.testCfg.updateLightClient(action, pr.verbose) + case StartConsumerEvidenceDetectorAction: + pr.testCfg.startConsumerEvidenceDetector(action, pr.verbose) + case SubmitChangeRewardDenomsProposalAction: + pr.testCfg.submitChangeRewardDenomsProposal(action, pr.verbose) + default: + log.Fatalf("unknown action in testRun %s: %#v", pr.testCfg.name, action) + } + return nil +} + +func (pr *DefaultDriver) getState(modelState State) (State, error) { + // forwarding it for now + return pr.testCfg.getState(modelState), nil +} diff --git a/tests/e2e/test_runner.go b/tests/e2e/test_runner.go new file mode 100644 index 0000000000..a3b3e22a42 --- /dev/null +++ b/tests/e2e/test_runner.go @@ -0,0 +1,52 @@ +package main + +import "fmt" + +// A test runner drives the execution of test cases +// It sets up the test environment and the test driver to run the tests +type TestRunner struct { + config TestConfig + steps []Step + testDriver TestCaseDriver + target ExecutionTarget + verbose bool +} + +// Run will set up the test environment and executes the tests +func (tr *TestRunner) Run() error { + err := tr.checkConfig() + if err != nil { + return err + } + + err = tr.setupEnvironment(tr.target) + if err != nil { + return fmt.Errorf("error setting up test environment: %v", err) + } + + tr.testDriver = GetTestCaseDriver(tr.config) + err = tr.testDriver.Run(tr.steps, tr.target, tr.verbose) + if err != nil { + // not tearing down environment for troubleshooting reasons on container + return fmt.Errorf("test run failed: %v", err) + } + return tr.teardownEnvironment() +} + +func (tr *TestRunner) checkConfig() error { + tr.config.validateStringLiterals() + return nil +} +func (tr *TestRunner) setupEnvironment(target ExecutionTarget) error { + tr.target = target + return target.Start() +} + +func (tr *TestRunner) teardownEnvironment() error { + return tr.target.Stop() +} + +func (tr *TestRunner) Setup(testCfg TestConfig) error { + tr.config = testCfg + return nil +} diff --git a/tests/e2e/test_target.go b/tests/e2e/test_target.go new file mode 100644 index 0000000000..48f7a56fa2 --- /dev/null +++ b/tests/e2e/test_target.go @@ -0,0 +1,210 @@ +package main + +import ( + "bufio" + "fmt" + "log" + "os/exec" + "strings" +) + +type ExecutionTarget interface { + GetTargetType() string + GetLogs(path string) []byte + GetStartChainScript(isProvider bool) string + ExecCommand(name string, arg ...string) *exec.Cmd + Start() error + Stop() error + Build() error + Delete() error +} + +type DockerContainer struct { + targetConfig TargetConfig + containerCfg ContainerConfig + images []string //images needed to build the target container + ImageName string +} + +func (dc *DockerContainer) GetTargetType() string { + return "docker" +} + +func (dc *DockerContainer) GetLogs(path string) []byte { + logs := []byte{} + return logs +} + +func generateImageName(version string, cfg TargetConfig) (string, error) { + // identify a tag name + tagName := "" + if version == "" { + tagName = "local" // this refers to build from local workspace + } else { + // use git hash of rev as docker image tag + //cmd := exec.Command("git", "rev-parse", "--verify", "--short", version) + cmd := exec.Command("git", "log", "--pretty=format:%h", "-n", "1", version) + out, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("error getting hash for revision: %v, '%s'", err, out) + } + tagName = strings.TrimSpace(string(out)) + } + + imageName := "cosmos-ics" + if cfg.useGaia { + imageName += "_gaia" + tagName += "-" + cfg.gaiaTag + } + if cfg.localSdkPath != "" { + imageName += "_sdk" + } + + return fmt.Sprintf("%s:%s", imageName, tagName), nil +} + +// Build the docker image for the target container +func (dc *DockerContainer) Build() error { + consumerVersion := dc.targetConfig.consumerVersion + providerVersion := dc.targetConfig.providerVersion + + consumerImageName, err := generateImageName(consumerVersion, dc.targetConfig) + if err != nil { + return fmt.Errorf("failed building docker image: %v", err) + } + err = buildDockerImage(consumerImageName, consumerVersion, dc.targetConfig) + if err != nil { + return err + } + dc.images = append(dc.images, consumerImageName) + + if consumerVersion == "" && consumerVersion == providerVersion { + dc.ImageName = consumerImageName + return nil + } + + // build image for provider as it's a different version to be run + providerImageName, err := generateImageName(providerVersion, dc.targetConfig) + if err != nil { + return fmt.Errorf("failed building docker image: %v", err) + } + err = buildDockerImage(providerImageName, providerVersion, dc.targetConfig) + if err != nil { + return err + } + + // build combined image using provider/consumer versions from images built above + combinedImageName := fmt.Sprintf("cosmos-ics-combined:%s_%s", + strings.Split(providerImageName, ":")[1], + strings.Split(consumerImageName, ":")[1]) + cmd := exec.Command("docker", "build", "-f", "Dockerfile-Combined", + "--build-arg", fmt.Sprintf("PROVIDER_IMAGE=%s", providerImageName), + "--build-arg", fmt.Sprintf("CONSUMER_IMAGE=%s", consumerImageName), + "-t", combinedImageName, + ".") + out, err := cmd.CombinedOutput() + if err != nil { + log.Printf("Failed running: %v", cmd) + return fmt.Errorf("error building combined docker image: %v, %s", err, string(out)) + } + dc.images = append(dc.images, combinedImageName) + dc.ImageName = combinedImageName + return err +} + +func (dc *DockerContainer) Delete() error { + for _, img := range dc.images { + cmd := exec.Command("docker", "image", "rm", img) + out, err := cmd.CombinedOutput() + //TODO: ignore errors related to non-existing images + if err != nil { + log.Printf("failed deleting image '%v' (%v): %v", cmd, err, string(out)) + } + } + return nil +} + +// ExecCommand returns the command struct to execute the named program with +// given arguments on the current target (docker container) +func (dc *DockerContainer) ExecCommand(name string, arg ...string) *exec.Cmd { + args := []string{"exec", dc.containerCfg.InstanceName, name} + args = append(args, arg...) + return exec.Command("docker", args...) +} + +// Get star-chain script to be used on target for a specific chain type +// Needed for different consumer/provider versions staged in one container +func (dc *DockerContainer) GetStartChainScript(isConsumer bool) string { + startChainScript := "/testnet-scripts/start-chain.sh" + if dc.targetConfig.providerVersion != "" && !isConsumer { + log.Printf("Using start-chain script for provider version '%s'", dc.targetConfig.providerVersion) + startChainScript = "/provider/testnet-scripts/start-chain.sh" + } + + if dc.targetConfig.consumerVersion != "" && isConsumer { + log.Printf("Using start-chain script for consumer version '%s'", dc.targetConfig.consumerVersion) + startChainScript = "/consumer/testnet-scripts/start-chain.sh" + } + return startChainScript + +} + +// Startup the container +func (dc *DockerContainer) Start() error { + fmt.Println("@@@@ starting existing containers", dc.containerCfg.InstanceName) + // Remove existing containers from previous runs + dc.Stop() + + // Run new test container instance with extended privileges. + // Extended privileges are granted to the container here to allow for network namespace manipulation (bringing a node up/down) + // See: https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities + cmd := exec.Command("docker", "run", "--name", dc.containerCfg.InstanceName, + "--cap-add=NET_ADMIN", "--privileged", dc.ImageName, + "/bin/bash", "/testnet-scripts/beacon.sh") + + cmdReader, err := cmd.StdoutPipe() + if err != nil { + log.Fatal(err) + } + cmd.Stderr = cmd.Stdout + + if err := cmd.Start(); err != nil { + log.Fatal(err) + } + scanner := bufio.NewScanner(cmdReader) + + // Wait until container is up + for scanner.Scan() { + out := scanner.Text() + if verbose != nil && *verbose { + fmt.Println("startDocker: " + out) + } + if out == "beacon!!!!!!!!!!" { + return nil + } + } + if err := scanner.Err(); err != nil { + return fmt.Errorf("error bringing up container: %v", err) + } + + err = cmd.Wait() + return fmt.Errorf("starting container exited with error: %v", err) +} + +// Stop will stop the container and remove it +func (dc *DockerContainer) Stop() error { + fmt.Println("@@@@ stopping existing containers", dc.containerCfg.InstanceName) + //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. + cmd := exec.Command("docker", "stop", dc.containerCfg.InstanceName) + bz, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("error stopping docker container: %v, %s", err, string(bz)) + } + + cmd = exec.Command("docker", "rm", dc.containerCfg.InstanceName) + bz, err = cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("error removing docker container: %v, %s", err, string(bz)) + } + return err +} diff --git a/tests/e2e/testnet-scripts/start-docker.sh b/tests/e2e/testnet-scripts/start-docker.sh index 4ed0f0a70c..59b5800b6f 100755 --- a/tests/e2e/testnet-scripts/start-docker.sh +++ b/tests/e2e/testnet-scripts/start-docker.sh @@ -65,20 +65,20 @@ if [[ ${USE_GAIA_TAG+x} ]] then printf "\n\nUsing gaia as provider\n\n" printf "\n\nUsing gaia tag %s\n\n" "$USE_GAIA_TAG" - docker build -f Dockerfile.gaia -t "$CONTAINER_NAME" --build-arg USE_GAIA_TAG="$USE_GAIA_TAG" ./ + docker build -f Dockerfile.gaia -t "$CONTAINER_NAME" --build-arg USE_GAIA_TAG="$USE_GAIA_TAG" . elif [ ${CONSUMER_VERSION+x} ] && [ ${PROVIDER_VERSION+x} ]; then printf "\n\nUsing ICS consumer app from image version ${CONSUMER_VERSION} and provider from image version ${PROVIDER_VERSION}" - docker build -f Dockerfile-Combined --build-arg PROVIDER_VERSION="${PROVIDER_VERSION}" --build-arg CONSUMER_VERSION="${CONSUMER_VERSION}" -t "$CONTAINER_NAME" ./ + docker build -f Dockerfile-Combined --build-arg PROVIDER_VERSION="${PROVIDER_VERSION}" --build-arg CONSUMER_VERSION="${CONSUMER_VERSION}" -t "$CONTAINER_NAME" . elif [ ${CONSUMER_VERSION+x} ]; then printf "\n\nUsing ICS consumer app from image version ${CONSUMER_VERSION}" - docker build -f Dockerfile-Consumer --build-arg CONSUMER_VERSION="${CONSUMER_VERSION}" -t "$CONTAINER_NAME" ./ + docker build -f Dockerfile-Consumer --build-arg CONSUMER_VERSION="${CONSUMER_VERSION}" -t "$CONTAINER_NAME" . elif [ ${PROVIDER_VERSION+x} ]; then printf "\n\nUsing ICS provider app from image version ${PROVIDER_VERSION}" - docker build -f Dockerfile-Provider --build-arg PROVIDER_VERSION="${PROVIDER_VERSION}" -t "$CONTAINER_NAME" ./ + docker build -f Dockerfile-Provider --build-arg PROVIDER_VERSION="${PROVIDER_VERSION}" -t "$CONTAINER_NAME" . else printf "\n\nUsing ICS provider app as provider\n\n\n" - docker build -f Dockerfile -t "$CONTAINER_NAME" ./ + docker build -f Dockerfile -t "$CONTAINER_NAME" . fi # Remove copied sdk directory From 15e007eb04cfdf98daeb139db8e43c0e76d858fc Mon Sep 17 00:00:00 2001 From: Bernd Mueller Date: Wed, 10 Jan 2024 09:29:30 +0100 Subject: [PATCH 04/16] adapt actions to target --- Dockerfile-Combined | 3 - tests/e2e/actions.go | 266 +++++++++++++-------------- tests/e2e/actions_sovereign_chain.go | 22 +-- tests/e2e/test_driver.go | 81 ++++---- tests/e2e/test_target.go | 32 ++-- 5 files changed, 197 insertions(+), 207 deletions(-) diff --git a/Dockerfile-Combined b/Dockerfile-Combined index c129b86cfc..89a6a840ee 100644 --- a/Dockerfile-Combined +++ b/Dockerfile-Combined @@ -45,8 +45,5 @@ COPY --from=consumer /testnet-scripts /consumer/testnet-scripts COPY --from=provider /usr/local/bin/interchain-security-pd /usr/local/bin/interchain-security-pd COPY --from=provider /testnet-scripts /provider/testnet-scripts -# Copy in the shell scripts that run the testnet -ADD ./tests/e2e/testnet-scripts /testnet-scripts - # Copy in the hermes config ADD ./tests/e2e/testnet-scripts/hermes-config.toml /root/.hermes/config.toml diff --git a/tests/e2e/actions.go b/tests/e2e/actions.go index 009419053d..bc426d5f33 100644 --- a/tests/e2e/actions.go +++ b/tests/e2e/actions.go @@ -32,11 +32,11 @@ const done = "done!!!!!!!!" func (tr TestConfig) sendTokens( action SendTokensAction, + target ExecutionTarget, verbose bool, ) { binaryName := tr.chainConfigs[action.Chain].BinaryName - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, binaryName, + cmd := target.ExecCommand(binaryName, "tx", "bank", "send", tr.validatorConfigs[action.From].DelAddress, @@ -133,7 +133,7 @@ func (tr *TestConfig) startChain( cometmockArg = "false" } - startChainScript := target.GetStartChainScript(action.IsConsumer) + startChainScript := target.GetTestScriptPath(action.IsConsumer, "start-chain.sh") cmd := target.ExecCommand("/bin/bash", startChainScript, chainConfig.BinaryName, string(vals), string(chainConfig.ChainId), chainConfig.IpPrefix, genesisChanges, @@ -175,7 +175,7 @@ func (tr *TestConfig) startChain( Chain: action.Chain, Validator: action.Validators[0].Id, IsConsumer: action.IsConsumer, - }, verbose) + }, target, verbose) // store the fact that we started the chain tr.runningChains[action.Chain] = true @@ -196,11 +196,12 @@ type SubmitTextProposalAction struct { func (tr TestConfig) submitTextProposal( action SubmitTextProposalAction, + target ExecutionTarget, verbose bool, ) { // TEXT PROPOSAL - //#nosec G204 -- Bypass linter warning for spa wning subprocess with cmd arguments. - bz, err := exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[action.Chain].BinaryName, + bz, err := target.ExecCommand( + tr.chainConfigs[action.Chain].BinaryName, "tx", "gov", "submit-legacy-proposal", `--title`, action.Title, `--description`, action.Description, @@ -233,6 +234,7 @@ type SubmitConsumerAdditionProposalAction struct { func (tr TestConfig) submitConsumerAdditionProposal( action SubmitConsumerAdditionProposalAction, + target ExecutionTarget, verbose bool, ) { spawnTime := tr.containerConfig.Now.Add(time.Duration(action.SpawnTime) * time.Millisecond) @@ -265,17 +267,17 @@ func (tr TestConfig) submitConsumerAdditionProposal( log.Fatal("prop json contains single quote") } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - bz, err = exec.Command("docker", "exec", tr.containerConfig.InstanceName, - "/bin/bash", "-c", fmt.Sprintf(`echo '%s' > %s`, jsonStr, "/temp-proposal.json")).CombinedOutput() + bz, err = target.ExecCommand( + "/bin/bash", "-c", fmt.Sprintf(`echo '%s' > %s`, jsonStr, "/temp-proposal.json"), + ).CombinedOutput() if err != nil { log.Fatal(err, "\n", string(bz)) } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. // CONSUMER ADDITION PROPOSAL - bz, err = exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[action.Chain].BinaryName, + bz, err = target.ExecCommand( + tr.chainConfigs[action.Chain].BinaryName, "tx", "gov", "submit-legacy-proposal", "consumer-addition", "/temp-proposal.json", `--from`, `validator`+fmt.Sprint(action.From), `--chain-id`, string(tr.chainConfigs[action.Chain].ChainId), @@ -304,6 +306,7 @@ type SubmitConsumerRemovalProposalAction struct { func (tr TestConfig) submitConsumerRemovalProposal( action SubmitConsumerRemovalProposalAction, + target ExecutionTarget, verbose bool, ) { stopTime := tr.containerConfig.Now.Add(action.StopTimeOffset) @@ -325,17 +328,15 @@ func (tr TestConfig) submitConsumerRemovalProposal( log.Fatal("prop json contains single quote") } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - bz, err = exec.Command("docker", "exec", tr.containerConfig.InstanceName, + bz, err = target.ExecCommand( "/bin/bash", "-c", fmt.Sprintf(`echo '%s' > %s`, jsonStr, "/temp-proposal.json")).CombinedOutput() if err != nil { log.Fatal(err, "\n", string(bz)) } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - bz, err = exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[action.Chain].BinaryName, - + bz, err = target.ExecCommand( + tr.chainConfigs[action.Chain].BinaryName, "tx", "gov", "submit-legacy-proposal", "consumer-removal", "/temp-proposal.json", `--from`, `validator`+fmt.Sprint(action.From), @@ -380,6 +381,7 @@ type paramChangeJSON struct { func (tr TestConfig) submitParamChangeProposal( action SubmitParamChangeLegacyProposalAction, + target ExecutionTarget, verbose bool, ) { prop := paramChangeProposalJSON{ @@ -400,17 +402,16 @@ func (tr TestConfig) submitParamChangeProposal( log.Fatal("prop json contains single quote") } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - bz, err = exec.Command("docker", "exec", tr.containerConfig.InstanceName, - "/bin/bash", "-c", fmt.Sprintf(`echo '%s' > %s`, jsonStr, "/params-proposal.json")).CombinedOutput() + bz, err = target.ExecCommand( + "/bin/bash", "-c", fmt.Sprintf(`echo '%s' > %s`, jsonStr, "/params-proposal.json"), + ).CombinedOutput() if err != nil { log.Fatal(err, "\n", string(bz)) } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - // PARAM CHANGE PROPOSAL - cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[action.Chain].BinaryName, + cmd := target.ExecCommand( + tr.chainConfigs[action.Chain].BinaryName, "tx", "gov", "submit-legacy-proposal", "param-change", "/params-proposal.json", @@ -441,6 +442,7 @@ type VoteGovProposalAction struct { func (tr *TestConfig) voteGovProposal( action VoteGovProposalAction, + target ExecutionTarget, verbose bool, ) { var wg sync.WaitGroup @@ -449,8 +451,8 @@ func (tr *TestConfig) voteGovProposal( vote := action.Vote[i] go func(val ValidatorID, vote string) { defer wg.Done() - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - bz, err := exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[action.Chain].BinaryName, + bz, err := target.ExecCommand( + tr.chainConfigs[action.Chain].BinaryName, "tx", "gov", "vote", fmt.Sprint(action.PropNumber), vote, @@ -483,7 +485,7 @@ type StartConsumerChainAction struct { } // Transform consumer genesis content from older version -func (tr *TestConfig) transformConsumerGenesis(consumerChain ChainID, genesis []byte) []byte { +func (tr *TestConfig) transformConsumerGenesis(consumerChain ChainID, genesis []byte, target ExecutionTarget) []byte { log.Print("Transforming consumer genesis") log.Printf("Original ccv genesis: %s\n", string(genesis)) @@ -510,8 +512,8 @@ func (tr *TestConfig) transformConsumerGenesis(consumerChain ChainID, genesis [] } consumerBinaryName := tr.chainConfigs[consumerChain].BinaryName - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd = exec.Command("docker", "exec", containerInstance, consumerBinaryName, + cmd = target.ExecCommand( + consumerBinaryName, "genesis", "transform", targetFile) result, err := cmd.CombinedOutput() if err != nil { @@ -522,13 +524,12 @@ func (tr *TestConfig) transformConsumerGenesis(consumerChain ChainID, genesis [] } // Get consumer genesis from provider -func (tr *TestConfig) getConsumerGenesis(providerChain, consumerChain ChainID) string { +func (tr *TestConfig) getConsumerGenesis(providerChain, consumerChain ChainID, target ExecutionTarget) string { log.Print("Exporting consumer genesis from provider") - containerInstance := tr.containerConfig.InstanceName providerBinaryName := tr.chainConfigs[providerChain].BinaryName - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", containerInstance, providerBinaryName, + cmd := target.ExecCommand( + providerBinaryName, "query", "provider", "consumer-genesis", string(tr.chainConfigs[consumerChain].ChainId), @@ -544,7 +545,7 @@ func (tr *TestConfig) getConsumerGenesis(providerChain, consumerChain ChainID) s // only needed when consumer is running v3.3.x and later if tr.transformGenesis { - return string(tr.transformConsumerGenesis(consumerChain, bz)) + return string(tr.transformConsumerGenesis(consumerChain, bz, target)) } return string(bz) } @@ -555,7 +556,7 @@ func (tr *TestConfig) startConsumerChain( verbose bool, ) { log.Printf("Starting consumer chain %s", action.ConsumerChain) - consumerGenesis := ".app_state.ccvconsumer = " + tr.getConsumerGenesis(action.ProviderChain, action.ConsumerChain) + consumerGenesis := ".app_state.ccvconsumer = " + tr.getConsumerGenesis(action.ProviderChain, action.ConsumerChain, target) consumerGenesisChanges := tr.chainConfigs[action.ConsumerChain].GenesisChanges if consumerGenesisChanges != "" { consumerGenesis = consumerGenesis + " | " + consumerGenesisChanges + " | " + action.GenesisChanges @@ -578,12 +579,13 @@ type ChangeoverChainAction struct { func (tr TestConfig) changeoverChain( action ChangeoverChainAction, + target ExecutionTarget, verbose bool, ) { // sleep until the consumer chain genesis is ready on consumer time.Sleep(5 * time.Second) - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[action.ProviderChain].BinaryName, + cmd := target.ExecCommand( + tr.chainConfigs[action.ProviderChain].BinaryName, "query", "provider", "consumer-genesis", string(tr.chainConfigs[action.SovereignChain].ChainId), @@ -610,11 +612,12 @@ func (tr TestConfig) changeoverChain( tr.startChangeover(ChangeoverChainAction{ Validators: action.Validators, GenesisChanges: consumerGenesis, - }, verbose) + }, target, verbose) } func (tr TestConfig) startChangeover( action ChangeoverChainAction, + target ExecutionTarget, verbose bool, ) { chainConfig := tr.chainConfigs[ChainID("sover")] @@ -663,9 +666,11 @@ func (tr TestConfig) startChangeover( genesisChanges = chainConfig.GenesisChanges } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "/bin/bash", - "/testnet-scripts/start-changeover.sh", chainConfig.UpgradeBinary, string(vals), + isConsumer := true + changeoverScript := target.GetTestScriptPath(isConsumer, "start-changeover.sh") + cmd := target.ExecCommand( + "/bin/bash", + changeoverScript, chainConfig.UpgradeBinary, string(vals), "sover", chainConfig.IpPrefix, genesisChanges, tr.tendermintConfigOverride, ) @@ -752,17 +757,19 @@ const gorelayerChainConfigTemplate = ` func (tr TestConfig) addChainToRelayer( action AddChainToRelayerAction, + target ExecutionTarget, verbose bool, ) { if !tr.useGorelayer { - tr.addChainToHermes(action, verbose) + tr.addChainToHermes(action, target, verbose) } else { - tr.addChainToGorelayer(action, verbose) + tr.addChainToGorelayer(action, target, verbose) } } func (tr TestConfig) addChainToGorelayer( action AddChainToRelayerAction, + target ExecutionTarget, verbose bool, ) { queryNodeIP := tr.getQueryNodeIP(action.Chain) @@ -774,8 +781,8 @@ func (tr TestConfig) addChainToGorelayer( rpcAddr, ) - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - bz, err := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "rly", "config", "init").CombinedOutput() + bz, err := target.ExecCommand( + "rly", "config", "init").CombinedOutput() if err != nil && !strings.Contains(string(bz), "config already exists") { log.Fatal(err, "\n", string(bz)) } @@ -783,24 +790,22 @@ func (tr TestConfig) addChainToGorelayer( chainConfigFileName := fmt.Sprintf("/root/%s_config.json", ChainId) bashCommand := fmt.Sprintf(`echo '%s' >> %s`, chainConfig, chainConfigFileName) - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - bz, err = exec.Command("docker", "exec", tr.containerConfig.InstanceName, "bash", "-c", + bz, err = target.ExecCommand("bash", "-c", bashCommand).CombinedOutput() if err != nil { log.Fatal(err, "\n", string(bz)) } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - addChainCommand := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "rly", "chains", "add", "--file", chainConfigFileName, string(ChainId)) + addChainCommand := target.ExecCommand("rly", "chains", "add", "--file", chainConfigFileName, string(ChainId)) executeCommand(addChainCommand, "add chain") - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - keyRestoreCommand := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "rly", "keys", "restore", string(ChainId), "default", tr.validatorConfigs[action.Validator].Mnemonic) + keyRestoreCommand := target.ExecCommand("rly", "keys", "restore", string(ChainId), "default", tr.validatorConfigs[action.Validator].Mnemonic) executeCommand(keyRestoreCommand, "restore keys") } func (tr TestConfig) addChainToHermes( action AddChainToRelayerAction, + target ExecutionTarget, verbose bool, ) { queryNodeIP := tr.getQueryNodeIP(action.Chain) @@ -821,10 +826,7 @@ func (tr TestConfig) addChainToHermes( bashCommand := fmt.Sprintf(`echo '%s' >> %s`, chainConfig, "/root/.hermes/config.toml") - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - bz, err := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "bash", "-c", - bashCommand, - ).CombinedOutput() + bz, err := target.ExecCommand("bash", "-c", bashCommand).CombinedOutput() if err != nil { log.Fatal(err, "\n", string(bz)) } @@ -839,16 +841,12 @@ func (tr TestConfig) addChainToHermes( saveMnemonicCommand := fmt.Sprintf(`echo '%s' > %s`, mnemonic, "/root/.hermes/mnemonic.txt") fmt.Println("Add to hermes", action.Validator) - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - bz, err = exec.Command("docker", "exec", tr.containerConfig.InstanceName, "bash", "-c", - saveMnemonicCommand, - ).CombinedOutput() + bz, err = target.ExecCommand("bash", "-c", saveMnemonicCommand).CombinedOutput() if err != nil { log.Fatal(err, "\n", string(bz)) } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - bz, err = exec.Command("docker", "exec", tr.containerConfig.InstanceName, "hermes", + bz, err = target.ExecCommand("hermes", "keys", "add", "--chain", string(tr.chainConfigs[action.Chain].ChainId), "--mnemonic-file", "/root/.hermes/mnemonic.txt", @@ -888,10 +886,11 @@ type AddIbcConnectionAction struct { func (tr TestConfig) addIbcConnection( action AddIbcConnectionAction, + target ExecutionTarget, verbose bool, ) { if !tr.useGorelayer { - tr.addIbcConnectionHermes(action, verbose) + tr.addIbcConnectionHermes(action, target, verbose) } else { tr.addIbcConnectionGorelayer(action, verbose) } @@ -957,10 +956,10 @@ type CreateIbcClientsAction struct { // otherwise, it would use client provided as CLI argument (-a-client) func (tr TestConfig) createIbcClientsHermes( action CreateIbcClientsAction, + target ExecutionTarget, verbose bool, ) { - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "hermes", + cmd := target.ExecCommand("hermes", "create", "connection", "--a-chain", string(tr.chainConfigs[action.ChainA].ChainId), "--b-chain", string(tr.chainConfigs[action.ChainB].ChainId), @@ -994,10 +993,10 @@ func (tr TestConfig) createIbcClientsHermes( func (tr TestConfig) addIbcConnectionHermes( action AddIbcConnectionAction, + target ExecutionTarget, verbose bool, ) { - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "hermes", + cmd := target.ExecCommand("hermes", "create", "connection", "--a-chain", string(tr.chainConfigs[action.ChainA].ChainId), "--a-client", "07-tendermint-"+fmt.Sprint(action.ClientA), @@ -1044,24 +1043,23 @@ type StartRelayerAction struct{} func (tr TestConfig) startRelayer( action StartRelayerAction, + target ExecutionTarget, verbose bool, ) { if tr.useGorelayer { - tr.startGorelayer(action, verbose) + tr.startGorelayer(action, target, verbose) } else { - tr.startHermes(action, verbose) + tr.startHermes(action, target, verbose) } } func (tr TestConfig) startGorelayer( action StartRelayerAction, + target ExecutionTarget, verbose bool, ) { // gorelayer start is running in detached mode - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", "-d", tr.containerConfig.InstanceName, "rly", - "start", - ) + cmd := target.ExecDetachedCommand("rly", "start") if err := cmd.Start(); err != nil { log.Fatal(err) @@ -1074,13 +1072,11 @@ func (tr TestConfig) startGorelayer( func (tr TestConfig) startHermes( action StartRelayerAction, + target ExecutionTarget, verbose bool, ) { // hermes start is running in detached mode - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", "-d", tr.containerConfig.InstanceName, "hermes", - "start", - ) + cmd := target.ExecDetachedCommand("hermes", "start") if err := cmd.Start(); err != nil { log.Fatal(err) @@ -1093,22 +1089,23 @@ func (tr TestConfig) startHermes( func (tr TestConfig) addIbcChannel( action AddIbcChannelAction, + target ExecutionTarget, verbose bool, ) { if tr.useGorelayer { - tr.addIbcChannelGorelayer(action, verbose) + tr.addIbcChannelGorelayer(action, target, verbose) } else { - tr.addIbcChannelHermes(action, verbose) + tr.addIbcChannelHermes(action, target, verbose) } } func (tr TestConfig) addIbcChannelGorelayer( action AddIbcChannelAction, + target ExecutionTarget, verbose bool, ) { pathName := tr.GetPathNameForGorelayer(action.ChainA, action.ChainB) - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "rly", + cmd := target.ExecCommand("rly", "transact", "channel", pathName, "--src-port", action.PortA, @@ -1122,6 +1119,7 @@ func (tr TestConfig) addIbcChannelGorelayer( func (tr TestConfig) addIbcChannelHermes( action AddIbcChannelAction, + target ExecutionTarget, verbose bool, ) { // if version is not specified, use the default version when creating ccv connections @@ -1131,8 +1129,7 @@ func (tr TestConfig) addIbcChannelHermes( chanVersion = tr.containerConfig.CcvVersion } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "hermes", + cmd := target.ExecCommand("hermes", "create", "channel", "--a-chain", string(tr.chainConfigs[action.ChainA].ChainId), "--a-connection", "connection-"+fmt.Sprint(action.ConnectionA), @@ -1185,14 +1182,14 @@ type TransferChannelCompleteAction struct { func (tr TestConfig) transferChannelComplete( action TransferChannelCompleteAction, + target ExecutionTarget, verbose bool, ) { if tr.useGorelayer { log.Fatal("transferChannelComplete is not implemented for rly") } - //#nosec G204 -- Bypass linter warning for spawning subprocess with chanOpenTryCmd arguments. - chanOpenTryCmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "hermes", + chanOpenTryCmd := target.ExecCommand("hermes", "tx", "chan-open-try", "--dst-chain", string(tr.chainConfigs[action.ChainB].ChainId), "--src-chain", string(tr.chainConfigs[action.ChainA].ChainId), @@ -1203,8 +1200,7 @@ func (tr TestConfig) transferChannelComplete( ) executeCommand(chanOpenTryCmd, "transferChanOpenTry") - //#nosec G204 -- Bypass linter warning for spawning subprocess with chanOpenAckCmd arguments. - chanOpenAckCmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "hermes", + chanOpenAckCmd := target.ExecCommand("hermes", "tx", "chan-open-ack", "--dst-chain", string(tr.chainConfigs[action.ChainA].ChainId), "--src-chain", string(tr.chainConfigs[action.ChainB].ChainId), @@ -1217,8 +1213,7 @@ func (tr TestConfig) transferChannelComplete( executeCommand(chanOpenAckCmd, "transferChanOpenAck") - //#nosec G204 -- Bypass linter warning for spawning subprocess with chanOpenConfirmCmd arguments. - chanOpenConfirmCmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "hermes", + chanOpenConfirmCmd := target.ExecCommand("hermes", "tx", "chan-open-confirm", "--dst-chain", string(tr.chainConfigs[action.ChainB].ChainId), "--src-chain", string(tr.chainConfigs[action.ChainA].ChainId), @@ -1273,24 +1268,25 @@ type RelayPacketsAction struct { func (tr TestConfig) relayPackets( action RelayPacketsAction, + target ExecutionTarget, verbose bool, ) { if tr.useGorelayer { - tr.relayPacketsGorelayer(action, verbose) + tr.relayPacketsGorelayer(action, target, verbose) } else { - tr.relayPacketsHermes(action, verbose) + tr.relayPacketsHermes(action, target, verbose) } } func (tr TestConfig) relayPacketsGorelayer( action RelayPacketsAction, + target ExecutionTarget, verbose bool, ) { pathName := tr.GetPathNameForGorelayer(action.ChainA, action.ChainB) // rly transact relay-packets [path-name] --channel [channel-id] - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "rly", "transact", "flush", + cmd := target.ExecCommand("rly", "transact", "flush", pathName, "channel-"+fmt.Sprint(action.Channel), ) @@ -1308,11 +1304,11 @@ func (tr TestConfig) relayPacketsGorelayer( func (tr TestConfig) relayPacketsHermes( action RelayPacketsAction, + target ExecutionTarget, verbose bool, ) { // hermes clear packets ibc0 transfer channel-13 - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "hermes", "clear", "packets", + cmd := target.ExecCommand("hermes", "clear", "packets", "--chain", string(tr.chainConfigs[action.ChainA].ChainId), "--port", action.Port, "--channel", "channel-"+fmt.Sprint(action.Channel), @@ -1339,6 +1335,7 @@ type RelayRewardPacketsToProviderAction struct { func (tr TestConfig) relayRewardPacketsToProvider( action RelayRewardPacketsToProviderAction, + target ExecutionTarget, verbose bool, ) { blockPerDistribution, _ := strconv.ParseUint(strings.Trim(tr.getParam(action.ConsumerChain, Param{Subspace: "ccvconsumer", Key: "BlocksPerDistributionTransmission"}), "\""), 10, 64) @@ -1347,7 +1344,7 @@ func (tr TestConfig) relayRewardPacketsToProvider( tr.waitBlocks(action.ConsumerChain, uint(blockPerDistribution-currentBlock+1), 60*time.Second) } - tr.relayPackets(RelayPacketsAction{ChainA: action.ConsumerChain, ChainB: action.ProviderChain, Port: action.Port, Channel: action.Channel}, verbose) + tr.relayPackets(RelayPacketsAction{ChainA: action.ConsumerChain, ChainB: action.ProviderChain, Port: action.Port, Channel: action.Channel}, target, verbose) tr.waitBlocks(action.ProviderChain, 1, 10*time.Second) } @@ -1360,6 +1357,7 @@ type DelegateTokensAction struct { func (tr TestConfig) delegateTokens( action DelegateTokensAction, + target ExecutionTarget, verbose bool, ) { toValCfg := tr.validatorConfigs[action.To] @@ -1367,8 +1365,8 @@ func (tr TestConfig) delegateTokens( if action.Chain != ChainID("provi") && toValCfg.UseConsumerKey { delegateAddr = toValCfg.ConsumerValoperAddress } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[action.Chain].BinaryName, + + cmd := target.ExecCommand(tr.chainConfigs[action.Chain].BinaryName, "tx", "staking", "delegate", delegateAddr, @@ -1404,6 +1402,7 @@ type UnbondTokensAction struct { func (tr TestConfig) unbondTokens( action UnbondTokensAction, + target ExecutionTarget, verbose bool, ) { unbondFrom := tr.validatorConfigs[action.UnbondFrom].ValoperAddress @@ -1411,8 +1410,7 @@ func (tr TestConfig) unbondTokens( unbondFrom = tr.validatorConfigs[action.UnbondFrom].ConsumerValoperAddress } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[action.Chain].BinaryName, + cmd := target.ExecCommand(tr.chainConfigs[action.Chain].BinaryName, "tx", "staking", "unbond", unbondFrom, @@ -1449,6 +1447,7 @@ type CancelUnbondTokensAction struct { func (tr TestConfig) cancelUnbondTokens( action CancelUnbondTokensAction, + target ExecutionTarget, verbose bool, ) { validator := tr.validatorConfigs[action.Validator].ValoperAddress @@ -1457,8 +1456,7 @@ func (tr TestConfig) cancelUnbondTokens( } // get creation-height from state - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[action.Chain].BinaryName, + cmd := target.ExecCommand(tr.chainConfigs[action.Chain].BinaryName, "q", "staking", "unbonding-delegation", tr.validatorConfigs[action.Delegator].DelAddress, validator, @@ -1479,8 +1477,7 @@ func (tr TestConfig) cancelUnbondTokens( log.Fatal("invalid creation height") } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd = exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[action.Chain].BinaryName, + cmd = target.ExecCommand(tr.chainConfigs[action.Chain].BinaryName, "tx", "staking", "cancel-unbond", validator, fmt.Sprint(action.Amount)+`stake`, @@ -1516,7 +1513,7 @@ type RedelegateTokensAction struct { Amount uint } -func (tr TestConfig) redelegateTokens(action RedelegateTokensAction, verbose bool) { +func (tr TestConfig) redelegateTokens(action RedelegateTokensAction, target ExecutionTarget, verbose bool) { srcCfg := tr.validatorConfigs[action.Src] dstCfg := tr.validatorConfigs[action.Dst] @@ -1529,9 +1526,8 @@ func (tr TestConfig) redelegateTokens(action RedelegateTokensAction, verbose boo if action.Chain != ChainID("provi") && dstCfg.UseConsumerKey { redelegateDst = dstCfg.ConsumerValoperAddress } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", - tr.containerConfig.InstanceName, + + cmd := target.ExecCommand( tr.chainConfigs[action.Chain].BinaryName, "tx", "staking", "redelegate", @@ -1580,17 +1576,17 @@ func (tr TestConfig) getValidatorKeyAddressFromString(keystring string) string { return key.Address } -func (tr TestConfig) invokeDowntimeSlash(action DowntimeSlashAction, verbose bool) { +func (tr TestConfig) invokeDowntimeSlash(action DowntimeSlashAction, target ExecutionTarget, verbose bool) { // Bring validator down - tr.setValidatorDowntime(action.Chain, action.Validator, true, verbose) + tr.setValidatorDowntime(action.Chain, action.Validator, true, target, verbose) // Wait appropriate amount of blocks for validator to be slashed tr.waitBlocks(action.Chain, 10, 3*time.Minute) // Bring validator back up - tr.setValidatorDowntime(action.Chain, action.Validator, false, verbose) + tr.setValidatorDowntime(action.Chain, action.Validator, false, target, verbose) } // Sets validator downtime by setting the virtual ethernet interface of a node to "up" or "down" -func (tr TestConfig) setValidatorDowntime(chain ChainID, validator ValidatorID, down, verbose bool) { +func (tr TestConfig) setValidatorDowntime(chain ChainID, validator ValidatorID, down bool, target ExecutionTarget, verbose bool) { var lastArg string if down { lastArg = "down" @@ -1611,11 +1607,7 @@ func (tr TestConfig) setValidatorDowntime(chain ChainID, validator ValidatorID, return } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command( - "docker", - "exec", - tr.containerConfig.InstanceName, + cmd := target.ExecCommand( "ip", "link", "set", @@ -1655,13 +1647,11 @@ type UnjailValidatorAction struct { } // Sends an unjail transaction to the provider chain -func (tr TestConfig) unjailValidator(action UnjailValidatorAction, verbose bool) { +func (tr TestConfig) unjailValidator(action UnjailValidatorAction, target ExecutionTarget, verbose bool) { // wait until downtime_jail_duration has elapsed, to make sure the validator can be unjailed tr.WaitTime(61 * time.Second) - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", - tr.containerConfig.InstanceName, + cmd := target.ExecCommand( tr.chainConfigs[action.Provider].BinaryName, "tx", "slashing", "unjail", // Validator is sender here @@ -1696,6 +1686,7 @@ type RegisterRepresentativeAction struct { func (tr TestConfig) registerRepresentative( action RegisterRepresentativeAction, + target ExecutionTarget, verbose bool, ) { var wg sync.WaitGroup @@ -1705,8 +1696,7 @@ func (tr TestConfig) registerRepresentative( go func(val ValidatorID, stake uint) { defer wg.Done() - //#nosec G204 -- Bypass linter warning for spawning subprocess with pubKeycmd arguments. - pubKeycmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[action.Chain].BinaryName, + pubKeycmd := target.ExecCommand(tr.chainConfigs[action.Chain].BinaryName, "tendermint", "show-validator", `--home`, tr.getValidatorHome(action.Chain, val), ) @@ -1716,8 +1706,7 @@ func (tr TestConfig) registerRepresentative( log.Fatal(err, "\n", string(bzPubKey)) } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - bz, err := exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[action.Chain].BinaryName, + bz, err := target.ExecCommand(tr.chainConfigs[action.Chain].BinaryName, "tx", "staking", "create-validator", `--amount`, fmt.Sprint(stake)+"stake", `--pubkey`, string(bzPubKey), @@ -1751,7 +1740,7 @@ type SubmitChangeRewardDenomsProposalAction struct { From ValidatorID } -func (tr TestConfig) submitChangeRewardDenomsProposal(action SubmitChangeRewardDenomsProposalAction, verbose bool) { +func (tr TestConfig) submitChangeRewardDenomsProposal(action SubmitChangeRewardDenomsProposalAction, target ExecutionTarget, verbose bool) { providerChain := tr.chainConfigs[ChainID("provi")] prop := client.ChangeRewardDenomsProposalJSON{ @@ -1775,17 +1764,15 @@ func (tr TestConfig) submitChangeRewardDenomsProposal(action SubmitChangeRewardD log.Fatal("prop json contains single quote") } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - bz, err = exec.Command("docker", "exec", tr.containerConfig.InstanceName, + bz, err = target.ExecCommand( "/bin/bash", "-c", fmt.Sprintf(`echo '%s' > %s`, jsonStr, "/change-reward-denoms-proposal.json")).CombinedOutput() if err != nil { log.Fatal(err, "\n", string(bz)) } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. // CHANGE REWARDS DENOM PROPOSAL - bz, err = exec.Command("docker", "exec", tr.containerConfig.InstanceName, providerChain.BinaryName, + bz, err = target.ExecCommand(providerChain.BinaryName, "tx", "gov", "submit-legacy-proposal", "change-reward-denoms", "/change-reward-denoms-proposal.json", `--from`, `validator`+fmt.Sprint(action.From), `--chain-id`, string(providerChain.ChainId), @@ -1822,13 +1809,15 @@ type DoublesignSlashAction struct { func (tr TestConfig) invokeDoublesignSlash( action DoublesignSlashAction, + target ExecutionTarget, verbose bool, ) { if !tr.useCometmock { chainConfig := tr.chainConfigs[action.Chain] - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - bz, err := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "/bin/bash", - "/testnet-scripts/cause-doublesign.sh", chainConfig.BinaryName, string(action.Validator), + isConsumer := false + doubleSignScript := target.GetTestScriptPath(isConsumer, "cause-doublesign.sh") + bz, err := target.ExecCommand("/bin/bash", + doubleSignScript, chainConfig.BinaryName, string(action.Validator), string(chainConfig.ChainId), chainConfig.IpPrefix).CombinedOutput() if err != nil { log.Fatal(err, "\n", string(bz)) @@ -1934,7 +1923,7 @@ type AssignConsumerPubKeyAction struct { ExpectedError string } -func (tr TestConfig) assignConsumerPubKey(action AssignConsumerPubKeyAction, verbose bool) { +func (tr TestConfig) assignConsumerPubKey(action AssignConsumerPubKeyAction, target ExecutionTarget, verbose bool) { valCfg := tr.validatorConfigs[action.Validator] // Note: to get error response reported back from this command '--gas auto' needs to be set. @@ -1955,9 +1944,7 @@ func (tr TestConfig) assignConsumerPubKey(action AssignConsumerPubKeyAction, ver gas, ) - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", - tr.containerConfig.InstanceName, + cmd := target.ExecCommand( "/bin/bash", "-c", assignKey, ) @@ -1984,9 +1971,10 @@ func (tr TestConfig) assignConsumerPubKey(action AssignConsumerPubKeyAction, ver // node was started with provider key // we swap the nodes's keys for consumer keys and restart it if action.ReconfigureNode { - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - configureNodeCmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "/bin/bash", - "/testnet-scripts/reconfigure-node.sh", tr.chainConfigs[action.Chain].BinaryName, + isConsumer := tr.chainConfigs[action.Chain].BinaryName != "interchain-security-pd" + reconfigureScript := target.GetTestScriptPath(isConsumer, "reconfigure-node.sh") + configureNodeCmd := target.ExecCommand("/bin/bash", + reconfigureScript, tr.chainConfigs[action.Chain].BinaryName, string(action.Validator), string(action.Chain), tr.chainConfigs[action.Chain].IpPrefix, valCfg.IpSuffix, valCfg.ConsumerMnemonic, valCfg.ConsumerPrivValidatorKey, @@ -2106,12 +2094,12 @@ type StartConsumerEvidenceDetectorAction struct { func (tc TestConfig) startConsumerEvidenceDetector( action StartConsumerEvidenceDetectorAction, + target ExecutionTarget, verbose bool, ) { chainConfig := tc.chainConfigs[action.Chain] // run in detached mode so it will keep running in the background - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - bz, err := exec.Command("docker", "exec", "-d", tc.containerConfig.InstanceName, + bz, err := target.ExecDetachedCommand( "hermes", "evidence", "--chain", string(chainConfig.ChainId)).CombinedOutput() if err != nil { log.Fatal(err, "\n", string(bz)) diff --git a/tests/e2e/actions_sovereign_chain.go b/tests/e2e/actions_sovereign_chain.go index 9a5bb0f182..187c9036cc 100644 --- a/tests/e2e/actions_sovereign_chain.go +++ b/tests/e2e/actions_sovereign_chain.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "log" - "os/exec" "time" ) @@ -20,6 +19,7 @@ type StartSovereignChainAction struct { // upgrades are simpler with a single validator node since only one node needs to be upgraded func (tr TestConfig) startSovereignChain( action StartSovereignChainAction, + target ExecutionTarget, verbose bool, ) { chainConfig := tr.chainConfigs["sover"] @@ -68,12 +68,11 @@ func (tr TestConfig) startSovereignChain( genesisChanges = chainConfig.GenesisChanges } - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "/bin/bash", - "/testnet-scripts/start-sovereign.sh", chainConfig.BinaryName, string(vals), + isConsumer := chainConfig.BinaryName != "interchain-security-pd" + testScriptPath := target.GetTestScriptPath(isConsumer, "start-sovereign.sh") + cmd := target.ExecCommand("/bin/bash", testScriptPath, string(vals), string(chainConfig.ChainId), chainConfig.IpPrefix, genesisChanges, - tr.tendermintConfigOverride, - ) + tr.tendermintConfigOverride) cmdReader, err := cmd.StdoutPipe() if err != nil { @@ -102,7 +101,7 @@ func (tr TestConfig) startSovereignChain( tr.addChainToRelayer(AddChainToRelayerAction{ Chain: action.Chain, Validator: action.Validators[0].Id, - }, verbose) + }, target, verbose) } type LegacyUpgradeProposalAction struct { @@ -112,7 +111,7 @@ type LegacyUpgradeProposalAction struct { UpgradeHeight uint64 } -func (tr *TestConfig) submitLegacyUpgradeProposal(action LegacyUpgradeProposalAction, verbose bool) { +func (tr *TestConfig) submitLegacyUpgradeProposal(action LegacyUpgradeProposalAction, target ExecutionTarget, verbose bool) { submit := fmt.Sprintf( `%s tx gov submit-legacy-proposal software-upgrade %s \ --title %s \ @@ -137,12 +136,7 @@ func (tr *TestConfig) submitLegacyUpgradeProposal(action LegacyUpgradeProposalAc tr.getValidatorHome(ChainID("sover"), action.Proposer), tr.getValidatorNode(ChainID("sover"), action.Proposer), ) - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - cmd := exec.Command("docker", "exec", - tr.containerConfig.InstanceName, - "/bin/bash", "-c", - submit, - ) + cmd := target.ExecCommand("/bin/bash", "-c", submit) if verbose { fmt.Println("submitUpgradeProposal cmd:", cmd.String()) diff --git a/tests/e2e/test_driver.go b/tests/e2e/test_driver.go index 98430bb12c..02bbdf0039 100644 --- a/tests/e2e/test_driver.go +++ b/tests/e2e/test_driver.go @@ -45,6 +45,7 @@ func (pr *DefaultDriver) Run(steps []Step, target ExecutionTarget, verbose bool) return nil } +// runStep executes an action and evaluates the result against expected state func (pr *DefaultDriver) runStep(step Step) error { err := pr.runAction(step.Action) if err != nil { @@ -71,86 +72,86 @@ func (pr *DefaultDriver) GetState(modelState State) { pr.testCfg.getState(modelState) } -func (pr *DefaultDriver) runAction(action interface{}) error { +func (td *DefaultDriver) runAction(action interface{}) error { switch action := action.(type) { case StartChainAction: - pr.testCfg.startChain(action, pr.target, pr.verbose) + td.testCfg.startChain(action, td.target, td.verbose) case StartSovereignChainAction: - pr.testCfg.startSovereignChain(action, pr.verbose) + td.testCfg.startSovereignChain(action, td.target, td.verbose) case LegacyUpgradeProposalAction: - pr.testCfg.submitLegacyUpgradeProposal(action, pr.verbose) + td.testCfg.submitLegacyUpgradeProposal(action, td.target, td.verbose) case WaitUntilBlockAction: - pr.testCfg.waitUntilBlockOnChain(action) + td.testCfg.waitUntilBlockOnChain(action) case ChangeoverChainAction: - pr.testCfg.changeoverChain(action, pr.verbose) + td.testCfg.changeoverChain(action, td.target, td.verbose) case SendTokensAction: - pr.testCfg.sendTokens(action, pr.verbose) + td.testCfg.sendTokens(action, td.target, td.verbose) case SubmitTextProposalAction: - pr.testCfg.submitTextProposal(action, pr.verbose) + td.testCfg.submitTextProposal(action, td.target, td.verbose) case SubmitConsumerAdditionProposalAction: - pr.testCfg.submitConsumerAdditionProposal(action, pr.verbose) + td.testCfg.submitConsumerAdditionProposal(action, td.target, td.verbose) case SubmitConsumerRemovalProposalAction: - pr.testCfg.submitConsumerRemovalProposal(action, pr.verbose) + td.testCfg.submitConsumerRemovalProposal(action, td.target, td.verbose) case SubmitParamChangeLegacyProposalAction: - pr.testCfg.submitParamChangeProposal(action, pr.verbose) + td.testCfg.submitParamChangeProposal(action, td.target, td.verbose) case VoteGovProposalAction: - pr.testCfg.voteGovProposal(action, pr.verbose) + td.testCfg.voteGovProposal(action, td.target, td.verbose) case StartConsumerChainAction: - pr.testCfg.startConsumerChain(action, pr.target, pr.verbose) + td.testCfg.startConsumerChain(action, td.target, td.verbose) case AddChainToRelayerAction: - pr.testCfg.addChainToRelayer(action, pr.verbose) + td.testCfg.addChainToRelayer(action, td.target, td.verbose) case CreateIbcClientsAction: - pr.testCfg.createIbcClientsHermes(action, pr.verbose) + td.testCfg.createIbcClientsHermes(action, td.target, td.verbose) case AddIbcConnectionAction: - pr.testCfg.addIbcConnection(action, pr.verbose) + td.testCfg.addIbcConnection(action, td.target, td.verbose) case AddIbcChannelAction: - pr.testCfg.addIbcChannel(action, pr.verbose) + td.testCfg.addIbcChannel(action, td.target, td.verbose) case TransferChannelCompleteAction: - pr.testCfg.transferChannelComplete(action, pr.verbose) + td.testCfg.transferChannelComplete(action, td.target, td.verbose) case RelayPacketsAction: - pr.testCfg.relayPackets(action, pr.verbose) + td.testCfg.relayPackets(action, td.target, td.verbose) case RelayRewardPacketsToProviderAction: - pr.testCfg.relayRewardPacketsToProvider(action, pr.verbose) + td.testCfg.relayRewardPacketsToProvider(action, td.target, td.verbose) case DelegateTokensAction: - pr.testCfg.delegateTokens(action, pr.verbose) + td.testCfg.delegateTokens(action, td.target, td.verbose) case UnbondTokensAction: - pr.testCfg.unbondTokens(action, pr.verbose) + td.testCfg.unbondTokens(action, td.target, td.verbose) case CancelUnbondTokensAction: - pr.testCfg.cancelUnbondTokens(action, pr.verbose) + td.testCfg.cancelUnbondTokens(action, td.target, td.verbose) case RedelegateTokensAction: - pr.testCfg.redelegateTokens(action, pr.verbose) + td.testCfg.redelegateTokens(action, td.target, td.verbose) case DowntimeSlashAction: - pr.testCfg.invokeDowntimeSlash(action, pr.verbose) + td.testCfg.invokeDowntimeSlash(action, td.target, td.verbose) case UnjailValidatorAction: - pr.testCfg.unjailValidator(action, pr.verbose) + td.testCfg.unjailValidator(action, td.target, td.verbose) case DoublesignSlashAction: - pr.testCfg.invokeDoublesignSlash(action, pr.verbose) + td.testCfg.invokeDoublesignSlash(action, td.target, td.verbose) case LightClientAmnesiaAttackAction: - pr.testCfg.lightClientAmnesiaAttack(action, pr.verbose) + td.testCfg.lightClientAmnesiaAttack(action, td.verbose) case LightClientEquivocationAttackAction: - pr.testCfg.lightClientEquivocationAttack(action, pr.verbose) + td.testCfg.lightClientEquivocationAttack(action, td.verbose) case LightClientLunaticAttackAction: - pr.testCfg.lightClientLunaticAttack(action, pr.verbose) + td.testCfg.lightClientLunaticAttack(action, td.verbose) case RegisterRepresentativeAction: - pr.testCfg.registerRepresentative(action, pr.verbose) + td.testCfg.registerRepresentative(action, td.target, td.verbose) case AssignConsumerPubKeyAction: - pr.testCfg.assignConsumerPubKey(action, pr.verbose) + td.testCfg.assignConsumerPubKey(action, td.target, td.verbose) case SlashMeterReplenishmentAction: - pr.testCfg.waitForSlashMeterReplenishment(action, pr.verbose) + td.testCfg.waitForSlashMeterReplenishment(action, td.verbose) case WaitTimeAction: - pr.testCfg.waitForTime(action, pr.verbose) + td.testCfg.waitForTime(action, td.verbose) case StartRelayerAction: - pr.testCfg.startRelayer(action, pr.verbose) + td.testCfg.startRelayer(action, td.target, td.verbose) case ForkConsumerChainAction: - pr.testCfg.forkConsumerChain(action, pr.verbose) + td.testCfg.forkConsumerChain(action, td.verbose) case UpdateLightClientAction: - pr.testCfg.updateLightClient(action, pr.verbose) + td.testCfg.updateLightClient(action, td.verbose) case StartConsumerEvidenceDetectorAction: - pr.testCfg.startConsumerEvidenceDetector(action, pr.verbose) + td.testCfg.startConsumerEvidenceDetector(action, td.target, td.verbose) case SubmitChangeRewardDenomsProposalAction: - pr.testCfg.submitChangeRewardDenomsProposal(action, pr.verbose) + td.testCfg.submitChangeRewardDenomsProposal(action, td.target, td.verbose) default: - log.Fatalf("unknown action in testRun %s: %#v", pr.testCfg.name, action) + log.Fatalf("unknown action in testRun %s: %#v", td.testCfg.name, action) } return nil } diff --git a/tests/e2e/test_target.go b/tests/e2e/test_target.go index 48f7a56fa2..62fdf99ce9 100644 --- a/tests/e2e/test_target.go +++ b/tests/e2e/test_target.go @@ -4,6 +4,7 @@ import ( "bufio" "fmt" "log" + "os" "os/exec" "strings" ) @@ -11,8 +12,9 @@ import ( type ExecutionTarget interface { GetTargetType() string GetLogs(path string) []byte - GetStartChainScript(isProvider bool) string + GetTestScriptPath(isConsumer bool, script string) string ExecCommand(name string, arg ...string) *exec.Cmd + ExecDetachedCommand(name string, args ...string) *exec.Cmd Start() error Stop() error Build() error @@ -132,21 +134,28 @@ func (dc *DockerContainer) ExecCommand(name string, arg ...string) *exec.Cmd { return exec.Command("docker", args...) } -// Get star-chain script to be used on target for a specific chain type +// ExecDetachedCommand returns the command struct to execute the named program with +// given arguments on the current target (docker container) in _detached_ mode +func (dc *DockerContainer) ExecDetachedCommand(name string, arg ...string) *exec.Cmd { + args := []string{"exec", "-d", dc.containerCfg.InstanceName, name} + args = append(args, arg...) + return exec.Command("docker", args...) +} + +// Get path to testnet-script on target for a specific chain type // Needed for different consumer/provider versions staged in one container -func (dc *DockerContainer) GetStartChainScript(isConsumer bool) string { - startChainScript := "/testnet-scripts/start-chain.sh" +func (dc *DockerContainer) GetTestScriptPath(isConsumer bool, script string) string { + path := "/testnet-scripts" if dc.targetConfig.providerVersion != "" && !isConsumer { - log.Printf("Using start-chain script for provider version '%s'", dc.targetConfig.providerVersion) - startChainScript = "/provider/testnet-scripts/start-chain.sh" + log.Printf("Using script path for provider version '%s'", dc.targetConfig.providerVersion) + path = "/provider/testnet-scripts" } if dc.targetConfig.consumerVersion != "" && isConsumer { - log.Printf("Using start-chain script for consumer version '%s'", dc.targetConfig.consumerVersion) - startChainScript = "/consumer/testnet-scripts/start-chain.sh" + log.Printf("Using script path for consumer version '%s'", dc.targetConfig.consumerVersion) + path = "/consumer/testnet-scripts" } - return startChainScript - + return strings.Join([]string{path, script}, string(os.PathSeparator)) } // Startup the container @@ -158,9 +167,10 @@ func (dc *DockerContainer) Start() error { // Run new test container instance with extended privileges. // Extended privileges are granted to the container here to allow for network namespace manipulation (bringing a node up/down) // See: https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities + beaconScript := dc.GetTestScriptPath(false, "beacon.sh") cmd := exec.Command("docker", "run", "--name", dc.containerCfg.InstanceName, "--cap-add=NET_ADMIN", "--privileged", dc.ImageName, - "/bin/bash", "/testnet-scripts/beacon.sh") + "/bin/bash", beaconScript) cmdReader, err := cmd.StdoutPipe() if err != nil { From cb8b83f3c6278b9f121792ac2ebe6cd6a475e820 Mon Sep 17 00:00:00 2001 From: Bernd Mueller Date: Wed, 10 Jan 2024 11:12:40 +0100 Subject: [PATCH 05/16] Cleanup docker files --- Dockerfile-Consumer | 72 ----------------- Dockerfile-Provider | 70 ----------------- Dockerfile-Combined => Dockerfile.combined | 0 tests/e2e/test_target.go | 6 +- tests/e2e/testnet-scripts/start-docker.sh | 90 ---------------------- 5 files changed, 3 insertions(+), 235 deletions(-) delete mode 100644 Dockerfile-Consumer delete mode 100644 Dockerfile-Provider rename Dockerfile-Combined => Dockerfile.combined (100%) delete mode 100755 tests/e2e/testnet-scripts/start-docker.sh diff --git a/Dockerfile-Consumer b/Dockerfile-Consumer deleted file mode 100644 index 19de7106d1..0000000000 --- a/Dockerfile-Consumer +++ /dev/null @@ -1,72 +0,0 @@ -# syntax=docker/dockerfile:1 - -# Consumer image to be used for compatibility tests with provider from workspace -# use docker's build argument --build-arg to overwrite the defaults -# e.g. docker build --build-arg CONSUMER_TAG=v3.1.0 -ARG CONSUMER_VERSION -ARG CONSUMER_IMAGE="cosmos-ics" - - -FROM golang:1.20-alpine AS is-builder - -ENV PACKAGES curl make git libc-dev bash gcc linux-headers -RUN apk add --no-cache $PACKAGES - -ENV CGO_ENABLED=0 -ENV GOOS=linux -ENV GOFLAGS="-buildvcs=false" - -# cache go modules - done before the files are copied to allow docker to better cache -COPY go.mod /go.mod -COPY go.sum /go.sum -RUN go mod download - - -# Copy in the repo under test -ADD . /interchain-security -WORKDIR /interchain-security - -# Do not specify version here. It leads to odd replacement behavior -RUN if [ -d "./cosmos-sdk" ]; then go mod edit -replace github.com/cosmos/cosmos-sdk=./cosmos-sdk; fi -RUN go mod tidy - -# Install interchain security binary -RUN make install - - -# The image from where the consumer implementation will be used -# Defaults to -FROM --platform=linux/amd64 ${CONSUMER_IMAGE}:${CONSUMER_VERSION} AS consumer - -# Get Hermes build -FROM --platform=linux/amd64 otacrew/hermes-ics:evidence-cmd AS hermes-builder - -# Get CometMock -FROM ghcr.io/informalsystems/cometmock:v0.37.x as cometmock-builder - -# Get GoRelayer -FROM ghcr.io/informalsystems/relayer-no-gas-sim:v2.3.0-rc4-no-gas-sim AS gorelayer-builder - -FROM --platform=linux/amd64 fedora:39 -RUN dnf update -y -RUN dnf install -y which iproute iputils procps-ng vim-minimal tmux net-tools htop jq -USER root - -COPY --from=hermes-builder /usr/bin/hermes /usr/local/bin/ -COPY --from=cometmock-builder /usr/local/bin/cometmock /usr/local/bin/cometmock -COPY --from=gorelayer-builder /bin/rly /usr/local/bin/ - -# Copy consumer from specified image -COPY --from=consumer /usr/local/bin/interchain-security-cd /usr/local/bin/interchain-security-cd -COPY --from=consumer /usr/local/bin/interchain-security-cdd /usr/local/bin/interchain-security-cdd -COPY --from=consumer /usr/local/bin/interchain-security-sd /usr/local/bin/interchain-security-sd -COPY --from=consumer /testnet-scripts /consumer/testnet-scripts - -# Copy provider from local build -COPY --from=is-builder /go/bin/interchain-security-pd /usr/local/bin/interchain-security-pd - -# Copy in the shell scripts that run the testnet -ADD ./tests/e2e/testnet-scripts /testnet-scripts - -# Copy in the hermes config -ADD ./tests/e2e/testnet-scripts/hermes-config.toml /root/.hermes/config.toml diff --git a/Dockerfile-Provider b/Dockerfile-Provider deleted file mode 100644 index ba231f35dc..0000000000 --- a/Dockerfile-Provider +++ /dev/null @@ -1,70 +0,0 @@ -# syntax=docker/dockerfile:1 - -# Provider image to be used for compatibility tests with consumer from workspace. -# use docker's build argument --build-arg to overwrite the defaults -# e.g. docker build --build-arg CONSUMER_TAG=v3.1.0 -ARG PROVIDER_VERSION="latest" -ARG PROVIDER_IMAGE="cosmos-ics" - -FROM golang:1.20-alpine AS is-builder - -ENV PACKAGES curl make git libc-dev bash gcc linux-headers -RUN apk add --no-cache $PACKAGES - -ENV CGO_ENABLED=0 -ENV GOOS=linux -ENV GOFLAGS="-buildvcs=false" - -# cache go modules - done before the files are copied to allow docker to better cache -COPY go.mod /go.mod -COPY go.sum /go.sum -RUN go mod download - -# Copy in the repo under test -ADD . /interchain-security -WORKDIR /interchain-security - -# Do not specify version here. It leads to odd replacement behavior -RUN if [ -d "./cosmos-sdk" ]; then go mod edit -replace github.com/cosmos/cosmos-sdk=./cosmos-sdk; fi -RUN go mod tidy - -# Install interchain security binary -RUN make install - - -# The image from where the consumer implementation will be used -# Defaults to -FROM --platform=linux/amd64 ${PROVIDER_IMAGE}:${PROVIDER_VERSION} AS provider - -# Get Hermes build -FROM --platform=linux/amd64 otacrew/hermes-ics:evidence-cmd AS hermes-builder - -# Get CometMock -FROM ghcr.io/informalsystems/cometmock:v0.37.x as cometmock-builder - -# Get GoRelayer -FROM ghcr.io/informalsystems/relayer-no-gas-sim:v2.3.0-rc4-no-gas-sim AS gorelayer-builder - -FROM --platform=linux/amd64 fedora:39 -RUN dnf update -y -RUN dnf install -y which iproute iputils procps-ng vim-minimal tmux net-tools htop jq -USER root - -COPY --from=hermes-builder /usr/bin/hermes /usr/local/bin/ -COPY --from=cometmock-builder /usr/local/bin/cometmock /usr/local/bin/cometmock -COPY --from=gorelayer-builder /bin/rly /usr/local/bin/ - -# Copy provider from specified image -COPY --from=provider /usr/local/bin/interchain-security-pd /usr/local/bin/interchain-security-pd -COPY --from=provider /testnet-scripts /provider/testnet-scripts - -# Copy provider from local build -COPY --from=is-builder /go/bin/interchain-security-cd /usr/local/bin/interchain-security-cd -COPY --from=is-builder /go/bin/interchain-security-cdd /usr/local/bin/interchain-security-cdd -COPY --from=is-builder /go/bin/interchain-security-sd /usr/local/bin/interchain-security-sd - -# Copy in the shell scripts that run the testnet -ADD ./tests/e2e/testnet-scripts /testnet-scripts - -# Copy in the hermes config -ADD ./tests/e2e/testnet-scripts/hermes-config.toml /root/.hermes/config.toml diff --git a/Dockerfile-Combined b/Dockerfile.combined similarity index 100% rename from Dockerfile-Combined rename to Dockerfile.combined diff --git a/tests/e2e/test_target.go b/tests/e2e/test_target.go index 62fdf99ce9..2e05fb1edd 100644 --- a/tests/e2e/test_target.go +++ b/tests/e2e/test_target.go @@ -99,7 +99,7 @@ func (dc *DockerContainer) Build() error { combinedImageName := fmt.Sprintf("cosmos-ics-combined:%s_%s", strings.Split(providerImageName, ":")[1], strings.Split(consumerImageName, ":")[1]) - cmd := exec.Command("docker", "build", "-f", "Dockerfile-Combined", + cmd := exec.Command("docker", "build", "-f", "Dockerfile.combined", "--build-arg", fmt.Sprintf("PROVIDER_IMAGE=%s", providerImageName), "--build-arg", fmt.Sprintf("CONSUMER_IMAGE=%s", consumerImageName), "-t", combinedImageName, @@ -160,7 +160,7 @@ func (dc *DockerContainer) GetTestScriptPath(isConsumer bool, script string) str // Startup the container func (dc *DockerContainer) Start() error { - fmt.Println("@@@@ starting existing containers", dc.containerCfg.InstanceName) + fmt.Println("Starting container: ", dc.containerCfg.InstanceName) // Remove existing containers from previous runs dc.Stop() @@ -203,7 +203,7 @@ func (dc *DockerContainer) Start() error { // Stop will stop the container and remove it func (dc *DockerContainer) Stop() error { - fmt.Println("@@@@ stopping existing containers", dc.containerCfg.InstanceName) + fmt.Println("Stopping existing containers: ", dc.containerCfg.InstanceName) //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. cmd := exec.Command("docker", "stop", dc.containerCfg.InstanceName) bz, err := cmd.CombinedOutput() diff --git a/tests/e2e/testnet-scripts/start-docker.sh b/tests/e2e/testnet-scripts/start-docker.sh deleted file mode 100755 index 59b5800b6f..0000000000 --- a/tests/e2e/testnet-scripts/start-docker.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash - -# If -e is not set then if the build fails, it will use the old container, resulting in a very confusing debugging situation -# Setting -e makes it error out if the build fails -set -eux - -usage() { - echo """ -Usage: $0 [-c ] [-g ] [-s ] [-h] container-name instance-name - [-c consumer-version] : the consumer version to be used in the container. - Mutial exclusive to -p - [-p provider-version] : the provider version to be used in the container. - Mutial exclusive to -c - [-g gaia-tag] : use gaia as provider with specified version - [-s SDK-path] : use custom SDK - [-h] : print this help -""" -} - -## Process the optional arguments if any -while getopts ":c:p:g:s:h" flag - do - case "${flag}" in - c) CONSUMER_VERSION=${OPTARG};; - p) PROVIDER_VERSION=${OPTARG};; - g) USE_GAIA_TAG=${OPTARG};; - s) LOCAL_SDK_PATH=${OPTARG};; - h) SHOW_HELP=1;; - *) usage;; - esac - done - -if [ ${SHOW_HELP+x} ]; then - usage - exit 0 -fi - -shift $((OPTIND - 1)) - -# Set positional arguments -CONTAINER_NAME=$1 -INSTANCE_NAME=$2 - -# Remove existing container instance -set +e -docker rm -f "$INSTANCE_NAME" -set -e - -# Delete old sdk directory if it exists -if [ -d "./cosmos-sdk" ]; then - rm -rf ./cosmos-sdk/ -fi - -# Copy sdk directory to working dir if path was specified -if [[ ${LOCAL_SDK_PATH+x} ]] -then - cp -n -r "$LOCAL_SDK_PATH" ./cosmos-sdk - printf "\n\nUsing local sdk version from %s\n\n\n" "$LOCAL_SDK_PATH" -else - printf "\n\nUsing default sdk version\n\n\n" -fi - -# Build the Docker container -if [[ ${USE_GAIA_TAG+x} ]] -then - printf "\n\nUsing gaia as provider\n\n" - printf "\n\nUsing gaia tag %s\n\n" "$USE_GAIA_TAG" - docker build -f Dockerfile.gaia -t "$CONTAINER_NAME" --build-arg USE_GAIA_TAG="$USE_GAIA_TAG" . -elif [ ${CONSUMER_VERSION+x} ] && [ ${PROVIDER_VERSION+x} ]; then - printf "\n\nUsing ICS consumer app from image version ${CONSUMER_VERSION} and provider from image version ${PROVIDER_VERSION}" - docker build -f Dockerfile-Combined --build-arg PROVIDER_VERSION="${PROVIDER_VERSION}" --build-arg CONSUMER_VERSION="${CONSUMER_VERSION}" -t "$CONTAINER_NAME" . - -elif [ ${CONSUMER_VERSION+x} ]; then - printf "\n\nUsing ICS consumer app from image version ${CONSUMER_VERSION}" - docker build -f Dockerfile-Consumer --build-arg CONSUMER_VERSION="${CONSUMER_VERSION}" -t "$CONTAINER_NAME" . -elif [ ${PROVIDER_VERSION+x} ]; then - printf "\n\nUsing ICS provider app from image version ${PROVIDER_VERSION}" - docker build -f Dockerfile-Provider --build-arg PROVIDER_VERSION="${PROVIDER_VERSION}" -t "$CONTAINER_NAME" . -else - printf "\n\nUsing ICS provider app as provider\n\n\n" - docker build -f Dockerfile -t "$CONTAINER_NAME" . -fi - -# Remove copied sdk directory -rm -rf ./cosmos-sdk/ - -# Run new test container instance with extended privileges. -# Extended privileges are granted to the container here to allow for network namespace manipulation (bringing a node up/down) -# See: https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities -docker run --name "$INSTANCE_NAME" --cap-add=NET_ADMIN --privileged "$CONTAINER_NAME" /bin/bash /testnet-scripts/beacon.sh & From 9724657728745a83c4b92ef8778a71478493d5fe Mon Sep 17 00:00:00 2001 From: Bernd Mueller Date: Wed, 10 Jan 2024 11:56:05 +0100 Subject: [PATCH 06/16] log cleanup --- tests/e2e/actions.go | 10 +++++----- tests/e2e/builder.go | 6 +++--- tests/e2e/main.go | 2 +- tests/e2e/state.go | 6 ++++-- tests/e2e/test_driver.go | 14 +------------- tests/e2e/test_target.go | 4 ++-- 6 files changed, 16 insertions(+), 26 deletions(-) diff --git a/tests/e2e/actions.go b/tests/e2e/actions.go index bc426d5f33..d69ff87457 100644 --- a/tests/e2e/actions.go +++ b/tests/e2e/actions.go @@ -486,8 +486,8 @@ type StartConsumerChainAction struct { // Transform consumer genesis content from older version func (tr *TestConfig) transformConsumerGenesis(consumerChain ChainID, genesis []byte, target ExecutionTarget) []byte { - log.Print("Transforming consumer genesis") - log.Printf("Original ccv genesis: %s\n", string(genesis)) + fmt.Println("Transforming consumer genesis") + fmt.Printf("Original ccv genesis: %s\n", string(genesis)) fileName := "consumer_genesis.json" file, err := os.CreateTemp("", fileName) @@ -519,13 +519,13 @@ func (tr *TestConfig) transformConsumerGenesis(consumerChain ChainID, genesis [] if err != nil { log.Fatal(err, "CCV consumer genesis transformation failed: %s", string(result)) } - log.Printf("Transformed genesis is: %s", string(result)) + fmt.Printf("Transformed genesis is: %s\n", string(result)) return result } // Get consumer genesis from provider func (tr *TestConfig) getConsumerGenesis(providerChain, consumerChain ChainID, target ExecutionTarget) string { - log.Print("Exporting consumer genesis from provider") + fmt.Println("Exporting consumer genesis from provider") providerBinaryName := tr.chainConfigs[providerChain].BinaryName cmd := target.ExecCommand( @@ -555,7 +555,7 @@ func (tr *TestConfig) startConsumerChain( target ExecutionTarget, verbose bool, ) { - log.Printf("Starting consumer chain %s", action.ConsumerChain) + fmt.Println("Starting consumer chain ", action.ConsumerChain) consumerGenesis := ".app_state.ccvconsumer = " + tr.getConsumerGenesis(action.ProviderChain, action.ConsumerChain, target) consumerGenesisChanges := tr.chainConfigs[action.ConsumerChain].GenesisChanges if consumerGenesisChanges != "" { diff --git a/tests/e2e/builder.go b/tests/e2e/builder.go index 3f0162b189..bb9d874610 100644 --- a/tests/e2e/builder.go +++ b/tests/e2e/builder.go @@ -17,13 +17,13 @@ func setupWorkSpace(revision string) (string, error) { return "", fmt.Errorf("error creating temp directory %v", err) } - log.Printf("Setting up worktree in '%s'", workSpace) + fmt.Printf("Setting up worktree in '%s'", workSpace) cmd := exec.Command("git", "worktree", "add", "--checkout", workSpace, revision) var errbuf bytes.Buffer cmd.Stderr = &errbuf - log.Printf("Running: %s", cmd.String()) + fmt.Printf("Running: %s", cmd.String()) if err := cmd.Start(); err != nil { return "", err } @@ -59,7 +59,7 @@ func dockerIsUp() bool { // Build docker image of ICS for a given revision func buildDockerImage(imageName, revision string, targetCfg TargetConfig) error { - log.Printf("Building ICS %s image %s", revision, imageName) + fmt.Printf("Building ICS %s image %s\n", revision, imageName) if !dockerIsUp() { return fmt.Errorf("docker engine is not running") } diff --git a/tests/e2e/main.go b/tests/e2e/main.go index 75ef0cf87b..e2afa5bee1 100644 --- a/tests/e2e/main.go +++ b/tests/e2e/main.go @@ -359,7 +359,7 @@ func executeTests(runners []TestRunner) error { runner.Run() }(runner) } else { - log.Printf("=============== running %s ===============\n", runner.config.name) + fmt.Printf("=============== running %s ===============\n", runner.config.name) err = runner.Run() } } diff --git a/tests/e2e/state.go b/tests/e2e/state.go index 6578aad03a..b3b431103a 100644 --- a/tests/e2e/state.go +++ b/tests/e2e/state.go @@ -101,10 +101,12 @@ type Param struct { Value string } -func (tr TestConfig) getState(modelState State) State { +func (tr TestConfig) getState(modelState State, verbose bool) State { systemState := State{} for k, modelState := range modelState { - log.Println("Getting model state for chain: ", k) + if verbose { + fmt.Println("Getting model state for chain: ", k) + } systemState[k] = tr.getChainState(k, modelState) } diff --git a/tests/e2e/test_driver.go b/tests/e2e/test_driver.go index 02bbdf0039..a52277558f 100644 --- a/tests/e2e/test_driver.go +++ b/tests/e2e/test_driver.go @@ -52,10 +52,7 @@ func (pr *DefaultDriver) runStep(step Step) error { return err } modelState := step.State - actualState, err := pr.getState(step.State) - if err != nil { - return err - } + actualState := pr.testCfg.getState(modelState, pr.verbose) // Check state if !reflect.DeepEqual(actualState, modelState) { @@ -68,10 +65,6 @@ func (pr *DefaultDriver) runStep(step Step) error { return nil } -func (pr *DefaultDriver) GetState(modelState State) { - pr.testCfg.getState(modelState) -} - func (td *DefaultDriver) runAction(action interface{}) error { switch action := action.(type) { case StartChainAction: @@ -155,8 +148,3 @@ func (td *DefaultDriver) runAction(action interface{}) error { } return nil } - -func (pr *DefaultDriver) getState(modelState State) (State, error) { - // forwarding it for now - return pr.testCfg.getState(modelState), nil -} diff --git a/tests/e2e/test_target.go b/tests/e2e/test_target.go index 2e05fb1edd..850e459467 100644 --- a/tests/e2e/test_target.go +++ b/tests/e2e/test_target.go @@ -147,12 +147,12 @@ func (dc *DockerContainer) ExecDetachedCommand(name string, arg ...string) *exec func (dc *DockerContainer) GetTestScriptPath(isConsumer bool, script string) string { path := "/testnet-scripts" if dc.targetConfig.providerVersion != "" && !isConsumer { - log.Printf("Using script path for provider version '%s'", dc.targetConfig.providerVersion) + fmt.Printf("Using script path for provider version '%s'\n", dc.targetConfig.providerVersion) path = "/provider/testnet-scripts" } if dc.targetConfig.consumerVersion != "" && isConsumer { - log.Printf("Using script path for consumer version '%s'", dc.targetConfig.consumerVersion) + fmt.Printf("Using script path for consumer version '%s'\n", dc.targetConfig.consumerVersion) path = "/consumer/testnet-scripts" } return strings.Join([]string{path, script}, string(os.PathSeparator)) From b758c2e0bade3b611fdf9bf98976dc8533e9ac5c Mon Sep 17 00:00:00 2001 From: Bernd Mueller Date: Wed, 10 Jan 2024 13:58:46 +0100 Subject: [PATCH 07/16] addressed comments --- Makefile | 4 -- tests/e2e/actions.go | 94 ++++++++++++++++---------------- tests/e2e/compatibility_tests.go | 20 ------- tests/e2e/main.go | 2 - 4 files changed, 47 insertions(+), 73 deletions(-) delete mode 100644 tests/e2e/compatibility_tests.go diff --git a/Makefile b/Makefile index fc460454ce..dfa42b3e82 100644 --- a/Makefile +++ b/Makefile @@ -70,10 +70,6 @@ test-e2e-multi-consumer: test-e2e-parallel: go run ./tests/e2e/... --include-multi-consumer --parallel -# run E2E compatibility tests for v3.x -test-e2e-compatibility-v3: - go run ./tests/e2e/... -parallel -tc compatibility -pv v3.2.0-rc1 -pv v3.1.0 -pv v3.0.0 -cv v3.2.0-rc1 -cv v3.1.0 -cv v3.0.0 - # run full E2E tests in sequence (including multiconsumer) using latest tagged gaia test-gaia-e2e: go run ./tests/e2e/... --include-multi-consumer --use-gaia diff --git a/tests/e2e/actions.go b/tests/e2e/actions.go index d69ff87457..048b603979 100644 --- a/tests/e2e/actions.go +++ b/tests/e2e/actions.go @@ -484,6 +484,53 @@ type StartConsumerChainAction struct { GenesisChanges string } +func (tr *TestConfig) startConsumerChain( + action StartConsumerChainAction, + target ExecutionTarget, + verbose bool, +) { + fmt.Println("Starting consumer chain ", action.ConsumerChain) + consumerGenesis := ".app_state.ccvconsumer = " + tr.getConsumerGenesis(action.ProviderChain, action.ConsumerChain, target) + consumerGenesisChanges := tr.chainConfigs[action.ConsumerChain].GenesisChanges + if consumerGenesisChanges != "" { + consumerGenesis = consumerGenesis + " | " + consumerGenesisChanges + " | " + action.GenesisChanges + } + + tr.startChain(StartChainAction{ + Chain: action.ConsumerChain, + Validators: action.Validators, + GenesisChanges: consumerGenesis, + IsConsumer: true, + }, target, verbose) +} + +// Get consumer genesis from provider +func (tr *TestConfig) getConsumerGenesis(providerChain, consumerChain ChainID, target ExecutionTarget) string { + fmt.Println("Exporting consumer genesis from provider") + providerBinaryName := tr.chainConfigs[providerChain].BinaryName + + cmd := target.ExecCommand( + providerBinaryName, + + "query", "provider", "consumer-genesis", + string(tr.chainConfigs[consumerChain].ChainId), + + `--node`, tr.getQueryNode(providerChain), + `-o`, `json`, + ) + + bz, err := cmd.CombinedOutput() + if err != nil { + log.Fatal(err, "\n", string(bz)) + } + + // only needed when consumer is running v3.3.x and later + if tr.transformGenesis { + return string(tr.transformConsumerGenesis(consumerChain, bz, target)) + } + return string(bz) +} + // Transform consumer genesis content from older version func (tr *TestConfig) transformConsumerGenesis(consumerChain ChainID, genesis []byte, target ExecutionTarget) []byte { fmt.Println("Transforming consumer genesis") @@ -523,53 +570,6 @@ func (tr *TestConfig) transformConsumerGenesis(consumerChain ChainID, genesis [] return result } -// Get consumer genesis from provider -func (tr *TestConfig) getConsumerGenesis(providerChain, consumerChain ChainID, target ExecutionTarget) string { - fmt.Println("Exporting consumer genesis from provider") - providerBinaryName := tr.chainConfigs[providerChain].BinaryName - - cmd := target.ExecCommand( - providerBinaryName, - - "query", "provider", "consumer-genesis", - string(tr.chainConfigs[consumerChain].ChainId), - - `--node`, tr.getQueryNode(providerChain), - `-o`, `json`, - ) - - bz, err := cmd.CombinedOutput() - if err != nil { - log.Fatal(err, "\n", string(bz)) - } - - // only needed when consumer is running v3.3.x and later - if tr.transformGenesis { - return string(tr.transformConsumerGenesis(consumerChain, bz, target)) - } - return string(bz) -} - -func (tr *TestConfig) startConsumerChain( - action StartConsumerChainAction, - target ExecutionTarget, - verbose bool, -) { - fmt.Println("Starting consumer chain ", action.ConsumerChain) - consumerGenesis := ".app_state.ccvconsumer = " + tr.getConsumerGenesis(action.ProviderChain, action.ConsumerChain, target) - consumerGenesisChanges := tr.chainConfigs[action.ConsumerChain].GenesisChanges - if consumerGenesisChanges != "" { - consumerGenesis = consumerGenesis + " | " + consumerGenesisChanges + " | " + action.GenesisChanges - } - - tr.startChain(StartChainAction{ - Chain: action.ConsumerChain, - Validators: action.Validators, - GenesisChanges: consumerGenesis, - IsConsumer: true, - }, target, verbose) -} - type ChangeoverChainAction struct { SovereignChain ChainID ProviderChain ChainID diff --git a/tests/e2e/compatibility_tests.go b/tests/e2e/compatibility_tests.go deleted file mode 100644 index 14bd48d7c7..0000000000 --- a/tests/e2e/compatibility_tests.go +++ /dev/null @@ -1,20 +0,0 @@ -package main - -// Get compatible provider versions for this consumer -// -// Note: This is a hardcoded list of tags which has to be updated manually -// and serves as base information of different versions to be tested -// against this consumer implementation -func GetCompatibleProviderVersions() []string { - return []string{"v3.0.x"} -} - -// Get compatible consumer versions for this provider -// -// Note: This is a hardcoded list of tags which has to be updated manually -// and serves as base information of different versions to be tested -// against this provider implementation -func GetCompatibleConsumerVersions() []string { - // For now it's the same as for provider - return GetCompatibleProviderVersions() -} diff --git a/tests/e2e/main.go b/tests/e2e/main.go index e2afa5bee1..3c2c47b85a 100644 --- a/tests/e2e/main.go +++ b/tests/e2e/main.go @@ -60,8 +60,6 @@ var ( ) var ( - //useConsumerVersion = flag.String("consumer-version", "", "ICS tag to specify the consumer version to test the provider against") - //useProviderVersion = flag.String("provider-version", "", "ICS tag to specify the provider version to test the consumer against") consumerVersions VersionSet providerVersions VersionSet transformGenesis = flag.Bool("transform-genesis", false, "do consumer genesis transformation for newer clients. Needed when provider chain is on an older version") From a6e58205d37ab017ed9bf984a1fa6fa8221180a4 Mon Sep 17 00:00:00 2001 From: Bernd Mueller Date: Wed, 10 Jan 2024 14:25:40 +0100 Subject: [PATCH 08/16] fix image cleanup --- tests/e2e/builder.go | 2 +- tests/e2e/test_target.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/e2e/builder.go b/tests/e2e/builder.go index bb9d874610..a49102ddd0 100644 --- a/tests/e2e/builder.go +++ b/tests/e2e/builder.go @@ -17,7 +17,7 @@ func setupWorkSpace(revision string) (string, error) { return "", fmt.Errorf("error creating temp directory %v", err) } - fmt.Printf("Setting up worktree in '%s'", workSpace) + fmt.Printf("Setting up worktree in '%s'\n", workSpace) cmd := exec.Command("git", "worktree", "add", "--checkout", workSpace, revision) diff --git a/tests/e2e/test_target.go b/tests/e2e/test_target.go index 850e459467..42a9efa9d2 100644 --- a/tests/e2e/test_target.go +++ b/tests/e2e/test_target.go @@ -94,6 +94,7 @@ func (dc *DockerContainer) Build() error { if err != nil { return err } + dc.images = append(dc.images, providerImageName) // build combined image using provider/consumer versions from images built above combinedImageName := fmt.Sprintf("cosmos-ics-combined:%s_%s", From 45eabc5e9d5824c50ed204dbe9220d5aa1926623 Mon Sep 17 00:00:00 2001 From: Bernd Mueller Date: Thu, 11 Jan 2024 12:35:46 +0100 Subject: [PATCH 09/16] addressed PR comments --- tests/e2e/actions.go | 5 +++-- tests/e2e/builder.go | 10 +++------- tests/e2e/main.go | 14 ++++++++++---- tests/e2e/test_runner.go | 2 +- tests/e2e/test_target.go | 2 ++ 5 files changed, 19 insertions(+), 14 deletions(-) diff --git a/tests/e2e/actions.go b/tests/e2e/actions.go index 048b603979..3fdb3ee4c3 100644 --- a/tests/e2e/actions.go +++ b/tests/e2e/actions.go @@ -267,6 +267,7 @@ func (tr TestConfig) submitConsumerAdditionProposal( log.Fatal("prop json contains single quote") } + //#nosec G204 -- bypass unsafe quoting warning (no production code) bz, err = target.ExecCommand( "/bin/bash", "-c", fmt.Sprintf(`echo '%s' > %s`, jsonStr, "/temp-proposal.json"), ).CombinedOutput() @@ -402,6 +403,7 @@ func (tr TestConfig) submitParamChangeProposal( log.Fatal("prop json contains single quote") } + //#nosec G204 -- bypass unsafe quoting warning (no production code) bz, err = target.ExecCommand( "/bin/bash", "-c", fmt.Sprintf(`echo '%s' > %s`, jsonStr, "/params-proposal.json"), ).CombinedOutput() @@ -1814,8 +1816,7 @@ func (tr TestConfig) invokeDoublesignSlash( ) { if !tr.useCometmock { chainConfig := tr.chainConfigs[action.Chain] - isConsumer := false - doubleSignScript := target.GetTestScriptPath(isConsumer, "cause-doublesign.sh") + doubleSignScript := target.GetTestScriptPath(false, "cause-doublesign.sh") bz, err := target.ExecCommand("/bin/bash", doubleSignScript, chainConfig.BinaryName, string(action.Validator), string(chainConfig.ChainId), chainConfig.IpPrefix).CombinedOutput() diff --git a/tests/e2e/builder.go b/tests/e2e/builder.go index a49102ddd0..f5059002ea 100644 --- a/tests/e2e/builder.go +++ b/tests/e2e/builder.go @@ -21,14 +21,10 @@ func setupWorkSpace(revision string) (string, error) { cmd := exec.Command("git", "worktree", "add", "--checkout", workSpace, revision) - var errbuf bytes.Buffer - cmd.Stderr = &errbuf + out, err := cmd.CombinedOutput() fmt.Printf("Running: %s", cmd.String()) - if err := cmd.Start(); err != nil { - return "", err - } - if err := cmd.Wait(); err != nil { - log.Printf("Error creating worktree (%v): %s", err, errbuf.String()) + if err != nil { + log.Printf("Error creating worktree (%v): %s", err, string(out)) return "", err } return workSpace, nil diff --git a/tests/e2e/main.go b/tests/e2e/main.go index 3c2c47b85a..43eba8331c 100644 --- a/tests/e2e/main.go +++ b/tests/e2e/main.go @@ -62,7 +62,7 @@ var ( var ( consumerVersions VersionSet providerVersions VersionSet - transformGenesis = flag.Bool("transform-genesis", false, "do consumer genesis transformation for newer clients. Needed when provider chain is on an older version") + transformGenesis = flag.Bool("transform-genesis", false, "enforces a consumer app to perform genesis transformation of exported ccv genesis data. For details see compatibility notes (RELEASES.md) of used versions") ) var ( @@ -201,8 +201,8 @@ func parseArguments() (err error) { flag.Var(&selectedTestfiles, "test-file", getTestFileUsageString()) - flag.Var(&consumerVersions, "cv", "Consumer version") - flag.Var(&providerVersions, "pv", "Provider version") + flag.Var(&consumerVersions, "cv", "Version (git tag, revison, branch) of the consumer to be tested. Tests will be run against combinations of all defined provider versions (-pv) with this consumer version. Default: consumer implementation of local workspace") + flag.Var(&providerVersions, "pv", "Version (git tag, revison, branch) of the provider to be tested. Tests will be run against combinations of all defined consumer versions (-cv) with this provider version. Default: provider implementation of local workspace") flag.Parse() @@ -354,7 +354,13 @@ func executeTests(runners []TestRunner) error { wg.Add(1) go func(runner TestRunner) { defer wg.Done() - runner.Run() + result := runner.Run() + if result != nil { + log.Printf("Test '%s' failed", runner.config.name) + } + if err == nil { + err = result + } }(runner) } else { fmt.Printf("=============== running %s ===============\n", runner.config.name) diff --git a/tests/e2e/test_runner.go b/tests/e2e/test_runner.go index a3b3e22a42..adcf31d592 100644 --- a/tests/e2e/test_runner.go +++ b/tests/e2e/test_runner.go @@ -28,7 +28,7 @@ func (tr *TestRunner) Run() error { err = tr.testDriver.Run(tr.steps, tr.target, tr.verbose) if err != nil { // not tearing down environment for troubleshooting reasons on container - return fmt.Errorf("test run failed: %v", err) + return fmt.Errorf("test run '%s' failed: %v", tr.config.name, err) } return tr.teardownEnvironment() } diff --git a/tests/e2e/test_target.go b/tests/e2e/test_target.go index 42a9efa9d2..5f6a1a3e8b 100644 --- a/tests/e2e/test_target.go +++ b/tests/e2e/test_target.go @@ -13,7 +13,9 @@ type ExecutionTarget interface { GetTargetType() string GetLogs(path string) []byte GetTestScriptPath(isConsumer bool, script string) string + // ExecCommand: when executed the command will run and return after completion ExecCommand(name string, arg ...string) *exec.Cmd + // ExecDetachedCommand: when executed the command will be run in the background and call will return immediately ExecDetachedCommand(name string, args ...string) *exec.Cmd Start() error Stop() error From 7db57bd6781b30197837153b2ae31dbf5f2f2f4a Mon Sep 17 00:00:00 2001 From: Bernd Mueller Date: Thu, 11 Jan 2024 14:36:22 +0100 Subject: [PATCH 10/16] some more gosec fixes --- tests/e2e/actions.go | 2 +- tests/e2e/builder.go | 4 +++- tests/e2e/main.go | 13 +++++++++---- tests/e2e/test_target.go | 9 ++++++++- 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/tests/e2e/actions.go b/tests/e2e/actions.go index 3fdb3ee4c3..f41d069935 100644 --- a/tests/e2e/actions.go +++ b/tests/e2e/actions.go @@ -544,7 +544,7 @@ func (tr *TestConfig) transformConsumerGenesis(consumerChain ChainID, genesis [] panic(fmt.Sprintf("failed writing ccv consumer file : %v", err)) } defer file.Close() - err = os.WriteFile(file.Name(), genesis, 0644) + err = os.WriteFile(file.Name(), genesis, 0600) if err != nil { log.Fatalf("Failed writing consumer genesis to file: %v", err) } diff --git a/tests/e2e/builder.go b/tests/e2e/builder.go index f5059002ea..027898217f 100644 --- a/tests/e2e/builder.go +++ b/tests/e2e/builder.go @@ -10,7 +10,7 @@ import ( ) // setupWorkSpace checks out given revision in a tmp directory -// and returns path where the workspace is located +// and returns the path where the workspace is located func setupWorkSpace(revision string) (string, error) { workSpace, err := os.MkdirTemp(os.TempDir(), "e2eWorkTree_") if err != nil { @@ -19,6 +19,7 @@ func setupWorkSpace(revision string) (string, error) { fmt.Printf("Setting up worktree in '%s'\n", workSpace) + //#nosec G204 -- Bypass linter warning for spawning subprocess with variable cmd := exec.Command("git", "worktree", "add", "--checkout", workSpace, revision) out, err := cmd.CombinedOutput() @@ -87,6 +88,7 @@ func buildDockerImage(imageName, revision string, targetCfg TargetConfig) error } if targetCfg.localSdkPath != "" { fmt.Printf("Using local SDK version from %s\n", targetCfg.localSdkPath) + //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. cmd := exec.Command("cp", "-n", "-r", targetCfg.localSdkPath, sdkPath) out, err := cmd.CombinedOutput() if err != nil { diff --git a/tests/e2e/main.go b/tests/e2e/main.go index 43eba8331c..1625d6fec8 100644 --- a/tests/e2e/main.go +++ b/tests/e2e/main.go @@ -284,10 +284,15 @@ func getTestCases(selectedPredefinedTests, selectedTestFiles TestSet) (tests []t } func deleteTargets(targets []ExecutionTarget) error { + var rc error = nil for _, target := range targets { - target.Delete() + if err := target.Delete(); err != nil { + rc = err + log.Println("error deleting target: ", err) + } + } - return nil + return rc } // Create targets where test cases should be executed on @@ -311,7 +316,7 @@ func createTargets(providerVersions, consumerVersions VersionSet) ([]ExecutionTa target := DockerContainer{targetConfig: targetCfg} err := target.Build() if err != nil { - deleteTargets(targets) + _ = deleteTargets(targets) return nil, err } targets = append(targets, &target) @@ -389,7 +394,7 @@ func main() { if err != nil { log.Fatal("failed creating test targets: ", err) } - defer func() { deleteTargets(targets) }() + defer func() { _ = deleteTargets(targets) }() testRunners := createTestRunners(targets, testCases) start := time.Now() diff --git a/tests/e2e/test_target.go b/tests/e2e/test_target.go index 5f6a1a3e8b..16664d49ee 100644 --- a/tests/e2e/test_target.go +++ b/tests/e2e/test_target.go @@ -102,6 +102,7 @@ func (dc *DockerContainer) Build() error { combinedImageName := fmt.Sprintf("cosmos-ics-combined:%s_%s", strings.Split(providerImageName, ":")[1], strings.Split(consumerImageName, ":")[1]) + //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. cmd := exec.Command("docker", "build", "-f", "Dockerfile.combined", "--build-arg", fmt.Sprintf("PROVIDER_IMAGE=%s", providerImageName), "--build-arg", fmt.Sprintf("CONSUMER_IMAGE=%s", consumerImageName), @@ -119,6 +120,7 @@ func (dc *DockerContainer) Build() error { func (dc *DockerContainer) Delete() error { for _, img := range dc.images { + //#nosec G204 -- Bypass linter warning for spawning subprocess with variable cmd := exec.Command("docker", "image", "rm", img) out, err := cmd.CombinedOutput() //TODO: ignore errors related to non-existing images @@ -134,6 +136,7 @@ func (dc *DockerContainer) Delete() error { func (dc *DockerContainer) ExecCommand(name string, arg ...string) *exec.Cmd { args := []string{"exec", dc.containerCfg.InstanceName, name} args = append(args, arg...) + //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. return exec.Command("docker", args...) } @@ -165,12 +168,15 @@ func (dc *DockerContainer) GetTestScriptPath(isConsumer bool, script string) str func (dc *DockerContainer) Start() error { fmt.Println("Starting container: ", dc.containerCfg.InstanceName) // Remove existing containers from previous runs - dc.Stop() + if err := dc.Stop(); err != nil { + return err + } // Run new test container instance with extended privileges. // Extended privileges are granted to the container here to allow for network namespace manipulation (bringing a node up/down) // See: https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities beaconScript := dc.GetTestScriptPath(false, "beacon.sh") + //#nosec G204 -- subprocess launched with potential tainted input (no production code) cmd := exec.Command("docker", "run", "--name", dc.containerCfg.InstanceName, "--cap-add=NET_ADMIN", "--privileged", dc.ImageName, "/bin/bash", beaconScript) @@ -214,6 +220,7 @@ func (dc *DockerContainer) Stop() error { return fmt.Errorf("error stopping docker container: %v, %s", err, string(bz)) } + //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. cmd = exec.Command("docker", "rm", dc.containerCfg.InstanceName) bz, err = cmd.CombinedOutput() if err != nil { From 8d46f319c4f9cbeda2a3459b4a3e5a089fa48124 Mon Sep 17 00:00:00 2001 From: Bernd Mueller Date: Thu, 11 Jan 2024 14:52:37 +0100 Subject: [PATCH 11/16] ignore errors when trying to remove non-existing containers --- tests/e2e/test_target.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/e2e/test_target.go b/tests/e2e/test_target.go index 16664d49ee..111992149d 100644 --- a/tests/e2e/test_target.go +++ b/tests/e2e/test_target.go @@ -216,15 +216,15 @@ func (dc *DockerContainer) Stop() error { //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. cmd := exec.Command("docker", "stop", dc.containerCfg.InstanceName) bz, err := cmd.CombinedOutput() - if err != nil { + if err != nil && !strings.Contains(string(bz), "No such container") { return fmt.Errorf("error stopping docker container: %v, %s", err, string(bz)) } //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. cmd = exec.Command("docker", "rm", dc.containerCfg.InstanceName) bz, err = cmd.CombinedOutput() - if err != nil { + if err != nil && !strings.Contains(string(bz), "No such container") { return fmt.Errorf("error removing docker container: %v, %s", err, string(bz)) } - return err + return nil } From a4065dff860cc9876fe7bbde0be9015a1a65a31e Mon Sep 17 00:00:00 2001 From: Bernd Mueller Date: Thu, 11 Jan 2024 15:10:42 +0100 Subject: [PATCH 12/16] updated e2e README --- tests/e2e/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 963494a2ee..cc74e57834 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -17,7 +17,7 @@ End-to-end tests can still be useful, and we need them, but when possible, we should prefer more local tests. At a high-level, every test case consists of the following steps. -* The test starts a docker container, see [the startup script](testnet-scripts/start-docker.sh) +* The test starts a docker container, see [setupEnvironment](test_runner.go) * We run a defined sequence of actions and expected states, see as an example the steps for [testing the democracy module](steps_democracy.go) * Actions are any event that might meaningfully modify the system state, such as submitting transactions to a node, making nodes double-sign, starting a relayer, starting a new chain, etc. * Expected states define the state we expect after the action was taken. @@ -35,7 +35,7 @@ At a high-level, every test case consists of the following steps. If you just want to run the end-to-end tests, see the commands in the Makefile in the repo root. -If you want to run the tests with a bit more control, see the help by running +If you want to run the tests with a bit more control, see the help by running ```go run ./tests/e2e/... --help``` in the repo root to see how to do that. @@ -102,7 +102,7 @@ the actions necessary to start a provider and multiple consumer chains are already "packaged together" and available as `stepsStartChains` in [steps_start_chains.go](steps_start_chains.go). -**Note:** The parts of the state that are *not* defined are just *not checked*. +**Note:** The parts of the state that are *not* defined are just *not checked*. For example, if the balance of a validator is not listed in a state, it means we do not care about the balance of that validator in this particular state. @@ -131,7 +131,7 @@ You can see the basic template for how to do this by looking at the actions in The basic principle is to use `exec.Command` to execute a command inside the docker container. The pattern for this looks something like this: ``` -cmd := exec.Command("docker", "exec", +cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[action.Chain].BinaryName, "the command to execute, for example tx", @@ -145,7 +145,7 @@ if err != nil { // potentially check something in the output, or log something, ... ``` -Don't forget to wire your action into [main.go](main.go):runStep, where +Don't forget to wire your action into [test_driver.go](test_driver.go):runAction, where the action structs and functions to run for each of them are wired together. **Note:** Actions don't need to check that the state was modified correctly, @@ -163,7 +163,7 @@ work and sometimes fail due to gas, as can happen with `--gas=auto` and no `--ga Essentially, sometimes the gas estimation will underestimate gas, but not always - it seems to be non-deterministic, and probably depends on subtle things like block times, or heights, which are not finely controlled in the end-to-end tests -and do not perfectly match each time. +and do not perfectly match each time. To be sure we don't introduce nondeterminism like this, we need to use a sufficient adjustment to make sure there is enough gas for transactions to pass. From 42b00f4aacfe7f3050f5574915827b9efafe9c19 Mon Sep 17 00:00:00 2001 From: Bernd Mueller Date: Mon, 15 Jan 2024 09:50:43 +0100 Subject: [PATCH 13/16] addressed remaining linter warning + caching & worktree improvement --- tests/e2e/builder.go | 15 ++++++++++++--- tests/e2e/test_target.go | 5 +++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/tests/e2e/builder.go b/tests/e2e/builder.go index 027898217f..63e9961661 100644 --- a/tests/e2e/builder.go +++ b/tests/e2e/builder.go @@ -21,9 +21,9 @@ func setupWorkSpace(revision string) (string, error) { //#nosec G204 -- Bypass linter warning for spawning subprocess with variable cmd := exec.Command("git", "worktree", "add", - "--checkout", workSpace, revision) + "--force", "--checkout", workSpace, revision) out, err := cmd.CombinedOutput() - fmt.Printf("Running: %s", cmd.String()) + fmt.Println("Running: ", cmd.String()) if err != nil { log.Printf("Error creating worktree (%v): %s", err, string(out)) return "", err @@ -55,7 +55,7 @@ func dockerIsUp() bool { } // Build docker image of ICS for a given revision -func buildDockerImage(imageName, revision string, targetCfg TargetConfig) error { +func buildDockerImage(imageName, revision string, targetCfg TargetConfig, noCache bool) error { fmt.Printf("Building ICS %s image %s\n", revision, imageName) if !dockerIsUp() { return fmt.Errorf("docker engine is not running") @@ -101,6 +101,10 @@ func buildDockerImage(imageName, revision string, targetCfg TargetConfig) error dockerFile := "Dockerfile" args := []string{"build", "-t", imageName} + if noCache { + args = append(args, "--no-cache") + } + if targetCfg.useGaia && targetCfg.gaiaTag != "" { dockerFile = "Dockerfile.gaia" args = append(args, "--build-arg", fmt.Sprintf("USE_GAIA_TAG=%s", targetCfg.gaiaTag)) @@ -111,6 +115,11 @@ func buildDockerImage(imageName, revision string, targetCfg TargetConfig) error cmd := exec.Command("docker", args...) cmd.Dir = workSpace out, err := cmd.CombinedOutput() + if err != nil && !noCache { + // Retry image creation from pristine state by enforcing --no-cache + log.Printf("Image creation failed '%v'. Re-trying without cache!", err) + return buildDockerImage(imageName, revision, targetCfg, true) + } if err != nil { return fmt.Errorf("building docker image failed running '%v' (%v): %s", cmd, err, out) } diff --git a/tests/e2e/test_target.go b/tests/e2e/test_target.go index 111992149d..026ad9b474 100644 --- a/tests/e2e/test_target.go +++ b/tests/e2e/test_target.go @@ -76,7 +76,7 @@ func (dc *DockerContainer) Build() error { if err != nil { return fmt.Errorf("failed building docker image: %v", err) } - err = buildDockerImage(consumerImageName, consumerVersion, dc.targetConfig) + err = buildDockerImage(consumerImageName, consumerVersion, dc.targetConfig, false) if err != nil { return err } @@ -92,7 +92,7 @@ func (dc *DockerContainer) Build() error { if err != nil { return fmt.Errorf("failed building docker image: %v", err) } - err = buildDockerImage(providerImageName, providerVersion, dc.targetConfig) + err = buildDockerImage(providerImageName, providerVersion, dc.targetConfig, false) if err != nil { return err } @@ -145,6 +145,7 @@ func (dc *DockerContainer) ExecCommand(name string, arg ...string) *exec.Cmd { func (dc *DockerContainer) ExecDetachedCommand(name string, arg ...string) *exec.Cmd { args := []string{"exec", "-d", dc.containerCfg.InstanceName, name} args = append(args, arg...) + //#nosec G204 -- Bypass linter warning for spawning subprocess with variable return exec.Command("docker", args...) } From 786b1ad47bef068babbf64f32a136f4a2bd48199 Mon Sep 17 00:00:00 2001 From: Bernd Mueller Date: Wed, 17 Jan 2024 14:13:11 +0100 Subject: [PATCH 14/16] docker image: use cometmock from provider image --- Dockerfile.combined | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile.combined b/Dockerfile.combined index 89a6a840ee..743eaafc61 100644 --- a/Dockerfile.combined +++ b/Dockerfile.combined @@ -18,8 +18,6 @@ FROM --platform=linux/amd64 ${CONSUMER_IMAGE} AS consumer # Get Hermes build FROM --platform=linux/amd64 otacrew/hermes-ics:evidence-cmd AS hermes-builder -# Get CometMock -FROM ghcr.io/informalsystems/cometmock:v0.37.x as cometmock-builder # Get GoRelayer FROM ghcr.io/informalsystems/relayer-no-gas-sim:v2.3.0-rc4-no-gas-sim AS gorelayer-builder @@ -31,7 +29,6 @@ RUN dnf install -y which iproute iputils procps-ng vim-minimal tmux net-tools ht USER root COPY --from=hermes-builder /usr/bin/hermes /usr/local/bin/ -COPY --from=cometmock-builder /usr/local/bin/cometmock /usr/local/bin/cometmock COPY --from=gorelayer-builder /bin/rly /usr/local/bin/ # Copy consumer from specified image @@ -45,5 +42,8 @@ COPY --from=consumer /testnet-scripts /consumer/testnet-scripts COPY --from=provider /usr/local/bin/interchain-security-pd /usr/local/bin/interchain-security-pd COPY --from=provider /testnet-scripts /provider/testnet-scripts +#Copy cometmock from provider image +COPY --from=provider /usr/local/bin/cometmock /usr/local/bin + # Copy in the hermes config ADD ./tests/e2e/testnet-scripts/hermes-config.toml /root/.hermes/config.toml From 7f0105d2666a5531e9cdd838bb8f0ed286e5d07f Mon Sep 17 00:00:00 2001 From: Bernd Mueller Date: Wed, 17 Jan 2024 16:46:28 +0100 Subject: [PATCH 15/16] fix testscript path --- tests/e2e/test_target.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/e2e/test_target.go b/tests/e2e/test_target.go index 026ad9b474..26836075b8 100644 --- a/tests/e2e/test_target.go +++ b/tests/e2e/test_target.go @@ -153,15 +153,18 @@ func (dc *DockerContainer) ExecDetachedCommand(name string, arg ...string) *exec // Needed for different consumer/provider versions staged in one container func (dc *DockerContainer) GetTestScriptPath(isConsumer bool, script string) string { path := "/testnet-scripts" - if dc.targetConfig.providerVersion != "" && !isConsumer { - fmt.Printf("Using script path for provider version '%s'\n", dc.targetConfig.providerVersion) - path = "/provider/testnet-scripts" - } - - if dc.targetConfig.consumerVersion != "" && isConsumer { - fmt.Printf("Using script path for consumer version '%s'\n", dc.targetConfig.consumerVersion) - path = "/consumer/testnet-scripts" + // in case the provider and consumer version differ the test-scripts are in dedicated directories on the target + // for each of them (see Docker.combined) + if dc.targetConfig.providerVersion != dc.targetConfig.consumerVersion { + if !isConsumer { + fmt.Printf("Using script path for provider version '%s'\n", dc.targetConfig.providerVersion) + path = "/provider/testnet-scripts" + } else { + fmt.Printf("Using script path for consumer version '%s'\n", dc.targetConfig.consumerVersion) + path = "/consumer/testnet-scripts" + } } + // no combined image (see Dockerfile) return strings.Join([]string{path, script}, string(os.PathSeparator)) } From f6055c62e65e771ec89f9e77346dac8ad61a7177 Mon Sep 17 00:00:00 2001 From: Bernd Mueller Date: Wed, 17 Jan 2024 16:11:29 +0100 Subject: [PATCH 16/16] Addressed comments from @sainoe --- Dockerfile.combined | 10 ++++++--- tests/e2e/main.go | 48 ++++++++++++++++++---------------------- tests/e2e/test_driver.go | 22 +++++++++--------- tests/e2e/test_target.go | 6 ----- 4 files changed, 39 insertions(+), 47 deletions(-) diff --git a/Dockerfile.combined b/Dockerfile.combined index 743eaafc61..5ab867c89c 100644 --- a/Dockerfile.combined +++ b/Dockerfile.combined @@ -1,8 +1,12 @@ # syntax=docker/dockerfile:1 -# Consumer image to be used for compatibility tests with provider from workspace -# use docker's build argument --build-arg to overwrite the defaults -# e.g. docker build --build-arg CONSUMER_TAG=v3.1.0 +# Dockerfile.combined defines a docker image using different provider/consumer versions +# originated from other docker images. +# This image is used to test different versions of provider and consumer together. +# +# Use docker's build argument --build-arg to specify the consumer/provider image to be used +# e.g. docker build --build-arg CONSUMER_IMAGE=v3.1.0 --build-arg PROVIDER_IMAGE=v3.1.0 + ARG PROVIDER_IMAGE ARG CONSUMER_IMAGE diff --git a/tests/e2e/main.go b/tests/e2e/main.go index 1625d6fec8..8ad10b03e9 100644 --- a/tests/e2e/main.go +++ b/tests/e2e/main.go @@ -27,21 +27,19 @@ func (t *TestSet) String() string { return fmt.Sprint(*t) } -type VersionSet []string +type VersionSet map[string]bool -func (c *VersionSet) Set(value string) error { - // Check and skip duplicates - for _, v := range *c { - if v == value { - return nil - } - } - *c = append(*c, value) +func (vs *VersionSet) Set(value string) error { + (*vs)[value] = true return nil } -func (t *VersionSet) String() string { - return fmt.Sprint(*t) +func (vs *VersionSet) String() string { + keys := []string{} + for k, _ := range *vs { + keys = append(keys, k) + } + return fmt.Sprint(keys) } var ( @@ -60,9 +58,9 @@ var ( ) var ( - consumerVersions VersionSet - providerVersions VersionSet - transformGenesis = flag.Bool("transform-genesis", false, "enforces a consumer app to perform genesis transformation of exported ccv genesis data. For details see compatibility notes (RELEASES.md) of used versions") + consumerVersions VersionSet = VersionSet{} + providerVersions VersionSet = VersionSet{} + transformGenesis = flag.Bool("transform-genesis", false, "enforces a consumer app to perform genesis transformation of exported ccv genesis data. For details see compatibility notes (RELEASES.md) of used versions") ) var ( @@ -283,16 +281,13 @@ func getTestCases(selectedPredefinedTests, selectedTestFiles TestSet) (tests []t return tests } -func deleteTargets(targets []ExecutionTarget) error { - var rc error = nil +// delete all test targets +func deleteTargets(targets []ExecutionTarget) { for _, target := range targets { if err := target.Delete(); err != nil { - rc = err log.Println("error deleting target: ", err) } - } - return rc } // Create targets where test cases should be executed on @@ -303,20 +298,21 @@ func createTargets(providerVersions, consumerVersions VersionSet) ([]ExecutionTa var targets []ExecutionTarget if len(consumerVersions) == 0 { - consumerVersions = append(consumerVersions, "") + consumerVersions[""] = true } if len(providerVersions) == 0 { - providerVersions = append(providerVersions, "") + providerVersions[""] = true } - for _, provider := range providerVersions { - for _, consumer := range consumerVersions { + for provider, _ := range providerVersions { + for consumer, _ := range consumerVersions { targetCfg.consumerVersion = consumer targetCfg.providerVersion = provider target := DockerContainer{targetConfig: targetCfg} err := target.Build() if err != nil { - _ = deleteTargets(targets) + log.Println("@@@ failed creating target") + deleteTargets(targets) return nil, err } targets = append(targets, &target) @@ -362,8 +358,6 @@ func executeTests(runners []TestRunner) error { result := runner.Run() if result != nil { log.Printf("Test '%s' failed", runner.config.name) - } - if err == nil { err = result } }(runner) @@ -394,7 +388,7 @@ func main() { if err != nil { log.Fatal("failed creating test targets: ", err) } - defer func() { _ = deleteTargets(targets) }() + defer func() { deleteTargets(targets) }() testRunners := createTestRunners(targets, testCases) start := time.Now() diff --git a/tests/e2e/test_driver.go b/tests/e2e/test_driver.go index a52277558f..26f44d9600 100644 --- a/tests/e2e/test_driver.go +++ b/tests/e2e/test_driver.go @@ -25,38 +25,38 @@ type DefaultDriver struct { } // Execute tests -func (pr *DefaultDriver) Run(steps []Step, target ExecutionTarget, verbose bool) error { - pr.target = target - pr.verbose = verbose +func (td *DefaultDriver) Run(steps []Step, target ExecutionTarget, verbose bool) error { + td.target = target + td.verbose = verbose - fmt.Printf("=============== started %s tests ===============\n", pr.testCfg.name) + fmt.Printf("=============== started %s tests ===============\n", td.testCfg.name) start := time.Now() for i, step := range steps { fmt.Printf("running %s: step %d/%d == %s \n", - pr.testCfg.name, i+1, len(steps), reflect.TypeOf(step.Action).Name()) + td.testCfg.name, i+1, len(steps), reflect.TypeOf(step.Action).Name()) - err := pr.runStep(step) + err := td.runStep(step) if err != nil { return err } } - fmt.Printf("=============== finished %s tests in %v ===============\n", pr.testCfg.name, time.Since(start)) + fmt.Printf("=============== finished %s tests in %v ===============\n", td.testCfg.name, time.Since(start)) return nil } // runStep executes an action and evaluates the result against expected state -func (pr *DefaultDriver) runStep(step Step) error { - err := pr.runAction(step.Action) +func (td *DefaultDriver) runStep(step Step) error { + err := td.runAction(step.Action) if err != nil { return err } modelState := step.State - actualState := pr.testCfg.getState(modelState, pr.verbose) + actualState := td.testCfg.getState(modelState, td.verbose) // Check state if !reflect.DeepEqual(actualState, modelState) { - fmt.Printf("=============== %s FAILED ===============\n", pr.testCfg.name) + fmt.Printf("=============== %s FAILED ===============\n", td.testCfg.name) fmt.Println("FAILED action", reflect.TypeOf(step.Action).Name()) pretty.Print("actual state", actualState) pretty.Print("model state", modelState) diff --git a/tests/e2e/test_target.go b/tests/e2e/test_target.go index 26836075b8..23de16127f 100644 --- a/tests/e2e/test_target.go +++ b/tests/e2e/test_target.go @@ -11,7 +11,6 @@ import ( type ExecutionTarget interface { GetTargetType() string - GetLogs(path string) []byte GetTestScriptPath(isConsumer bool, script string) string // ExecCommand: when executed the command will run and return after completion ExecCommand(name string, arg ...string) *exec.Cmd @@ -34,11 +33,6 @@ func (dc *DockerContainer) GetTargetType() string { return "docker" } -func (dc *DockerContainer) GetLogs(path string) []byte { - logs := []byte{} - return logs -} - func generateImageName(version string, cfg TargetConfig) (string, error) { // identify a tag name tagName := ""