From 7ed00cbaa4ba221a7e76093a92dc9124588e1d69 Mon Sep 17 00:00:00 2001 From: MSalopek <35486649+MSalopek@users.noreply.github.com> Date: Fri, 20 Jan 2023 12:09:51 +0100 Subject: [PATCH] run happy path tests on push; bump hermes version (#659) * use official hermes release * refactor integration test main.go * update automated-tests integration test run * fix worng container teardown * refactor main.go; add parallel execution * update Makefile * simplify code * refactor for naming consistency * fix string formatting --- .github/workflows/automated-tests.yml | 2 +- Dockerfile | 10 +-- Makefile | 12 ++- tests/integration/config.go | 4 +- tests/integration/main.go | 105 ++++++++++++++++---------- 5 files changed, 81 insertions(+), 52 deletions(-) diff --git a/.github/workflows/automated-tests.yml b/.github/workflows/automated-tests.yml index 64f27f49fb..15a34a2f96 100644 --- a/.github/workflows/automated-tests.yml +++ b/.github/workflows/automated-tests.yml @@ -19,4 +19,4 @@ jobs: run: go test ./... - name: Integration tests - run: go run ./tests/integration/... + run: make test-integration-short diff --git a/Dockerfile b/Dockerfile index e4a5ef01e6..cecbd11828 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,19 +22,13 @@ RUN go mod tidy RUN make install # Get Hermes build -# FROM informalsystems/hermes:1.0.0 AS hermes-builder -# remove line below and use the line above once hermes is updated -FROM majita/hermes-unofficial:latest AS hermes-builder +FROM informalsystems/hermes:1.2.0 AS hermes-builder FROM --platform=linux/amd64 fedora:36 RUN dnf update -y -RUN dnf install -y which iproute iputils procps-ng vim-minimal tmux net-tools htop jq openssl1.1 +RUN dnf install -y which iproute iputils procps-ng vim-minimal tmux net-tools htop jq USER root -# Copy Hermes and IS binaries to final image -COPY --chown=0:0 --from=hermes-builder /usr/lib/x86_64-linux-gnu/libssl.so.1.1 /usr/lib/x86_64-linux-gnu/libssl.so.1.1 -COPY --chown=0:0 --from=hermes-builder /usr/lib/x86_64-linux-gnu/libcrypto.so.1.1 /usr/lib/x86_64-linux-gnu/libcrypto.so.1.1 - COPY --from=hermes-builder /usr/bin/hermes /usr/local/bin/ COPY --from=is-builder /go/bin/interchain-security-pd /usr/local/bin/interchain-security-pd diff --git a/Makefile b/Makefile index 3e992796b6..e15ec30681 100644 --- a/Makefile +++ b/Makefile @@ -20,9 +20,17 @@ test-short: test-diff: go test ./tests/difference/... -# run integration tests +# run only happy path integration tests +test-integration-short: + go run ./tests/integration/... --happy-path-only + +# run full integration tests in sequence (including multiconsumer) test-integration: - go run ./tests/integration/... + go run ./tests/integration/... --include-multi-consumer + +# run full integration tests in parallel (including multiconsumer) +test-integration-parallel: + go run ./tests/integration/... --include-multi-consumer --parallel # run all tests with caching disabled test-no-cache: diff --git a/tests/integration/config.go b/tests/integration/config.go index 5795d4335b..0df21b26eb 100644 --- a/tests/integration/config.go +++ b/tests/integration/config.go @@ -309,14 +309,14 @@ func (s *TestRun) SetLocalSDKPath(path string) { s.localSdkPath = path } -// ValidateStringLiterals enforces that configs follow the constraints +// validateStringLiterals enforces that configs follow the constraints // necessary to to execute the tests // // Note: Network interfaces (name of virtual ethernet interfaces for ip link) // within the container will be named as "$CHAIN_ID-$VAL_ID-out" etc. // where this name is constrained to 15 bytes or less. Therefore each string literal // used as a validatorID or chainID needs to be 5 char or less. -func (s *TestRun) ValidateStringLiterals() { +func (s *TestRun) validateStringLiterals() { for valID, valConfig := range s.validatorConfigs { if len(valID) > 5 { diff --git a/tests/integration/main.go b/tests/integration/main.go index 8b51b104b4..e28885f31b 100644 --- a/tests/integration/main.go +++ b/tests/integration/main.go @@ -14,7 +14,9 @@ import ( ) var verbose = flag.Bool("verbose", false, "turn verbose logging on/off") -var multiconsumer = flag.Bool("multiconsumer", false, "run happy path and multiconsumer tests in parallel") +var happyPathOnly = flag.Bool("happy-path-only", false, "run happy path tests only") +var includeMultiConsumer = flag.Bool("include-multi-consumer", false, "include multiconsumer tests in run") +var parallel = flag.Bool("parallel", false, "run all tests in parallel") var localSdkPath = flag.String("local-sdk-path", "", "path of a local sdk version to build and reference in integration tests") @@ -24,46 +26,59 @@ var localSdkPath = flag.String("local-sdk-path", "", func main() { flag.Parse() - // wg waits for all runners to complete - var wg sync.WaitGroup - - start := time.Now() - tr := DefaultTestRun() - tr.SetLocalSDKPath(*localSdkPath) - tr.ValidateStringLiterals() - tr.startDocker() - - dmc := DemocracyTestRun() - dmc.SetLocalSDKPath(*localSdkPath) - dmc.ValidateStringLiterals() - dmc.startDocker() - - slash := SlashThrottleTestRun() - slash.SetLocalSDKPath(*localSdkPath) - slash.ValidateStringLiterals() - slash.startDocker() - - if *multiconsumer { - mul := MultiConsumerTestRun() - mul.SetLocalSDKPath(*localSdkPath) - mul.ValidateStringLiterals() - mul.startDocker() + if happyPathOnly != nil && *happyPathOnly { + fmt.Println("=============== running happy path only ===============") + tr := DefaultTestRun() + tr.Run(happyPathSteps, *localSdkPath) + return + } - wg.Add(1) - go mul.ExecuteSteps(&wg, multipleConsumers) + testRuns := []testRunWithSteps{ + {DefaultTestRun(), happyPathSteps}, + {DemocracyTestRun(), democracySteps}, + {SlashThrottleTestRun(), slashThrottleSteps}, + } + if includeMultiConsumer != nil && *includeMultiConsumer { + testRuns = append(testRuns, testRunWithSteps{MultiConsumerTestRun(), multipleConsumers}) } - wg.Add(1) - go tr.ExecuteSteps(&wg, happyPathSteps) + start := time.Now() + if parallel != nil && *parallel { + fmt.Println("=============== running all tests in parallel ===============") + var wg sync.WaitGroup + for _, run := range testRuns { + wg.Add(1) + go func(run testRunWithSteps) { + defer wg.Done() + tr := run.testRun + tr.Run(run.steps, *localSdkPath) + }(run) + } + wg.Wait() + fmt.Printf("TOTAL TIME ELAPSED: %v\n", time.Since(start)) + return + } - wg.Add(1) - go dmc.ExecuteSteps(&wg, democracySteps) + for _, run := range testRuns { + tr := run.testRun + tr.Run(run.steps, *localSdkPath) + } + fmt.Printf("TOTAL TIME ELAPSED: %v\n", time.Since(start)) +} - wg.Add(1) - go slash.ExecuteSteps(&wg, slashThrottleSteps) +// Run sets up docker container and executes the steps in the test run. +// Docker containers are torn down after the test run is complete. +func (tr *TestRun) Run(steps []Step, localSdkPath string) { + tr.SetLocalSDKPath(localSdkPath) + tr.validateStringLiterals() + tr.startDocker() + tr.executeSteps(steps) + tr.teardownDocker() +} - wg.Wait() - fmt.Printf("TOTAL TIME ELAPSED: %v\n", time.Since(start)) +type testRunWithSteps struct { + testRun TestRun + steps []Step } func (tr *TestRun) runStep(step Step, verbose bool) { @@ -115,7 +130,7 @@ func (tr *TestRun) runStep(step Step, verbose bool) { case slashThrottleDequeue: tr.waitForSlashThrottleDequeue(action, verbose) default: - log.Fatalf(fmt.Sprintf(`unknown action in testRun %s: %#v`, tr.name, action)) + log.Fatalf("unknown action in testRun %s: %#v", tr.name, action) } modelState := step.state @@ -131,9 +146,8 @@ func (tr *TestRun) runStep(step Step, verbose bool) { } } -// Starts docker container and sequentially runs steps -func (tr *TestRun) ExecuteSteps(wg *sync.WaitGroup, steps []Step) { - defer wg.Done() +// executeSteps sequentially runs steps. +func (tr *TestRun) executeSteps(steps []Step) { fmt.Printf("=============== started %s tests ===============\n", tr.name) start := time.Now() @@ -184,3 +198,16 @@ func (tr *TestRun) startDocker() { err = cmd.Wait() log.Fatalf("StartDocker exited with error: %v", err) } + +// remove docker container to reduce resource usage +// otherwise the chain will keep running in the background +func (tr *TestRun) teardownDocker() { + fmt.Printf("=============== tearing down %s testRun ===============\n", tr.name) + //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. + cmd := exec.Command("docker", "kill", tr.containerConfig.instanceName) + + bz, err := cmd.CombinedOutput() + if err != nil { + log.Fatal(err, "\n", string(bz)) + } +}