run happy path tests on push; bump hermes version (#659)

* use official hermes release

* refactor integration test main.go

* update automated-tests integration test run

* fix worng container teardown

* refactor main.go; add parallel execution

* update Makefile

* simplify code

* refactor for naming consistency

* fix string formatting

.github/workflows/automated-tests.yml

@@ -19,4 +19,4 @@ jobs:
         run: go test ./...
 
       - name: Integration tests
-        run: go run ./tests/integration/...
+        run: make test-integration-short

Dockerfile

@@ -22,19 +22,13 @@ RUN go mod tidy
 RUN make install
 
 # Get Hermes build
-# FROM informalsystems/hermes:1.0.0 AS hermes-builder
-# remove line below and use the line above once hermes is updated
-FROM majita/hermes-unofficial:latest AS hermes-builder
+FROM informalsystems/hermes:1.2.0 AS hermes-builder
 
 FROM --platform=linux/amd64 fedora:36
 RUN dnf update -y
-RUN dnf install -y which iproute iputils procps-ng vim-minimal tmux net-tools htop jq openssl1.1
+RUN dnf install -y which iproute iputils procps-ng vim-minimal tmux net-tools htop jq
 USER root
 
-# Copy Hermes and IS binaries to final image
-COPY --chown=0:0 --from=hermes-builder /usr/lib/x86_64-linux-gnu/libssl.so.1.1 /usr/lib/x86_64-linux-gnu/libssl.so.1.1
-COPY --chown=0:0 --from=hermes-builder /usr/lib/x86_64-linux-gnu/libcrypto.so.1.1 /usr/lib/x86_64-linux-gnu/libcrypto.so.1.1
-
 COPY --from=hermes-builder /usr/bin/hermes /usr/local/bin/
 
 COPY --from=is-builder /go/bin/interchain-security-pd /usr/local/bin/interchain-security-pd

Makefile

@@ -20,9 +20,17 @@ test-short:
 test-diff:
 	go test ./tests/difference/...
 
-# run integration tests
+# run only happy path integration tests
+test-integration-short:
+	go run ./tests/integration/... --happy-path-only
+
+# run full integration tests in sequence (including multiconsumer)
 test-integration:
-	go run ./tests/integration/...
+	go run ./tests/integration/... --include-multi-consumer
+
+# run full integration tests in parallel (including multiconsumer)
+test-integration-parallel:
+	go run ./tests/integration/... --include-multi-consumer --parallel
 
 # run all tests with caching disabled
 test-no-cache:

tests/integration/config.go

@@ -309,14 +309,14 @@ func (s *TestRun) SetLocalSDKPath(path string) {
 	s.localSdkPath = path
 }
 
-// ValidateStringLiterals enforces that configs follow the constraints
+// validateStringLiterals enforces that configs follow the constraints
 // necessary to to execute the tests
 //
 // Note: Network interfaces (name of virtual ethernet interfaces for ip link)
 // within the container will be named as "$CHAIN_ID-$VAL_ID-out" etc.
 // where this name is constrained to 15 bytes or less. Therefore each string literal
 // used as a validatorID or chainID needs to be 5 char or less.
-func (s *TestRun) ValidateStringLiterals() {
+func (s *TestRun) validateStringLiterals() {
 	for valID, valConfig := range s.validatorConfigs {
 
 		if len(valID) > 5 {

tests/integration/main.go

@@ -14,7 +14,9 @@ import (
 )
 
 var verbose = flag.Bool("verbose", false, "turn verbose logging on/off")
-var multiconsumer = flag.Bool("multiconsumer", false, "run happy path and multiconsumer tests in parallel")
+var happyPathOnly = flag.Bool("happy-path-only", false, "run happy path tests only")
+var includeMultiConsumer = flag.Bool("include-multi-consumer", false, "include multiconsumer tests in run")
+var parallel = flag.Bool("parallel", false, "run all tests in parallel")
 var localSdkPath = flag.String("local-sdk-path", "",
 	"path of a local sdk version to build and reference in integration tests")
 
@@ -24,46 +26,59 @@ var localSdkPath = flag.String("local-sdk-path", "",
 func main() {
 	flag.Parse()
 
-	// wg waits for all runners to complete
-	var wg sync.WaitGroup
-
-	start := time.Now()
-	tr := DefaultTestRun()
-	tr.SetLocalSDKPath(*localSdkPath)
-	tr.ValidateStringLiterals()
-	tr.startDocker()
-
-	dmc := DemocracyTestRun()
-	dmc.SetLocalSDKPath(*localSdkPath)
-	dmc.ValidateStringLiterals()
-	dmc.startDocker()
-
-	slash := SlashThrottleTestRun()
-	slash.SetLocalSDKPath(*localSdkPath)
-	slash.ValidateStringLiterals()
-	slash.startDocker()
-
-	if *multiconsumer {
-		mul := MultiConsumerTestRun()
-		mul.SetLocalSDKPath(*localSdkPath)
-		mul.ValidateStringLiterals()
-		mul.startDocker()
+	if happyPathOnly != nil && *happyPathOnly {
+		fmt.Println("=============== running happy path only ===============")
+		tr := DefaultTestRun()
+		tr.Run(happyPathSteps, *localSdkPath)
+		return
+	}
 
-		wg.Add(1)
-		go mul.ExecuteSteps(&wg, multipleConsumers)
+	testRuns := []testRunWithSteps{
+		{DefaultTestRun(), happyPathSteps},
+		{DemocracyTestRun(), democracySteps},
+		{SlashThrottleTestRun(), slashThrottleSteps},
+	}
+	if includeMultiConsumer != nil && *includeMultiConsumer {
+		testRuns = append(testRuns, testRunWithSteps{MultiConsumerTestRun(), multipleConsumers})
 	}
 
-	wg.Add(1)
-	go tr.ExecuteSteps(&wg, happyPathSteps)
+	start := time.Now()
+	if parallel != nil && *parallel {
+		fmt.Println("=============== running all tests in parallel ===============")
+		var wg sync.WaitGroup
+		for _, run := range testRuns {
+			wg.Add(1)
+			go func(run testRunWithSteps) {
+				defer wg.Done()
+				tr := run.testRun
+				tr.Run(run.steps, *localSdkPath)
+			}(run)
+		}
+		wg.Wait()
+		fmt.Printf("TOTAL TIME ELAPSED: %v\n", time.Since(start))
+		return
+	}
 
-	wg.Add(1)
-	go dmc.ExecuteSteps(&wg, democracySteps)
+	for _, run := range testRuns {
+		tr := run.testRun
+		tr.Run(run.steps, *localSdkPath)
+	}
+	fmt.Printf("TOTAL TIME ELAPSED: %v\n", time.Since(start))
+}
 
-	wg.Add(1)
-	go slash.ExecuteSteps(&wg, slashThrottleSteps)
+// Run sets up docker container and executes the steps in the test run.
+// Docker containers are torn down after the test run is complete.
+func (tr *TestRun) Run(steps []Step, localSdkPath string) {
+	tr.SetLocalSDKPath(localSdkPath)
+	tr.validateStringLiterals()
+	tr.startDocker()
+	tr.executeSteps(steps)
+	tr.teardownDocker()
+}
 
-	wg.Wait()
-	fmt.Printf("TOTAL TIME ELAPSED: %v\n", time.Since(start))
+type testRunWithSteps struct {
+	testRun TestRun
+	steps   []Step
 }
 
 func (tr *TestRun) runStep(step Step, verbose bool) {
@@ -115,7 +130,7 @@ func (tr *TestRun) runStep(step Step, verbose bool) {
 	case slashThrottleDequeue:
 		tr.waitForSlashThrottleDequeue(action, verbose)
 	default:
-		log.Fatalf(fmt.Sprintf(`unknown action in testRun %s: %#v`, tr.name, action))
+		log.Fatalf("unknown action in testRun %s: %#v", tr.name, action)
 	}
 
 	modelState := step.state
@@ -131,9 +146,8 @@ func (tr *TestRun) runStep(step Step, verbose bool) {
 	}
 }
 
-// Starts docker container and sequentially runs steps
-func (tr *TestRun) ExecuteSteps(wg *sync.WaitGroup, steps []Step) {
-	defer wg.Done()
+// executeSteps sequentially runs steps.
+func (tr *TestRun) executeSteps(steps []Step) {
 	fmt.Printf("=============== started %s tests ===============\n", tr.name)
 
 	start := time.Now()
@@ -184,3 +198,16 @@ func (tr *TestRun) startDocker() {
 	err = cmd.Wait()
 	log.Fatalf("StartDocker exited with error: %v", err)
 }
+
+// remove docker container to reduce resource usage
+// otherwise the chain will keep running in the background
+func (tr *TestRun) teardownDocker() {
+	fmt.Printf("===============  tearing down %s testRun ===============\n", tr.name)
+	//#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments.
+	cmd := exec.Command("docker", "kill", tr.containerConfig.instanceName)
+
+	bz, err := cmd.CombinedOutput()
+	if err != nil {
+		log.Fatal(err, "\n", string(bz))
+	}
+}