Change steps/test run delimiter

cosmos · Sep 15, 2023 · 60d510f · 60d510f
1 parent efd6d53
commit 60d510f
Show file tree

Hide file tree

Showing 5 changed files with 46 additions and 30 deletions.
diff --git a/.github/workflows/manual-e2e.yml b/.github/workflows/manual-e2e.yml
@@ -20,7 +20,7 @@ jobs:
         with:
           go-version: "1.20" # The Go version to download (if necessary) and use.
       - name: E2E happy-path test
-        run: go run ./tests/e2e/... --tc happy-path/default
+        run: go run ./tests/e2e/... --tc happy-path::default
   changeover-test:
     runs-on: ubuntu-latest
     timeout-minutes: 20
@@ -36,7 +36,7 @@ jobs:
         with:
           go-version: "1.20" # The Go version to download (if necessary) and use.
       - name: E2E changeover test
-        run: go run ./tests/e2e/... --tc changeover/changeover
+        run: go run ./tests/e2e/... --tc changeover::changeover
   democracy-reward-test:
       runs-on: ubuntu-latest
       timeout-minutes: 20
@@ -52,7 +52,7 @@ jobs:
           with:
             go-version: "1.20" # The Go version to download (if necessary) and use.
         - name: E2E democracy-reward tests
-          run: go run ./tests/e2e/... --tc democracy-reward
+          run: go run ./tests/e2e/... --tc democracy-reward::democracy-reward
   democracy-test:
       runs-on: ubuntu-latest
       timeout-minutes: 20
@@ -68,7 +68,7 @@ jobs:
           with:
             go-version: "1.20" # The Go version to download (if necessary) and use.
         - name: E2E democracy tests
-          run: go run ./tests/e2e/... --tc democracy/democracy
+          run: go run ./tests/e2e/... --tc democracy::democracy
   slash-throttle-test:
       runs-on: ubuntu-latest
       timeout-minutes: 20
@@ -84,7 +84,7 @@ jobs:
           with:
             go-version: "1.20" # The Go version to download (if necessary) and use.
         - name: E2E slash-throttle tests
-          run: go run ./tests/e2e/... --tc slash-throttle/slash-throttle
+          run: go run ./tests/e2e/... --tc slash-throttle::slash-throttle
   multiconsumer-test:
       runs-on: ubuntu-latest
       timeout-minutes: 40
@@ -100,4 +100,4 @@ jobs:
           with:
             go-version: "1.20" # The Go version to download (if necessary) and use.
         - name: E2E multi-consumer tests
-          run: go run ./tests/e2e/... --tc multiconsumer/multiconsumer
+          run: go run ./tests/e2e/... --tc multiconsumer::multiconsumer
diff --git a/.github/workflows/nightly-e2e.yml b/.github/workflows/nightly-e2e.yml
@@ -33,7 +33,7 @@ jobs:
         with:
           go-version: "1.20" # The Go version to download (if necessary) and use.
       - name: E2E happy-path test
-        run: go run ./tests/e2e/... --tc happy-path/default
+        run: go run ./tests/e2e/... --tc happy-path::default
   changeover-test:
     runs-on: ubuntu-latest
     timeout-minutes: 20
@@ -49,7 +49,7 @@ jobs:
         with:
           go-version: "1.20" # The Go version to download (if necessary) and use.
       - name: E2E changeover test
-        run: go run ./tests/e2e/... --tc changeover/changeover
+        run: go run ./tests/e2e/... --tc changeover::changeover
   democracy-reward-test:
     runs-on: ubuntu-latest
     timeout-minutes: 20
@@ -65,7 +65,7 @@ jobs:
         with:
           go-version: "1.20" # The Go version to download (if necessary) and use.
       - name: E2E democracy-reward tests
-        run: go run ./tests/e2e/... --tc democracy-reward/democracy-reward
+        run: go run ./tests/e2e/... --tc democracy-reward::democracy-reward
   democracy-test:
     runs-on: ubuntu-latest
     timeout-minutes: 20
@@ -81,7 +81,7 @@ jobs:
         with:
           go-version: "1.20" # The Go version to download (if necessary) and use.
       - name: E2E democracy tests
-        run: go run ./tests/e2e/... --tc democracy/democracy
+        run: go run ./tests/e2e/... --tc democracy::democracy
   slash-throttle-test:
     runs-on: ubuntu-latest
     timeout-minutes: 20
@@ -97,7 +97,7 @@ jobs:
         with:
           go-version: "1.20" # The Go version to download (if necessary) and use.
       - name: E2E slash-throttle tests
-        run: go run ./tests/e2e/... --tc slash-throttle/slash-throttle
+        run: go run ./tests/e2e/... --tc slash-throttle::slash-throttle
   multiconsumer-test:
     runs-on: ubuntu-latest
     timeout-minutes: 40
@@ -113,7 +113,7 @@ jobs:
         with:
           go-version: "1.20" # The Go version to download (if necessary) and use.
       - name: E2E multi-consumer tests
-        run: go run ./tests/e2e/... --tc multiconsumer/multiconsumer
+        run: go run ./tests/e2e/... --tc multiconsumer::multiconsumer
 
   nightly-test-fail:
     needs:

diff --git a/Makefile b/Makefile
@@ -31,12 +31,12 @@ test-diff:
 
 # run only happy path E2E tests
 test-e2e-short:
-	go run ./tests/e2e/... --tc happy-path/default
+	go run ./tests/e2e/... --tc happy-path::default
 
 # run only happy path E2E tests with cometmock
 # this set of traces does not test equivocation but it does check downtime
 test-e2e-short-cometmock:
-	go run ./tests/e2e/... --tc happy-path-short/default --use-cometmock --use-gorelayer
+	go run ./tests/e2e/... --tc happy-path-short::default --use-cometmock --use-gorelayer
 
 # run full E2E tests in sequence (including multiconsumer)
 test-e2e-multi-consumer:
@@ -52,7 +52,7 @@ test-gaia-e2e:
 
 # run only happy path E2E tests using latest tagged gaia
 test-gaia-e2e-short:
-	go run ./tests/e2e/... --tc happy-path/default --use-gaia
+	go run ./tests/e2e/... --tc happy-path::default --use-gaia
 
 # run full E2E tests in parallel (including multiconsumer) using latest tagged gaia
 test-gaia-e2e-parallel:
@@ -66,7 +66,7 @@ test-gaia-e2e-tagged:
 # run only happy path E2E tests using latest tagged gaia
 # usage: GAIA_TAG=v9.0.0 make test-gaia-e2e-short-tagged
 test-gaia-e2e-short-tagged:
-	go run ./tests/e2e/... --tc happy-path/default --use-gaia --gaia-tag $(GAIA_TAG)
+	go run ./tests/e2e/... --tc happy-path::default --use-gaia --gaia-tag $(GAIA_TAG)
 
 # run full E2E tests in parallel (including multiconsumer) using specific tagged version of gaia
 # usage: GAIA_TAG=v9.0.0 make test-gaia-e2e-parallel-tagged

diff --git a/tests/e2e/main.go b/tests/e2e/main.go
@@ -157,20 +157,36 @@ func getTestCaseUsageString() string {
 	builder.WriteString("\n")
 
 	// Example
-	builder.WriteString("Example: -tc multiconsumer/multiconsumer -tc happy-path/default")
+	builder.WriteString("Example: -tc multiconsumer::multiconsumer -tc happy-path::default")
 
 	return builder.String()
 }
 
 func getTestFileUsageString() string {
+	var builder strings.Builder
+
+	builder.WriteString("This flag is used to reference files containing step traces to be run.\n")
+	builder.WriteString("Each filename should be separated by '::' from the test runner name.\n")
+
+	// Test runner selection
+	builder.WriteString("Test runner selection:\nSelection of test runners to be executed:\n")
+	for _, testRunChoice := range testRuns {
+		builder.WriteString(fmt.Sprintf("- %s : %s\n", testRunChoice.name, testRunChoice.description))
+	}
+	builder.WriteString("\n")
+
+	// Example
+	builder.WriteString("Example: -test-file awesome-trace.json::default -test-file other-trace.json::default")
+
+	return builder.String()
 }
 
 func parseArguments() (err error) {
 	flag.Var(&testSelection, "tc",
 		getTestCaseUsageString())
 	flag.Parse()
 
-	flag.Var(&testSelection, "test-files",
+	flag.Var(&testSelection, "test-file",
 		getTestFileUsageString())
 	flag.Parse()
 
@@ -193,19 +209,19 @@ func getTestCases(selection TestSet) (tests []testRunWithSteps) {
 	// Run default tests if no test cases were selected
 	if len(selection) == 0 {
 		selection = TestSet{
-			"changeover/changeover", "happy-path/default",
-			"democracy-reward/democracy-reward", "democracy/democracy", "slash-throttle/slash-throttle",
+			"changeover::changeover", "happy-path::default",
+			"democracy-reward::democracy-reward", "democracy::democracy", "slash-throttle::slash-throttle",
 		}
 		if includeMultiConsumer != nil && *includeMultiConsumer {
-			selection = append(selection, "multiconsumer/multiconsumer")
+			selection = append(selection, "multiconsumer::multiconsumer")
 		}
 	}
 
 	// Get tests from selection
 	tests = []testRunWithSteps{}
 	for _, tc := range selection {
 		// first part of tc is the test runner, second part is the test case
-		splitTcString := strings.Split(tc, "/")
+		splitTcString := strings.Split(tc, "::")
 		if len(splitTcString) != 2 {
 			log.Fatalf("Test case '%s' is invalid.\nsee usage info:\n%s", tc, getTestCaseUsageString())
 		}

diff --git a/tests/e2e/trace_handlers_test.go b/tests/e2e/trace_handlers_test.go
@@ -90,14 +90,14 @@ func TestWriteExamples(t *testing.T) {
 	tests := map[string]struct {
 		trace []Step
 	}{
-		"start_provider_chain": {stepStartProviderChain()},
-		"happyPath":            {happyPathSteps},
-		"democracy":            {democracySteps},
-		"slashThrottle":        {slashThrottleSteps},
-		"multipleConsumers":    {multipleConsumers},
-		"shorthappy":           {shortHappyPathSteps},
-		"rewardDenomConsumer":  {rewardDenomConsumerSteps},
-		"changeover":           {changeoverSteps},
+		"start_provider_chain":  {stepStartProviderChain()},
+		"happyPath":             {happyPathSteps},
+		"democracy":             {democracySteps},
+		"slashThrottle":         {slashThrottleSteps},
+		"multipleConsumers":     {multipleConsumers},
+		"shorthappy":            {shortHappyPathSteps},
+		"democracyRewardsSteps": {democracyRewardsSteps},
+		"changeover":            {changeoverSteps},
 	}
 
 	dir := "tracehandler_testdata"