test: fix e2e testcase selection (#1706)

* fix e2e testcase selection

* tmp2

tests/e2e/actions.go

@@ -1040,12 +1040,13 @@ func (tr TestConfig) addIbcConnection(
 	if !tr.useGorelayer {
 		tr.addIbcConnectionHermes(action, target, verbose)
 	} else {
-		tr.addIbcConnectionGorelayer(action, verbose)
+		tr.addIbcConnectionGorelayer(action, target, verbose)
 	}
 }
 
 func (tr TestConfig) addIbcConnectionGorelayer(
 	action AddIbcConnectionAction,
+	target ExecutionTarget,
 	verbose bool,
 ) {
 	pathName := tr.GetPathNameForGorelayer(action.ChainA, action.ChainB)
@@ -1055,13 +1056,13 @@ func (tr TestConfig) addIbcConnectionGorelayer(
 	pathConfigFileName := fmt.Sprintf("/root/%s_config.json", pathName)
 
 	bashCommand := fmt.Sprintf(`echo '%s' >> %s`, pathConfig, pathConfigFileName)
+
 	//#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments.
-	pathConfigCommand := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "bash", "-c",
-		bashCommand)
+	pathConfigCommand := target.ExecCommand("bash", "-c", bashCommand)
 	executeCommand(pathConfigCommand, "add path config")
 
 	//#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments.
-	newPathCommand := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "rly",
+	newPathCommand := target.ExecCommand("rly",
 		"paths", "add",
 		string(tr.chainConfigs[action.ChainA].ChainId),
 		string(tr.chainConfigs[action.ChainB].ChainId),
@@ -1072,21 +1073,15 @@ func (tr TestConfig) addIbcConnectionGorelayer(
 	executeCommand(newPathCommand, "new path")
 
 	//#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments.
-	newClientsCommand := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "rly",
-		"transact", "clients",
-		pathName,
-	)
+	newClientsCommand := target.ExecCommand("rly", "transact", "clients", pathName)
 
 	executeCommand(newClientsCommand, "new clients")
 
 	tr.waitBlocks(action.ChainA, 1, 10*time.Second)
 	tr.waitBlocks(action.ChainB, 1, 10*time.Second)
 
 	//#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments.
-	newConnectionCommand := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "rly",
-		"transact", "connection",
-		pathName,
-	)
+	newConnectionCommand := target.ExecCommand("rly", "transact", "connection", pathName)
 
 	executeCommand(newConnectionCommand, "new connection")
 

tests/e2e/config.go

@@ -248,26 +248,30 @@ func GetTestConfig(cfgType TestConfigType, providerVersion, consumerVersion stri
 	cv := getIcsVersion(consumerVersion)
 	fmt.Println("Config version for provider :", pv)
 	fmt.Println("Config version for consumer :", cv)
+	var testCfg TestConfig
 	switch cfgType {
 	case DefaultTestCfg:
-		return DefaultTestConfig()
+		testCfg = DefaultTestConfig()
 	case ChangeoverTestCfg:
-		return ChangeoverTestConfig()
+		testCfg = ChangeoverTestConfig()
 	case DemocracyTestCfg:
-		return DemocracyTestConfig(false)
+		testCfg = DemocracyTestConfig(false)
 	case DemocracyRewardTestCfg:
-		return DemocracyTestConfig(true)
+		testCfg = DemocracyTestConfig(true)
 	case SlashThrottleTestCfg:
-		return SlashThrottleTestConfig()
+		testCfg = SlashThrottleTestConfig()
 	case MulticonsumerTestCfg:
-		return MultiConsumerTestConfig()
+		testCfg = MultiConsumerTestConfig()
 	case ConsumerMisbehaviourTestCfg:
-		return ConsumerMisbehaviourTestConfig()
+		testCfg = ConsumerMisbehaviourTestConfig()
 	case CompatibilityTestCfg:
-		return CompatibilityTestConfig(pv, cv)
+		testCfg = CompatibilityTestConfig(pv, cv)
 	default:
 		panic(fmt.Sprintf("Invalid test config: %s", cfgType))
 	}
+	testCfg.consumerVersion = consumerVersion
+	testCfg.providerVersion = providerVersion
+	return testCfg
 }
 
 func getDefaultValidators() map[ValidatorID]ValidatorConfig {

tests/e2e/main.go

@@ -250,17 +250,27 @@ func getTestCases(selectedPredefinedTests, selectedTestFiles TestSet, providerVe
 	tests = []testStepsWithConfig{}
 	// Get predefined from selection
 	for _, tc := range selectedPredefinedTests {
+		testConfig := TestConfigType("")
+		testSteps := []Step{}
+
 		// first part of tc is the steps, second part is the test config
+		splitTcString := strings.Split(tc, "::")
+		if len(splitTcString) == 2 {
+			tc = splitTcString[0]
+			testConfig = TestConfigType(splitTcString[1])
+		}
 
 		if _, exists := stepChoices[tc]; !exists {
 			log.Fatalf("Step choice '%s' not found.\nsee usage info:\n%s", tc, getTestCaseUsageString())
 		}
 
-		stepChoice := stepChoices[tc]
-
+		testSteps = stepChoices[tc].steps
+		if testConfig == "" {
+			testConfig = stepChoices[tc].testConfig
+		}
 		tests = append(tests, testStepsWithConfig{
-			config: stepChoice.testConfig,
-			steps:  stepChoice.steps,
+			config: testConfig,
+			steps:  testSteps,
 		},
 		)
 	}
@@ -297,19 +307,16 @@ func getTestCases(selectedPredefinedTests, selectedTestFiles TestSet, providerVe
 }
 
 // delete all test targets
-func deleteTargets(targets []ExecutionTarget) {
-	for _, target := range targets {
-		if err := target.Delete(); err != nil {
+func deleteTargets(runners []TestRunner) {
+	for _, runner := range runners {
+		if err := runner.target.Delete(); err != nil {
 			log.Println("error deleting target: ", err)
 		}
 	}
 }
 
-// Create targets where test cases should be executed on.
-// For each combination of provider & consumer versions an ExecutionTarget
-// is created.
-func createTargets(providerVersions, consumerVersions VersionSet) ([]ExecutionTarget, error) {
-	var targets []ExecutionTarget
+func createTestConfigs(cfgType TestConfigType, providerVersions, consumerVersions VersionSet) []TestConfig {
+	var configs []TestConfig
 
 	if len(consumerVersions) == 0 {
 		consumerVersions[""] = true
@@ -318,47 +325,39 @@ func createTargets(providerVersions, consumerVersions VersionSet) ([]ExecutionTa
 		providerVersions[""] = true
 	}
 
-	// Create targets as a combination of "provider versions" with "consumer version"
+	// Create test configs as a combination of "provider versions" with "consumer version" and "test case"
 	for provider, _ := range providerVersions {
-		targetCfg := TargetConfig{useGaia: *useGaia, localSdkPath: *localSdkPath, gaiaTag: *gaiaTag}
-		targetCfg.providerVersion = provider
 		for consumer, _ := range consumerVersions {
 			// Skip target creation for same version of provider and consumer
 			// if multiple versions need to be tested.
 			// This is to reduce the tests to be run for compatibility testing.
 			if (len(consumerVersions) > 1 || len(providerVersions) > 1) && consumer == provider {
 				continue
 			}
-			targetCfg.consumerVersion = consumer
-			target := DockerContainer{targetConfig: targetCfg}
-			targets = append(targets, &target)
-		}
-	}
-
-	for _, target := range targets {
-		err := target.Build()
-		if err != nil {
-			return targets, fmt.Errorf("failed building target %s\n: %v", target.Info(), err)
+			config := GetTestConfig(cfgType, provider, consumer)
+			config.SetRelayerConfig(*useGorelayer)
+			config.SetCometMockConfig(*useCometmock)
+			config.transformGenesis = *transformGenesis
+			config.useGorelayer = *useGorelayer
+			configs = append(configs, config)
 		}
 	}
-	return targets, nil
+	return configs
 }
 
 // createTestRunners creates test runners to run each test case on each target
-func createTestRunners(targets []ExecutionTarget, testCases []testStepsWithConfig) []TestRunner {
+func createTestRunners(testCases []testStepsWithConfig) []TestRunner {
 	runners := []TestRunner{}
-	for _, target := range targets {
-		for _, tc := range testCases {
-			providerVersion := target.GetTargetConfig().providerVersion
-			consumerVersion := target.GetTargetConfig().consumerVersion
-			config := GetTestConfig(tc.config, providerVersion, consumerVersion)
-			config.SetRelayerConfig(*useGorelayer)
-			config.SetCometMockConfig(*useCometmock)
-			config.transformGenesis = *transformGenesis
-			config.useGorelayer = *useGorelayer
-			err, tr := CreateTestRunner(config, tc.steps, target, *verbose)
+	targetCfg := TargetConfig{useGaia: *useGaia, localSdkPath: *localSdkPath, gaiaTag: *gaiaTag}
+
+	for _, tc := range testCases {
+		testConfigs := createTestConfigs(tc.config, providerVersions, consumerVersions)
+		for _, cfg := range testConfigs {
+			target, err := createTarget(cfg, targetCfg)
+			tr := CreateTestRunner(cfg, tc.steps, &target, *verbose)
 			if err == nil {
-				fmt.Println("Created test runner for provider", config.name, "with provVers=", providerVersion, "consVers=", consumerVersion)
+				fmt.Println("Created test runner for ", cfg.name,
+					"with provVers=", cfg.providerVersion, "consVers=", cfg.consumerVersion)
 				runners = append(runners, tr)
 			} else {
 				fmt.Println("No test runner created:", err)
@@ -417,36 +416,44 @@ func printReport(runners []TestRunner, duration time.Duration) {
 		}
 	}
 	numTotalTests := len(runners)
-	report := fmt.Sprintf(`
+	report := `
 =================================================
+TEST RESULTS
+-------------------------------------------------
+
+`
+	if len(failedTests) > 0 {
+		report += fmt.Sprintln("\nFAILED TESTS:")
+		for _, t := range failedTests {
+			report += t.Report()
+		}
+	}
+	if len(passedTests) > 0 {
+		report += fmt.Sprintln("\n\nPASSED TESTS:\n")
+		for _, t := range passedTests {
+			report += t.Report()
+		}
+	}
+	if len(remainingTests) > 0 {
+		report += fmt.Sprintln("\n\nREMAINING TESTS:\n")
+		for _, t := range remainingTests {
+			report += t.Report()
+		}
+	}
+	report += fmt.Sprintf(`
+-------------------------------------------------
 Summary:
 - time elapsed: %s
 - %d/%d tests passed
 - %d/%d tests failed
 - %d/%d tests with misc status (check details)
--------------------------------------------------
-
-
 `,
 		duration,
 		len(passedTests), numTotalTests,
 		len(failedTests), numTotalTests,
 		len(remainingTests), numTotalTests,
 	)
 
-	report += fmt.Sprintln("\nFAILED TESTS:")
-	for _, t := range failedTests {
-		report += t.Report()
-	}
-	report += fmt.Sprintln("\n\nPASSED TESTS:\n")
-	for _, t := range passedTests {
-		report += t.Report()
-	}
-
-	report += fmt.Sprintln("\n\nREMAINING TESTS:\n")
-	for _, t := range remainingTests {
-		report += t.Report()
-	}
 	report += "=================================================="
 	fmt.Print(report)
 }
@@ -461,15 +468,11 @@ func main() {
 	}
 
 	testCases := getTestCases(selectedTests, selectedTestfiles, providerVersions, consumerVersions)
-	targets, err := createTargets(providerVersions, consumerVersions)
-	if err != nil {
-		log.Fatal("failed creating test targets: ", err)
-	}
-	defer func() { deleteTargets(targets) }()
+	testRunners := createTestRunners(testCases)
+	defer deleteTargets(testRunners)
 
-	testRunners := createTestRunners(targets, testCases)
 	start := time.Now()
-	err = executeTests(testRunners)
+	err := executeTests(testRunners)
 	if err != nil {
 		log.Fatalf("Test execution failed '%s'", err)
 	}

tests/e2e/test_runner.go

@@ -117,20 +117,14 @@ func (tr *TestRunner) Setup(testCfg TestConfig) error {
 	return nil
 }
 
-func CreateTestRunner(config TestConfig, steps []Step, target ExecutionTarget, verbose bool) (error, TestRunner) {
-	//targetConfig := target.GetTargetConfig()
-	switch target.(type) {
-	case *DockerContainer:
-		target.(*DockerContainer).containerCfg = config.containerConfig
-	}
-	tr := TestRunner{
+func CreateTestRunner(config TestConfig, steps []Step, target ExecutionTarget, verbose bool) TestRunner {
+	return TestRunner{
 		target:  target,
 		steps:   steps,
 		config:  config,
 		verbose: verbose,
 		result:  TestResult{Status: TEST_STATUS_NOTRUN},
 	}
-	return nil, tr
 }
 
 // Info returns a header string containing useful information about the test runner

tests/e2e/test_target.go

@@ -32,6 +32,21 @@ type DockerContainer struct {
 	ImageName    string
 }
 
+func createTarget(testCfg TestConfig, targetCfg TargetConfig) (DockerContainer, error) {
+	targetCfg.providerVersion = testCfg.providerVersion
+	targetCfg.consumerVersion = testCfg.consumerVersion
+	target := DockerContainer{
+		targetConfig: targetCfg,
+		containerCfg: testCfg.containerConfig,
+	}
+
+	err := target.Build()
+	if err != nil {
+		return target, fmt.Errorf("failed building target %s\n: %v", target.Info(), err)
+	}
+	return target, nil
+}
+
 func (dc *DockerContainer) GetTargetConfig() TargetConfig {
 	return dc.targetConfig
 }