From cd0d0753f9f5427bd2f5a1f693be7be7f9bd9514 Mon Sep 17 00:00:00 2001 From: bernd-m <43466467+bermuell@users.noreply.github.com> Date: Mon, 18 Mar 2024 09:57:30 +0100 Subject: [PATCH] test: fix e2e testcase selection (#1706) * fix e2e testcase selection * tmp2 --- tests/e2e/actions.go | 19 +++--- tests/e2e/config.go | 20 +++--- tests/e2e/main.go | 127 ++++++++++++++++++++------------------- tests/e2e/test_runner.go | 10 +-- tests/e2e/test_target.go | 15 +++++ 5 files changed, 101 insertions(+), 90 deletions(-) diff --git a/tests/e2e/actions.go b/tests/e2e/actions.go index 9db1bac2ca..5ef7c58c6a 100644 --- a/tests/e2e/actions.go +++ b/tests/e2e/actions.go @@ -1040,12 +1040,13 @@ func (tr TestConfig) addIbcConnection( if !tr.useGorelayer { tr.addIbcConnectionHermes(action, target, verbose) } else { - tr.addIbcConnectionGorelayer(action, verbose) + tr.addIbcConnectionGorelayer(action, target, verbose) } } func (tr TestConfig) addIbcConnectionGorelayer( action AddIbcConnectionAction, + target ExecutionTarget, verbose bool, ) { pathName := tr.GetPathNameForGorelayer(action.ChainA, action.ChainB) @@ -1055,13 +1056,13 @@ func (tr TestConfig) addIbcConnectionGorelayer( pathConfigFileName := fmt.Sprintf("/root/%s_config.json", pathName) bashCommand := fmt.Sprintf(`echo '%s' >> %s`, pathConfig, pathConfigFileName) + //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - pathConfigCommand := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "bash", "-c", - bashCommand) + pathConfigCommand := target.ExecCommand("bash", "-c", bashCommand) executeCommand(pathConfigCommand, "add path config") //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - newPathCommand := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "rly", + newPathCommand := target.ExecCommand("rly", "paths", "add", string(tr.chainConfigs[action.ChainA].ChainId), string(tr.chainConfigs[action.ChainB].ChainId), @@ -1072,10 +1073,7 @@ func (tr TestConfig) addIbcConnectionGorelayer( executeCommand(newPathCommand, "new path") //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - newClientsCommand := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "rly", - "transact", "clients", - pathName, - ) + newClientsCommand := target.ExecCommand("rly", "transact", "clients", pathName) executeCommand(newClientsCommand, "new clients") @@ -1083,10 +1081,7 @@ func (tr TestConfig) addIbcConnectionGorelayer( tr.waitBlocks(action.ChainB, 1, 10*time.Second) //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - newConnectionCommand := exec.Command("docker", "exec", tr.containerConfig.InstanceName, "rly", - "transact", "connection", - pathName, - ) + newConnectionCommand := target.ExecCommand("rly", "transact", "connection", pathName) executeCommand(newConnectionCommand, "new connection") diff --git a/tests/e2e/config.go b/tests/e2e/config.go index e530ca3a9d..4ae8133568 100644 --- a/tests/e2e/config.go +++ b/tests/e2e/config.go @@ -248,26 +248,30 @@ func GetTestConfig(cfgType TestConfigType, providerVersion, consumerVersion stri cv := getIcsVersion(consumerVersion) fmt.Println("Config version for provider :", pv) fmt.Println("Config version for consumer :", cv) + var testCfg TestConfig switch cfgType { case DefaultTestCfg: - return DefaultTestConfig() + testCfg = DefaultTestConfig() case ChangeoverTestCfg: - return ChangeoverTestConfig() + testCfg = ChangeoverTestConfig() case DemocracyTestCfg: - return DemocracyTestConfig(false) + testCfg = DemocracyTestConfig(false) case DemocracyRewardTestCfg: - return DemocracyTestConfig(true) + testCfg = DemocracyTestConfig(true) case SlashThrottleTestCfg: - return SlashThrottleTestConfig() + testCfg = SlashThrottleTestConfig() case MulticonsumerTestCfg: - return MultiConsumerTestConfig() + testCfg = MultiConsumerTestConfig() case ConsumerMisbehaviourTestCfg: - return ConsumerMisbehaviourTestConfig() + testCfg = ConsumerMisbehaviourTestConfig() case CompatibilityTestCfg: - return CompatibilityTestConfig(pv, cv) + testCfg = CompatibilityTestConfig(pv, cv) default: panic(fmt.Sprintf("Invalid test config: %s", cfgType)) } + testCfg.consumerVersion = consumerVersion + testCfg.providerVersion = providerVersion + return testCfg } func getDefaultValidators() map[ValidatorID]ValidatorConfig { diff --git a/tests/e2e/main.go b/tests/e2e/main.go index a14610def5..4a8e17be8b 100644 --- a/tests/e2e/main.go +++ b/tests/e2e/main.go @@ -250,17 +250,27 @@ func getTestCases(selectedPredefinedTests, selectedTestFiles TestSet, providerVe tests = []testStepsWithConfig{} // Get predefined from selection for _, tc := range selectedPredefinedTests { + testConfig := TestConfigType("") + testSteps := []Step{} + // first part of tc is the steps, second part is the test config + splitTcString := strings.Split(tc, "::") + if len(splitTcString) == 2 { + tc = splitTcString[0] + testConfig = TestConfigType(splitTcString[1]) + } if _, exists := stepChoices[tc]; !exists { log.Fatalf("Step choice '%s' not found.\nsee usage info:\n%s", tc, getTestCaseUsageString()) } - stepChoice := stepChoices[tc] - + testSteps = stepChoices[tc].steps + if testConfig == "" { + testConfig = stepChoices[tc].testConfig + } tests = append(tests, testStepsWithConfig{ - config: stepChoice.testConfig, - steps: stepChoice.steps, + config: testConfig, + steps: testSteps, }, ) } @@ -297,19 +307,16 @@ func getTestCases(selectedPredefinedTests, selectedTestFiles TestSet, providerVe } // delete all test targets -func deleteTargets(targets []ExecutionTarget) { - for _, target := range targets { - if err := target.Delete(); err != nil { +func deleteTargets(runners []TestRunner) { + for _, runner := range runners { + if err := runner.target.Delete(); err != nil { log.Println("error deleting target: ", err) } } } -// Create targets where test cases should be executed on. -// For each combination of provider & consumer versions an ExecutionTarget -// is created. -func createTargets(providerVersions, consumerVersions VersionSet) ([]ExecutionTarget, error) { - var targets []ExecutionTarget +func createTestConfigs(cfgType TestConfigType, providerVersions, consumerVersions VersionSet) []TestConfig { + var configs []TestConfig if len(consumerVersions) == 0 { consumerVersions[""] = true @@ -318,10 +325,8 @@ func createTargets(providerVersions, consumerVersions VersionSet) ([]ExecutionTa providerVersions[""] = true } - // Create targets as a combination of "provider versions" with "consumer version" + // Create test configs as a combination of "provider versions" with "consumer version" and "test case" for provider, _ := range providerVersions { - targetCfg := TargetConfig{useGaia: *useGaia, localSdkPath: *localSdkPath, gaiaTag: *gaiaTag} - targetCfg.providerVersion = provider for consumer, _ := range consumerVersions { // Skip target creation for same version of provider and consumer // if multiple versions need to be tested. @@ -329,36 +334,30 @@ func createTargets(providerVersions, consumerVersions VersionSet) ([]ExecutionTa if (len(consumerVersions) > 1 || len(providerVersions) > 1) && consumer == provider { continue } - targetCfg.consumerVersion = consumer - target := DockerContainer{targetConfig: targetCfg} - targets = append(targets, &target) - } - } - - for _, target := range targets { - err := target.Build() - if err != nil { - return targets, fmt.Errorf("failed building target %s\n: %v", target.Info(), err) + config := GetTestConfig(cfgType, provider, consumer) + config.SetRelayerConfig(*useGorelayer) + config.SetCometMockConfig(*useCometmock) + config.transformGenesis = *transformGenesis + config.useGorelayer = *useGorelayer + configs = append(configs, config) } } - return targets, nil + return configs } // createTestRunners creates test runners to run each test case on each target -func createTestRunners(targets []ExecutionTarget, testCases []testStepsWithConfig) []TestRunner { +func createTestRunners(testCases []testStepsWithConfig) []TestRunner { runners := []TestRunner{} - for _, target := range targets { - for _, tc := range testCases { - providerVersion := target.GetTargetConfig().providerVersion - consumerVersion := target.GetTargetConfig().consumerVersion - config := GetTestConfig(tc.config, providerVersion, consumerVersion) - config.SetRelayerConfig(*useGorelayer) - config.SetCometMockConfig(*useCometmock) - config.transformGenesis = *transformGenesis - config.useGorelayer = *useGorelayer - err, tr := CreateTestRunner(config, tc.steps, target, *verbose) + targetCfg := TargetConfig{useGaia: *useGaia, localSdkPath: *localSdkPath, gaiaTag: *gaiaTag} + + for _, tc := range testCases { + testConfigs := createTestConfigs(tc.config, providerVersions, consumerVersions) + for _, cfg := range testConfigs { + target, err := createTarget(cfg, targetCfg) + tr := CreateTestRunner(cfg, tc.steps, &target, *verbose) if err == nil { - fmt.Println("Created test runner for provider", config.name, "with provVers=", providerVersion, "consVers=", consumerVersion) + fmt.Println("Created test runner for ", cfg.name, + "with provVers=", cfg.providerVersion, "consVers=", cfg.consumerVersion) runners = append(runners, tr) } else { fmt.Println("No test runner created:", err) @@ -417,16 +416,37 @@ func printReport(runners []TestRunner, duration time.Duration) { } } numTotalTests := len(runners) - report := fmt.Sprintf(` + report := ` ================================================= +TEST RESULTS +------------------------------------------------- + +` + if len(failedTests) > 0 { + report += fmt.Sprintln("\nFAILED TESTS:") + for _, t := range failedTests { + report += t.Report() + } + } + if len(passedTests) > 0 { + report += fmt.Sprintln("\n\nPASSED TESTS:\n") + for _, t := range passedTests { + report += t.Report() + } + } + if len(remainingTests) > 0 { + report += fmt.Sprintln("\n\nREMAINING TESTS:\n") + for _, t := range remainingTests { + report += t.Report() + } + } + report += fmt.Sprintf(` +------------------------------------------------- Summary: - time elapsed: %s - %d/%d tests passed - %d/%d tests failed - %d/%d tests with misc status (check details) -------------------------------------------------- - - `, duration, len(passedTests), numTotalTests, @@ -434,19 +454,6 @@ Summary: len(remainingTests), numTotalTests, ) - report += fmt.Sprintln("\nFAILED TESTS:") - for _, t := range failedTests { - report += t.Report() - } - report += fmt.Sprintln("\n\nPASSED TESTS:\n") - for _, t := range passedTests { - report += t.Report() - } - - report += fmt.Sprintln("\n\nREMAINING TESTS:\n") - for _, t := range remainingTests { - report += t.Report() - } report += "==================================================" fmt.Print(report) } @@ -461,15 +468,11 @@ func main() { } testCases := getTestCases(selectedTests, selectedTestfiles, providerVersions, consumerVersions) - targets, err := createTargets(providerVersions, consumerVersions) - if err != nil { - log.Fatal("failed creating test targets: ", err) - } - defer func() { deleteTargets(targets) }() + testRunners := createTestRunners(testCases) + defer deleteTargets(testRunners) - testRunners := createTestRunners(targets, testCases) start := time.Now() - err = executeTests(testRunners) + err := executeTests(testRunners) if err != nil { log.Fatalf("Test execution failed '%s'", err) } diff --git a/tests/e2e/test_runner.go b/tests/e2e/test_runner.go index af4ea51eb3..4fac32df11 100644 --- a/tests/e2e/test_runner.go +++ b/tests/e2e/test_runner.go @@ -117,20 +117,14 @@ func (tr *TestRunner) Setup(testCfg TestConfig) error { return nil } -func CreateTestRunner(config TestConfig, steps []Step, target ExecutionTarget, verbose bool) (error, TestRunner) { - //targetConfig := target.GetTargetConfig() - switch target.(type) { - case *DockerContainer: - target.(*DockerContainer).containerCfg = config.containerConfig - } - tr := TestRunner{ +func CreateTestRunner(config TestConfig, steps []Step, target ExecutionTarget, verbose bool) TestRunner { + return TestRunner{ target: target, steps: steps, config: config, verbose: verbose, result: TestResult{Status: TEST_STATUS_NOTRUN}, } - return nil, tr } // Info returns a header string containing useful information about the test runner diff --git a/tests/e2e/test_target.go b/tests/e2e/test_target.go index 845ede8481..6fcec7de1b 100644 --- a/tests/e2e/test_target.go +++ b/tests/e2e/test_target.go @@ -32,6 +32,21 @@ type DockerContainer struct { ImageName string } +func createTarget(testCfg TestConfig, targetCfg TargetConfig) (DockerContainer, error) { + targetCfg.providerVersion = testCfg.providerVersion + targetCfg.consumerVersion = testCfg.consumerVersion + target := DockerContainer{ + targetConfig: targetCfg, + containerCfg: testCfg.containerConfig, + } + + err := target.Build() + if err != nil { + return target, fmt.Errorf("failed building target %s\n: %v", target.Info(), err) + } + return target, nil +} + func (dc *DockerContainer) GetTargetConfig() TargetConfig { return dc.targetConfig }