From cb4c52e429858e378aa5c4d3b9c365a2f374bf67 Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Mon, 16 Sep 2024 09:49:12 -0700 Subject: [PATCH] Add new fields to the metrics publish libraries Signed-off-by: Prudhvi Godithi --- build.gradle | 2 +- ...TestPublishDistributionBuildResults.groovy | 17 +++-- .../TestPublishIntegTestResults.groovy | 38 +++++++++-- vars/publishDistributionBuildResults.groovy | 14 ++-- vars/publishIntegTestResults.groovy | 66 ++++++++++++++----- 5 files changed, 104 insertions(+), 33 deletions(-) diff --git a/build.gradle b/build.gradle index dca8e5be..3c48cbdd 100644 --- a/build.gradle +++ b/build.gradle @@ -127,7 +127,7 @@ jacocoTestReport { } } -String version = '6.9.0' +String version = '6.9.1' task updateVersion { doLast { diff --git a/tests/jenkins/TestPublishDistributionBuildResults.groovy b/tests/jenkins/TestPublishDistributionBuildResults.groovy index 87f8c50f..41831ab7 100644 --- a/tests/jenkins/TestPublishDistributionBuildResults.groovy +++ b/tests/jenkins/TestPublishDistributionBuildResults.groovy @@ -74,6 +74,9 @@ class TestPublishDistributionBuildResults extends BuildPipelineTest { "component_repo": { "type": "keyword" }, + "component_repo_url": { + "type": "keyword" + }, "component_ref": { "type": "keyword" }, @@ -132,14 +135,15 @@ class TestPublishDistributionBuildResults extends BuildPipelineTest { void testGenerateJson() { def script = loadScript('vars/publishDistributionBuildResults.groovy') def result = script.generateJson( - 'component1', 'repo1', 'ref1', '1.0', 123, + 'component1', 'componentRepo', 'https://componentRepoUrl', 'ref1', '1.0', 123, 'http://example.com/build/123', System.currentTimeMillis(), 'rc1', 1, 'test-category', 'failed' ) def parsedResult = new JsonSlurper().parseText(result) def expectedJson = [ component: 'component1', - component_repo: 'repo1', + component_repo: 'componentRepo', + component_repo_url: 'https://componentRepoUrl', component_ref: 'ref1', version: '1.0', distribution_build_number: 123, @@ -166,7 +170,8 @@ class TestPublishDistributionBuildResults extends BuildPipelineTest { // Test valid parameters def indexName = "test-index" def component = "componentA" - def componentRepo = "repoA" + def componentRepo = "componentRepo" + def componentRepoUrl = "https://componentRepoUrl" def componentRef = "refA" def version = "1.0.0" def distributionBuildNumber = "123" @@ -177,10 +182,11 @@ class TestPublishDistributionBuildResults extends BuildPipelineTest { def componentCategory = "categoryA" def status = "success" - def result = script.generateAndAppendJson(indexName, component, componentRepo, componentRef, version, distributionBuildNumber, distributionBuildUrl, buildStartTime, rc, rcNumber, componentCategory, status) + def result = script.generateAndAppendJson(component, componentRepo, componentRepoUrl, componentRef, version, distributionBuildNumber, distributionBuildUrl, buildStartTime, rc, rcNumber, componentCategory, status) def expectedJson = JsonOutput.toJson([ component: component, component_repo: componentRepo, + component_repo_url: componentRepoUrl, component_ref: componentRef, version: version, distribution_build_number: distributionBuildNumber, @@ -193,10 +199,11 @@ class TestPublishDistributionBuildResults extends BuildPipelineTest { ]) assert result == expectedJson - result = script.generateAndAppendJson(indexName, null, null, null, null, null, null, null, null, null, null, null) + result = script.generateAndAppendJson(null, null, null, null, null, null, null, null, null, null, null, null) expectedJson = JsonOutput.toJson([ component: null, component_repo: null, + component_repo_url: null, component_ref: null, version: null, distribution_build_number: null, diff --git a/tests/jenkins/TestPublishIntegTestResults.groovy b/tests/jenkins/TestPublishIntegTestResults.groovy index f8184678..4f875b4c 100644 --- a/tests/jenkins/TestPublishIntegTestResults.groovy +++ b/tests/jenkins/TestPublishIntegTestResults.groovy @@ -71,6 +71,12 @@ class TestPublishIntegTestResults extends BuildPipelineTest { "component": { "type": "keyword" }, + "component_repo": { + "type": "keyword" + }, + "component_repo_url": { + "type": "keyword" + }, "version": { "type": "keyword" }, @@ -123,9 +129,15 @@ class TestPublishIntegTestResults extends BuildPipelineTest { "with_security_cluster_stdout": { "type": "keyword" }, + "with_security_test_stdout": { + "type": "keyword" + }, "with_security_cluster_stderr": { "type": "keyword" }, + "with_security_test_stderr": { + "type": "keyword" + }, "without_security": { "type": "keyword" }, @@ -135,8 +147,14 @@ class TestPublishIntegTestResults extends BuildPipelineTest { "without_security_cluster_stdout": { "type": "keyword" }, + "without_security_test_stdout": { + "type": "keyword" + }, "without_security_cluster_stderr": { "type": "keyword" + }, + "without_security_test_stderr": { + "type": "keyword" } } } @@ -169,16 +187,18 @@ class TestPublishIntegTestResults extends BuildPipelineTest { void testGenerateJson() { def script = loadScript('vars/publishIntegTestResults.groovy') def result = script.generateJson( - 'component1', '1.0', 123, + 'component1', 'componentRepo', 'https://componentRepoUrl', '1.0', 123, 'http://example.com/build/123', 456, 'http://example.com/distribution/456', System.currentTimeMillis(), 'rc1', 1, 'linux', 'x64', 'tar', 'test-category', - 'failed', 'http://example.com/test-report.yml', 'pass', 'yml1', ['stdout1'], ['stderr1'], - 'fail', 'yml2', ['stdout2'], ['stderr2'] + 'failed', 'http://example.com/test-report.yml', 'pass', 'yml1', ['cluster_stdout1'], ['cluster_stderr1'], ['test_stdout1'], ['test_stderr1'], + 'fail', 'yml2', ['cluster_stdout2'], ['cluster_stderr2'], ['test_stdout2'], ['test_stderr2'] ) def parsedResult = new JsonSlurper().parseText(result) def expectedJson = [ component: 'component1', + component_repo: 'componentRepo', + component_repo_url: 'https://componentRepoUrl', version: '1.0', integ_test_build_number: 123, integ_test_build_url: 'http://example.com/build/123', @@ -195,12 +215,16 @@ class TestPublishIntegTestResults extends BuildPipelineTest { test_report_manifest_yml: 'http://example.com/test-report.yml', with_security: 'pass', with_security_build_yml: 'yml1', - with_security_cluster_stdout: ['stdout1'], - with_security_cluster_stderr: ['stderr1'], + with_security_cluster_stdout: ['cluster_stdout1'], + with_security_cluster_stderr: ['cluster_stderr1'], + with_security_test_stdout: ['test_stdout1'], + with_security_test_stderr: ['test_stderr1'], without_security: 'fail', without_security_build_yml: 'yml2', - without_security_cluster_stdout: ['stdout2'], - without_security_cluster_stderr: ['stderr2'] + without_security_cluster_stdout: ['cluster_stdout2'], + without_security_cluster_stderr: ['cluster_stderr2'], + without_security_test_stdout: ['test_stdout2'], + without_security_test_stderr: ['test_stderr2'] ] // Remove the dynamic field for comparison diff --git a/vars/publishDistributionBuildResults.groovy b/vars/publishDistributionBuildResults.groovy index 735955b3..a762e2a9 100644 --- a/vars/publishDistributionBuildResults.groovy +++ b/vars/publishDistributionBuildResults.groovy @@ -41,13 +41,13 @@ void call(Map args = [:]) { List passedComponents = extractComponents(passMessages, /(?<=\bSuccessfully built\s).*/, 0) inputManifest.components.each { component -> if (failedComponents.contains(component.name)) { - def jsonData = generateAndAppendJson(indexName, component.name, component.repository, component.ref, + def jsonData = generateAndAppendJson(component.name, component.repository.split('/')[-1].replace('.git', ''), component.repository.substring(component.repository.indexOf("github.com")).replace(".git", ""), component.ref, version, distributionBuildNumber, distributionBuildUrl, buildStartTime, rc, rcNumber, componentCategory, "failed" ) finalJsonDoc += "{\"index\": {\"_index\": \"${indexName}\"}}\n${jsonData}\n" } else if (passedComponents.contains(component.name)) { - def jsonData = generateAndAppendJson(indexName, component.name, component.repository, component.ref, + def jsonData = generateAndAppendJson(component.name, component.repository.split('/')[-1].replace('.git', ''), component.repository.substring(component.repository.indexOf("github.com")).replace(".git", ""), component.ref, version, distributionBuildNumber, distributionBuildUrl, buildStartTime, rc, rcNumber, componentCategory, "passed" ) @@ -82,6 +82,9 @@ void indexFailedTestData(indexName, testRecordsFile) { "component_repo": { "type": "keyword" }, + "component_repo_url": { + "type": "keyword" + }, "component_ref": { "type": "keyword" }, @@ -137,10 +140,11 @@ void indexFailedTestData(indexName, testRecordsFile) { } } -def generateJson(component, componentRepo, componentRef, version, distributionBuildNumber, distributionBuildUrl, buildStartTime, rc, rcNumber, componentCategory, componentResult) { +def generateJson(component, componentRepo, componentRepoUrl, componentRef, version, distributionBuildNumber, distributionBuildUrl, buildStartTime, rc, rcNumber, componentCategory, componentResult) { def json = [ component: component, component_repo: componentRepo, + component_repo_url: componentRepoUrl, component_ref: componentRef, version: version, distribution_build_number: distributionBuildNumber, @@ -154,9 +158,9 @@ def generateJson(component, componentRepo, componentRef, version, distributionBu return JsonOutput.toJson(json) } -def generateAndAppendJson(indexName, component, componentRepo, componentRef, version, distributionBuildNumber, distributionBuildUrl, buildStartTime, rc, rcNumber, componentCategory, status) { +def generateAndAppendJson(component, componentRepo, componentRepoUrl, componentRef, version, distributionBuildNumber, distributionBuildUrl, buildStartTime, rc, rcNumber, componentCategory, status) { def jsonData = generateJson( - component, componentRepo, componentRef, version, + component, componentRepo, componentRepoUrl, componentRef, version, distributionBuildNumber, distributionBuildUrl, buildStartTime, rc, rcNumber, componentCategory, status ) diff --git a/vars/publishIntegTestResults.groovy b/vars/publishIntegTestResults.groovy index 092bda5f..75910e41 100644 --- a/vars/publishIntegTestResults.groovy +++ b/vars/publishIntegTestResults.groovy @@ -54,25 +54,32 @@ void call(Map args = [:]) { manifest.components.each { component -> def componentName = component.name + def componentRepo = component.repository.split('/')[-1].replace('.git', '') + def componentRepoUrl = component.repository.substring(component.repository.indexOf("github.com")).replace(".git", "") def componentCategory = manifest.name def withSecurity = component.configs.find { it.name == 'with-security' }?.status?.toLowerCase() ?: 'unknown' def withoutSecurity = component.configs.find { it.name == 'without-security' }?.status?.toLowerCase() ?: 'unknown' def componentResult = (withSecurity == 'fail' || withoutSecurity == 'fail' || withSecurity == 'Not Available' || withoutSecurity == 'Not Available') ? 'failed' : 'passed' def withSecurityYml = component.configs.find { it.name == 'with-security' }?.yml ?: '' - def withSecurityStdout = component.configs.find { it.name == 'with-security' }?.cluster_stdout ?: [] - def withSecurityStderr = component.configs.find { it.name == 'with-security' }?.cluster_stderr ?: [] + def withSecurityClusterStdout = component.configs.find { it.name == 'with-security' }?.cluster_stdout ?: [] + def withSecurityClusterStderr = component.configs.find { it.name == 'with-security' }?.cluster_stderr ?: [] + def withSecurityTestStdout = component.configs.find { it.name == 'with-security' }?.test_stdout ?: '' + def withSecurityTestStderr = component.configs.find { it.name == 'with-security' }?.test_stderr ?: '' def withoutSecurityYml = component.configs.find { it.name == 'without-security' }?.yml ?: '' - def withoutSecurityStdout = component.configs.find { it.name == 'without-security' }?.cluster_stdout ?: [] - def withoutSecurityStderr = component.configs.find { it.name == 'without-security' }?.cluster_stderr ?: [] + def withoutSecurityClusterStdout = component.configs.find { it.name == 'without-security' }?.cluster_stdout ?: [] + def withoutSecurityClusterStderr = component.configs.find { it.name == 'without-security' }?.cluster_stderr ?: [] + def withoutSecurityTestStdout = component.configs.find { it.name == 'without-security' }?.test_stdout ?: '' + def withoutSecurityTestStderr = component.configs.find { it.name == 'without-security' }?.test_stderr ?: '' def jsonContent = generateJson( - componentName, version, integTestBuildNumber, + componentName, componentRepo, componentRepoUrl, version, integTestBuildNumber, integTestBuildUrl, distributionBuildNumber, distributionBuildUrl, buildStartTime, rc, rcNumber, platform, architecture, distribution, componentCategory, componentResult, testReportManifestYmlUrl, - withSecurity, withSecurityYml, withSecurityStdout, - withSecurityStderr, withoutSecurity, withoutSecurityYml, - withoutSecurityStdout, withoutSecurityStderr + withSecurity, withSecurityYml, withSecurityClusterStdout, + withSecurityClusterStderr, withSecurityTestStdout, withSecurityTestStderr, + withoutSecurity, withoutSecurityYml, withoutSecurityClusterStdout, withoutSecurityClusterStderr, + withoutSecurityTestStdout, withoutSecurityTestStderr ) finalJsonDoc += "{\"index\": {\"_index\": \"${indexName}\"}}\n" + "${jsonContent}\n" } @@ -102,6 +109,12 @@ void indexFailedTestData(indexName, testRecordsFile) { "component": { "type": "keyword" }, + "component_repo": { + "type": "keyword" + }, + "component_repo_url": { + "type": "keyword" + }, "version": { "type": "keyword" }, @@ -154,9 +167,15 @@ void indexFailedTestData(indexName, testRecordsFile) { "with_security_cluster_stdout": { "type": "keyword" }, + "with_security_test_stdout": { + "type": "keyword" + }, "with_security_cluster_stderr": { "type": "keyword" }, + "with_security_test_stderr": { + "type": "keyword" + }, "without_security": { "type": "keyword" }, @@ -166,8 +185,14 @@ void indexFailedTestData(indexName, testRecordsFile) { "without_security_cluster_stdout": { "type": "keyword" }, + "without_security_test_stdout": { + "type": "keyword" + }, "without_security_cluster_stderr": { "type": "keyword" + }, + "without_security_test_stderr": { + "type": "keyword" } } } @@ -196,9 +221,17 @@ void indexFailedTestData(indexName, testRecordsFile) { } } -def generateJson(component, version, integTestBuildNumber, integTestBuildUrl, distributionBuildNumber, distributionBuildUrl, buildStartTime, rc, rcNumber, platform, architecture, distribution, componentCategory, componentResult, testReportManifestYmlUrl, withSecurity, withSecurityYml, withSecurityStdout, withSecurityStderr, withoutSecurity, withoutSecurityYml, withoutSecurityStdout, withoutSecurityStderr) { +def generateJson(componentName, componentRepo, componentRepoUrl, version, + integTestBuildNumber, integTestBuildUrl, distributionBuildNumber, distributionBuildUrl, + buildStartTime, rc, rcNumber, platform, architecture, distribution, componentCategory, + componentResult, testReportManifestYmlUrl, withSecurity, withSecurityYml, withSecurityClusterStdout, + withSecurityClusterStderr, withSecurityTestStdout,withSecurityTestStderr, withoutSecurity, + withoutSecurityYml, withoutSecurityClusterStdout, withoutSecurityClusterStderr, withoutSecurityTestStdout, + withoutSecurityTestStderr) { def json = [ - component: component, + component: componentName, + component_repo: componentRepo, + component_repo_url: componentRepoUrl, version: version, integ_test_build_number: integTestBuildNumber, integ_test_build_url: integTestBuildUrl, @@ -215,13 +248,16 @@ def generateJson(component, version, integTestBuildNumber, integTestBuildUrl, di test_report_manifest_yml: testReportManifestYmlUrl, with_security: withSecurity, with_security_build_yml: withSecurityYml, - with_security_cluster_stdout: withSecurityStdout, - with_security_cluster_stderr: withSecurityStderr, + with_security_cluster_stdout: withSecurityClusterStdout, + with_security_cluster_stderr: withSecurityClusterStderr, + with_security_test_stdout: withSecurityTestStdout, + with_security_test_stderr: withSecurityTestStderr, without_security: withoutSecurity, without_security_build_yml: withoutSecurityYml, - without_security_cluster_stdout: withoutSecurityStdout, - without_security_cluster_stderr: withoutSecurityStderr + without_security_cluster_stdout: withoutSecurityClusterStdout, + without_security_cluster_stderr: withoutSecurityClusterStderr, + without_security_test_stdout: withoutSecurityTestStdout, + without_security_test_stderr: withoutSecurityTestStderr ] return JsonOutput.toJson(json) } -