diff --git a/build.gradle b/build.gradle index a903150fb..fb9836091 100644 --- a/build.gradle +++ b/build.gradle @@ -120,7 +120,7 @@ jacocoTestReport { } } -String version = '5.9.2' +String version = '5.10.0' task updateVersion { doLast { diff --git a/tests/jenkins/TestRunBenchmarkTestScript.groovy b/tests/jenkins/TestRunBenchmarkTestScript.groovy index 8a27d454b..9df08c71f 100644 --- a/tests/jenkins/TestRunBenchmarkTestScript.groovy +++ b/tests/jenkins/TestRunBenchmarkTestScript.groovy @@ -22,6 +22,8 @@ class TestRunBenchmarkTestScript extends BuildPipelineTest { void setUp() { this.registerLibTester(new RunBenchmarkTestScriptLibTester( 'tests/data/opensearch-1.3.0-bundle.yml', + '', + '', 'true', 'nyc_taxis', 'true', @@ -82,10 +84,10 @@ class TestRunBenchmarkTestScript extends BuildPipelineTest { assertThat(testScriptCommands.size(), equalTo(2)) assertThat(testScriptCommands, hasItem( - "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,security-enabled:true --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307-secure --data-instance-type r5.8xlarge --workload-params '{\"key2\":\"value2\"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ".toString() + "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,security-enabled:true --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307-secure --data-instance-type r5.8xlarge --workload-params '{\"key2\":\"value2\"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ".toString() )) assertThat(testScriptCommands, hasItem( - "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,security-enabled:false --without-security --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307 --data-instance-type r5.8xlarge --workload-params '{\"key2\":\"value2\"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ".toString() + "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,security-enabled:false --without-security --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307 --data-instance-type r5.8xlarge --workload-params '{\"key2\":\"value2\"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ".toString() )) } diff --git a/tests/jenkins/TestRunBenchmarkTestScriptMultiNode.groovy b/tests/jenkins/TestRunBenchmarkTestScriptMultiNode.groovy index 864fa87be..d657a9dd4 100644 --- a/tests/jenkins/TestRunBenchmarkTestScriptMultiNode.groovy +++ b/tests/jenkins/TestRunBenchmarkTestScriptMultiNode.groovy @@ -22,6 +22,8 @@ class TestRunBenchmarkTestScriptMultiNode extends BuildPipelineTest { void setUp() { this.registerLibTester(new RunBenchmarkTestScriptLibTester( 'tests/data/opensearch-1.3.0-bundle.yml', + '', + '', 'true', 'nyc_taxis', 'false', @@ -48,7 +50,7 @@ class TestRunBenchmarkTestScriptMultiNode extends BuildPipelineTest { @Test - void testRunPerfTestScript_verifyScriptExecutionsMultiNode() { + void testRunBenchmarkTestScript_verifyScriptExecutionsMultiNode() { runScript("tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile") def testScriptCommands = getCommandExecutions('sh', './test.sh').findAll { @@ -57,10 +59,10 @@ class TestRunBenchmarkTestScriptMultiNode extends BuildPipelineTest { assertThat(testScriptCommands.size(), equalTo(2)) assertThat(testScriptCommands, hasItem( - "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:true --use-50-percent-heap --enable-remote-store --capture-node-stat --suffix 307-secure --manager-node-count 3 --data-node-count 3 --workload-params '{\"key2\":\"value2\"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString() + "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:true --use-50-percent-heap --enable-remote-store --capture-node-stat --suffix 307-secure --manager-node-count 3 --data-node-count 3 --workload-params '{\"key2\":\"value2\"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString() )) assertThat(testScriptCommands, hasItem( - "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:false --without-security --use-50-percent-heap --enable-remote-store --capture-node-stat --suffix 307 --manager-node-count 3 --data-node-count 3 --workload-params '{\"key2\":\"value2\"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString() + "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:false --without-security --use-50-percent-heap --enable-remote-store --capture-node-stat --suffix 307 --manager-node-count 3 --data-node-count 3 --workload-params '{\"key2\":\"value2\"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString() )) } diff --git a/tests/jenkins/TestRunBenchmarkTestScriptNoTags.groovy b/tests/jenkins/TestRunBenchmarkTestScriptNoTags.groovy new file mode 100644 index 000000000..9d695f3cf --- /dev/null +++ b/tests/jenkins/TestRunBenchmarkTestScriptNoTags.groovy @@ -0,0 +1,83 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import jenkins.tests.BuildPipelineTest +import org.junit.Before +import org.junit.Test + +import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString +import static org.hamcrest.CoreMatchers.equalTo +import static org.hamcrest.CoreMatchers.hasItem +import static org.hamcrest.MatcherAssert.assertThat + +class TestRunBenchmarkTestScriptNoManifest extends BuildPipelineTest { + + @Before + void setUp() { + this.registerLibTester(new RunBenchmarkTestScriptLibTester( + '', + 'https://www.exmaple.com/example.tar.gz', + '3.0.0', + 'true', + 'nyc_taxis', + 'true', + 'false', + 'true', + 'false', + '', + '', + 'r5.8xlarge', + '', + '', + 'cluster.indices.replication.strategy:SEGMENT', + 'false', + 'true', + '' + )) + super.setUp() + } + + @Test + public void testRunBenchmarkTestScript_PipelineSingleNode() { + super.testPipeline("tests/jenkins/jobs/BenchmarkTestNoManifest_Jenkinsfile") + } + + @Test + void testRunBenchmarkTestScript_verifyScriptExecutionsNoManifest() { + runScript("tests/jenkins/jobs/BenchmarkTestNoManifest_Jenkinsfile") + + def testScriptCommands = getCommandExecutions('sh', './test.sh').findAll { + shCommand -> shCommand.contains('./test.sh') + } + + assertThat(testScriptCommands.size(), equalTo(2)) + assertThat(testScriptCommands, hasItem( + "./test.sh benchmark-test --distribution-url https://www.exmaple.com/example.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag security-enabled:true --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307-secure --data-instance-type r5.8xlarge --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ".toString() + )) + assertThat(testScriptCommands, hasItem( + "./test.sh benchmark-test --distribution-url https://www.exmaple.com/example.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag security-enabled:false --without-security --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307 --data-instance-type r5.8xlarge --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ".toString() + )) + } + + def getCommandExecutions(methodName, command) { + def shCommands = helper.callStack.findAll { + call -> + call.methodName == methodName + }. + collect { + call -> + callArgsToString(call) + }.findAll { + shCommand -> + shCommand.contains(command) + } + + return shCommands + } +} diff --git a/tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile b/tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile index fffaaefe1..240234b71 100644 --- a/tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile +++ b/tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile @@ -19,6 +19,8 @@ pipeline { path: BUNDLE_MANIFEST ) runBenchmarkTestScript(bundleManifest: BUNDLE_MANIFEST, + distributionUrl: DISTRIBUTION_URL, + distributionVersion: DISTRIBUTION_VERSION, workload: TEST_WORKLOAD, insecure: "false", singleNode: SINGLE_NODE_CLUSTER, @@ -53,6 +55,8 @@ pipeline { path: BUNDLE_MANIFEST ) runBenchmarkTestScript(bundleManifest: BUNDLE_MANIFEST, + distributionUrl: DISTRIBUTION_URL, + distributionVersion: DISTRIBUTION_VERSION, workload: TEST_WORKLOAD, insecure: "true", singleNode: SINGLE_NODE_CLUSTER, diff --git a/tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile.txt b/tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile.txt index 559b53447..78758afa8 100644 --- a/tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile.txt +++ b/tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile.txt @@ -9,7 +9,7 @@ downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml) downloadBuildManifest.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml}) BuildManifest.asBoolean() - BenchmarkTestMultinode_Jenkinsfile.runBenchmarkTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, workload=nyc_taxis, insecure=false, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=true, suffix=307-secure, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=, userTag=key1:value1,security-enabled:true, workloadParams={"key2":"value2"}, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=true, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}}) + BenchmarkTestMultinode_Jenkinsfile.runBenchmarkTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc_taxis, insecure=false, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=true, suffix=307-secure, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=, userTag=key1:value1,security-enabled:true, workloadParams={"key2":"value2"}, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=true, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}}) runBenchmarkTestScript.legacySCM(groovy.lang.Closure) runBenchmarkTestScript.library({identifier=jenkins@5.9.2, retriever=null}) runBenchmarkTestScript.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml}) @@ -28,7 +28,7 @@ BuildManifest.getArtifactBuildId() BuildManifest.getArtifactArchitecture() BuildManifest.getCommitId(OpenSearch) - runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:true --use-50-percent-heap --enable-remote-store --capture-node-stat --suffix 307-secure --manager-node-count 3 --data-node-count 3 --workload-params '{"key2":"value2"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 --telemetry-params '{"telemetry_setting":"value"}') + runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:true --use-50-percent-heap --enable-remote-store --capture-node-stat --suffix 307-secure --manager-node-count 3 --data-node-count 3 --workload-params '{"key2":"value2"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 --telemetry-params '{"telemetry_setting":"value"}') BenchmarkTestMultinode_Jenkinsfile.stage(test-without-security, groovy.lang.Closure) BenchmarkTestMultinode_Jenkinsfile.script(groovy.lang.Closure) BenchmarkTestMultinode_Jenkinsfile.downloadBuildManifest({url=test://artifact.url, path=tests/data/opensearch-1.3.0-bundle.yml}) @@ -37,7 +37,7 @@ downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml) downloadBuildManifest.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml}) BuildManifest.asBoolean() - BenchmarkTestMultinode_Jenkinsfile.runBenchmarkTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, workload=nyc_taxis, insecure=true, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=true, suffix=307, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=, userTag=key1:value1,security-enabled:false, workloadParams={"key2":"value2"}, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=true, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}}) + BenchmarkTestMultinode_Jenkinsfile.runBenchmarkTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc_taxis, insecure=true, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=true, suffix=307, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=, userTag=key1:value1,security-enabled:false, workloadParams={"key2":"value2"}, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=true, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}}) runBenchmarkTestScript.legacySCM(groovy.lang.Closure) runBenchmarkTestScript.library({identifier=jenkins@5.9.2, retriever=null}) runBenchmarkTestScript.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml}) @@ -56,4 +56,4 @@ BuildManifest.getArtifactBuildId() BuildManifest.getArtifactArchitecture() BuildManifest.getCommitId(OpenSearch) - runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:false --without-security --use-50-percent-heap --enable-remote-store --capture-node-stat --suffix 307 --manager-node-count 3 --data-node-count 3 --workload-params '{"key2":"value2"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 --telemetry-params '{"telemetry_setting":"value"}') + runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:false --without-security --use-50-percent-heap --enable-remote-store --capture-node-stat --suffix 307 --manager-node-count 3 --data-node-count 3 --workload-params '{"key2":"value2"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 --telemetry-params '{"telemetry_setting":"value"}') diff --git a/tests/jenkins/jobs/BenchmarkTestNoManifest_Jenkinsfile b/tests/jenkins/jobs/BenchmarkTestNoManifest_Jenkinsfile new file mode 100644 index 000000000..80ff23dd3 --- /dev/null +++ b/tests/jenkins/jobs/BenchmarkTestNoManifest_Jenkinsfile @@ -0,0 +1,81 @@ + /* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +pipeline { + agent none + stages { + stage('benchmark-test') { + parallel { + stage('test-with-security') { + steps { + script { + runBenchmarkTestScript(bundleManifest: BUNDLE_MANIFEST, + distributionUrl: DISTRIBUTION_URL, + distributionVersion: DISTRIBUTION_VERSION, + workload: TEST_WORKLOAD, + insecure: "false", + singleNode: SINGLE_NODE_CLUSTER, + minDistribution: MIN_DISTRIBUTION, + use50PercentHeap: USE_50_PERCENT_HEAP, + enableRemoteStore: ENABLE_REMOTE_STORE, + suffix: "${env.BUILD_NUMBER}-secure", + managerNodeCount: MANAGER_NODE_COUNT, + dataNodeCount: DATA_NODE_COUNT, + clientNodeCount: CLIENT_NODE_COUNT, + ingestNodeCount: INGEST_NODE_COUNT, + mlNodeCount: ML_NODE_COUNT, + dataInstanceType: DATA_INSTANCE_TYPE, + userTag: USER_TAGS.isEmpty() ? "security-enabled:true" : "${USER_TAGS},security-enabled:true", + workloadParams: WORKLOAD_PARAMS, + additionalConfig: ADDITIONAL_CONFIG, + dataStorageSize: DATA_NODE_STORAGE, + mlStorageSize: ML_NODE_STORAGE, + jvmSysProps: JVM_SYS_PROPS, + captureNodeStat: CAPTURE_NODE_STAT, + captureSegmentReplicationStat: CAPTURE_SEGMENT_REPLICATION_STAT, + telemetryParams: TELEMETRY_PARAMS + ) + } + } + } + stage('test-without-security') { + steps { + script { + runBenchmarkTestScript(bundleManifest: BUNDLE_MANIFEST, + distributionUrl: DISTRIBUTION_URL, + distributionVersion: DISTRIBUTION_VERSION, + workload: TEST_WORKLOAD, + insecure: "true", + singleNode: SINGLE_NODE_CLUSTER, + minDistribution: MIN_DISTRIBUTION, + use50PercentHeap: USE_50_PERCENT_HEAP, + enableRemoteStore: ENABLE_REMOTE_STORE, + suffix: "${env.BUILD_NUMBER}", + managerNodeCount: MANAGER_NODE_COUNT, + dataNodeCount: DATA_NODE_COUNT, + clientNodeCount: CLIENT_NODE_COUNT, + ingestNodeCount: INGEST_NODE_COUNT, + mlNodeCount: ML_NODE_COUNT, + dataInstanceType: DATA_INSTANCE_TYPE, + userTag: USER_TAGS.isEmpty() ? "security-enabled:false" : "${USER_TAGS},security-enabled:false", + workloadParams: WORKLOAD_PARAMS, + additionalConfig: ADDITIONAL_CONFIG, + dataStorageSize: DATA_NODE_STORAGE, + mlStorageSize: ML_NODE_STORAGE, + jvmSysProps: JVM_SYS_PROPS, + captureNodeStat: CAPTURE_NODE_STAT, + captureSegmentReplicationStat: CAPTURE_SEGMENT_REPLICATION_STAT, + telemetryParams: TELEMETRY_PARAMS + ) + } + } + } + } + } + } +} diff --git a/tests/jenkins/jobs/BenchmarkTestNoManifest_Jenkinsfile.txt b/tests/jenkins/jobs/BenchmarkTestNoManifest_Jenkinsfile.txt new file mode 100644 index 000000000..5cf78b307 --- /dev/null +++ b/tests/jenkins/jobs/BenchmarkTestNoManifest_Jenkinsfile.txt @@ -0,0 +1,37 @@ + BenchmarkTestNoManifest_Jenkinsfile.run() + BenchmarkTestNoManifest_Jenkinsfile.pipeline(groovy.lang.Closure) + BenchmarkTestNoManifest_Jenkinsfile.echo(Executing on agent [label:none]) + BenchmarkTestNoManifest_Jenkinsfile.stage(test-with-security, groovy.lang.Closure) + BenchmarkTestNoManifest_Jenkinsfile.script(groovy.lang.Closure) + BenchmarkTestNoManifest_Jenkinsfile.runBenchmarkTestScript({bundleManifest=, distributionUrl=https://www.exmaple.com/example.tar.gz, distributionVersion=3.0.0, workload=nyc_taxis, insecure=false, singleNode=true, minDistribution=false, use50PercentHeap=true, enableRemoteStore=false, suffix=307-secure, managerNodeCount=, dataNodeCount=, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5.8xlarge, userTag=security-enabled:true, workloadParams=, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=true, telemetryParams=}) + runBenchmarkTestScript.legacySCM(groovy.lang.Closure) + runBenchmarkTestScript.library({identifier=jenkins@main, retriever=null}) + runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + runBenchmarkTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + runBenchmarkTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure) + runBenchmarkTestScript.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + runBenchmarkTestScript.s3Download({file=config.yml, bucket=ARTIFACT_BUCKET_NAME, path=test_config/config.yml, force=true}) + runBenchmarkTestScript.s3Download({file=benchmark.ini, bucket=ARTIFACT_BUCKET_NAME, path=test_config/benchmark.ini, force=true}) + runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-user, variable=DATASTORE_USER}) + runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-password, variable=DATASTORE_PASSWORD}) + runBenchmarkTestScript.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD], groovy.lang.Closure) + runBenchmarkTestScript.readFile({file=/tmp/workspace/benchmark.ini}) + runBenchmarkTestScript.writeFile({file=/tmp/workspace/benchmark.ini, text=}) + runBenchmarkTestScript.sh(./test.sh benchmark-test --distribution-url https://www.exmaple.com/example.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag security-enabled:true --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307-secure --data-instance-type r5.8xlarge --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ) + BenchmarkTestNoManifest_Jenkinsfile.stage(test-without-security, groovy.lang.Closure) + BenchmarkTestNoManifest_Jenkinsfile.script(groovy.lang.Closure) + BenchmarkTestNoManifest_Jenkinsfile.runBenchmarkTestScript({bundleManifest=, distributionUrl=https://www.exmaple.com/example.tar.gz, distributionVersion=3.0.0, workload=nyc_taxis, insecure=true, singleNode=true, minDistribution=false, use50PercentHeap=true, enableRemoteStore=false, suffix=307, managerNodeCount=, dataNodeCount=, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5.8xlarge, userTag=security-enabled:false, workloadParams=, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=true, telemetryParams=}) + runBenchmarkTestScript.legacySCM(groovy.lang.Closure) + runBenchmarkTestScript.library({identifier=jenkins@main, retriever=null}) + runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + runBenchmarkTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + runBenchmarkTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure) + runBenchmarkTestScript.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + runBenchmarkTestScript.s3Download({file=config.yml, bucket=ARTIFACT_BUCKET_NAME, path=test_config/config.yml, force=true}) + runBenchmarkTestScript.s3Download({file=benchmark.ini, bucket=ARTIFACT_BUCKET_NAME, path=test_config/benchmark.ini, force=true}) + runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-user, variable=DATASTORE_USER}) + runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-password, variable=DATASTORE_PASSWORD}) + runBenchmarkTestScript.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD], groovy.lang.Closure) + runBenchmarkTestScript.readFile({file=/tmp/workspace/benchmark.ini}) + runBenchmarkTestScript.writeFile({file=/tmp/workspace/benchmark.ini, text=}) + runBenchmarkTestScript.sh(./test.sh benchmark-test --distribution-url https://www.exmaple.com/example.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag security-enabled:false --without-security --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307 --data-instance-type r5.8xlarge --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ) diff --git a/tests/jenkins/jobs/BenchmarkTest_Jenkinsfile b/tests/jenkins/jobs/BenchmarkTest_Jenkinsfile index fffaaefe1..240234b71 100644 --- a/tests/jenkins/jobs/BenchmarkTest_Jenkinsfile +++ b/tests/jenkins/jobs/BenchmarkTest_Jenkinsfile @@ -19,6 +19,8 @@ pipeline { path: BUNDLE_MANIFEST ) runBenchmarkTestScript(bundleManifest: BUNDLE_MANIFEST, + distributionUrl: DISTRIBUTION_URL, + distributionVersion: DISTRIBUTION_VERSION, workload: TEST_WORKLOAD, insecure: "false", singleNode: SINGLE_NODE_CLUSTER, @@ -53,6 +55,8 @@ pipeline { path: BUNDLE_MANIFEST ) runBenchmarkTestScript(bundleManifest: BUNDLE_MANIFEST, + distributionUrl: DISTRIBUTION_URL, + distributionVersion: DISTRIBUTION_VERSION, workload: TEST_WORKLOAD, insecure: "true", singleNode: SINGLE_NODE_CLUSTER, diff --git a/tests/jenkins/jobs/BenchmarkTest_Jenkinsfile.txt b/tests/jenkins/jobs/BenchmarkTest_Jenkinsfile.txt index 5f38243cc..3e3d9db62 100644 --- a/tests/jenkins/jobs/BenchmarkTest_Jenkinsfile.txt +++ b/tests/jenkins/jobs/BenchmarkTest_Jenkinsfile.txt @@ -9,7 +9,7 @@ downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml) downloadBuildManifest.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml}) BuildManifest.asBoolean() - BenchmarkTest_Jenkinsfile.runBenchmarkTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, workload=nyc_taxis, insecure=false, singleNode=true, minDistribution=false, use50PercentHeap=true, enableRemoteStore=false, suffix=307-secure, managerNodeCount=, dataNodeCount=, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5.8xlarge, userTag=security-enabled:true, workloadParams={"key2":"value2"}, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=true, telemetryParams=}) + BenchmarkTest_Jenkinsfile.runBenchmarkTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc_taxis, insecure=false, singleNode=true, minDistribution=false, use50PercentHeap=true, enableRemoteStore=false, suffix=307-secure, managerNodeCount=, dataNodeCount=, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5.8xlarge, userTag=security-enabled:true, workloadParams={"key2":"value2"}, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=true, telemetryParams=}) runBenchmarkTestScript.legacySCM(groovy.lang.Closure) runBenchmarkTestScript.library({identifier=jenkins@5.9.2, retriever=null}) runBenchmarkTestScript.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml}) @@ -28,7 +28,7 @@ BuildManifest.getArtifactBuildId() BuildManifest.getArtifactArchitecture() BuildManifest.getCommitId(OpenSearch) - runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,security-enabled:true --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307-secure --data-instance-type r5.8xlarge --workload-params '{"key2":"value2"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ) + runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,security-enabled:true --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307-secure --data-instance-type r5.8xlarge --workload-params '{"key2":"value2"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ) BenchmarkTest_Jenkinsfile.stage(test-without-security, groovy.lang.Closure) BenchmarkTest_Jenkinsfile.script(groovy.lang.Closure) BenchmarkTest_Jenkinsfile.downloadBuildManifest({url=test://artifact.url, path=tests/data/opensearch-1.3.0-bundle.yml}) @@ -37,7 +37,7 @@ downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml) downloadBuildManifest.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml}) BuildManifest.asBoolean() - BenchmarkTest_Jenkinsfile.runBenchmarkTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, workload=nyc_taxis, insecure=true, singleNode=true, minDistribution=false, use50PercentHeap=true, enableRemoteStore=false, suffix=307, managerNodeCount=, dataNodeCount=, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5.8xlarge, userTag=security-enabled:false, workloadParams={"key2":"value2"}, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=true, telemetryParams=}) + BenchmarkTest_Jenkinsfile.runBenchmarkTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc_taxis, insecure=true, singleNode=true, minDistribution=false, use50PercentHeap=true, enableRemoteStore=false, suffix=307, managerNodeCount=, dataNodeCount=, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5.8xlarge, userTag=security-enabled:false, workloadParams={"key2":"value2"}, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=true, telemetryParams=}) runBenchmarkTestScript.legacySCM(groovy.lang.Closure) runBenchmarkTestScript.library({identifier=jenkins@5.9.2, retriever=null}) runBenchmarkTestScript.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml}) @@ -56,4 +56,4 @@ BuildManifest.getArtifactBuildId() BuildManifest.getArtifactArchitecture() BuildManifest.getCommitId(OpenSearch) - runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,security-enabled:false --without-security --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307 --data-instance-type r5.8xlarge --workload-params '{"key2":"value2"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ) + runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,security-enabled:false --without-security --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307 --data-instance-type r5.8xlarge --workload-params '{"key2":"value2"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ) diff --git a/tests/jenkins/lib-testers/RunBenchmarkTestScriptLibTest.groovy b/tests/jenkins/lib-testers/RunBenchmarkTestScriptLibTest.groovy index 8a28ae575..742a3823a 100644 --- a/tests/jenkins/lib-testers/RunBenchmarkTestScriptLibTest.groovy +++ b/tests/jenkins/lib-testers/RunBenchmarkTestScriptLibTest.groovy @@ -12,6 +12,8 @@ import static org.hamcrest.MatcherAssert.assertThat class RunBenchmarkTestScriptLibTester extends LibFunctionTester{ private String bundleManifest + private String distributionUrl + private String distributionVersion private String insecure private String workload private String singleNode @@ -35,10 +37,12 @@ class RunBenchmarkTestScriptLibTester extends LibFunctionTester{ private String captureSegmentReplicationStat private String telemetryParams - public RunBenchmarkTestScriptLibTester(bundleManifest, insecure, workload, singleNode, minDistribution, use50PercentHeap, + public RunBenchmarkTestScriptLibTester(bundleManifest, distributionUrl, distributionVersion, insecure, workload, singleNode, minDistribution, use50PercentHeap, enableRemoteStore, managerNodeCount, dataNodeCount, dataInstanceType, userTag, workloadParams, additionalConfig, captureNodeStat, captureSegmentReplicationStat, telemetryParams){ this.bundleManifest = bundleManifest + this.distributionUrl = distributionUrl + this.distributionVersion = distributionVersion this.insecure = insecure this.workload = workload this.singleNode = singleNode @@ -64,18 +68,27 @@ class RunBenchmarkTestScriptLibTester extends LibFunctionTester{ @Override void parameterInvariantsAssertions(Object call) { - assertThat(call.args.bundleManifest.first(), notNullValue()) + if (!this.bundleManifest.isEmpty()) { + assertThat(call.args.bundleManifest.first(), notNullValue()) + } if (!this.insecure.isEmpty()) { assertThat(call.args.insecure.first(), notNullValue()) } if (!this.workload.isEmpty()) { assertThat(call.args.workload.first(), notNullValue()) } + if (!this.distributionUrl.isEmpty()) { + assertThat(call.args.distributionUrl.first(), notNullValue()) + assertThat(call.args.distributionVersion.first(), notNullValue()) + } } @Override boolean expectedParametersMatcher(Object call) { - return call.args.bundleManifest.first().toString().equals(this.bundleManifest) + if (!this.bundleManifest.isEmpty()) { + return call.args.bundleManifest.first().toString().equals(this.bundleManifest) + } + return call.args.distributionUrl.first().toString().equals(this.distributionUrl) } @Override @@ -105,6 +118,8 @@ class RunBenchmarkTestScriptLibTester extends LibFunctionTester{ binding.setVariable('BUILD_NUMBER', '307') binding.setVariable('BUILD_URL', 'test://artifact.url') binding.setVariable('BUNDLE_MANIFEST', bundleManifest) + binding.setVariable('DISTRIBUTION_URL', distributionUrl) + binding.setVariable('DISTRIBUTION_VERSION', distributionVersion) binding.setVariable('BUNDLE_MANIFEST_URL', 'test://artifact.url') binding.setVariable('GITHUB_BOT_TOKEN_NAME', 'bot_token_name') binding.setVariable('GITHUB_USER', 'test_user') diff --git a/vars/runBenchmarkTestScript.groovy b/vars/runBenchmarkTestScript.groovy index 83c819b15..0c5e7ca81 100644 --- a/vars/runBenchmarkTestScript.groovy +++ b/vars/runBenchmarkTestScript.groovy @@ -9,7 +9,9 @@ /** Library to execute benchmark-test using opensearch-benchmark and opensearch-cluster-cdk * * @param Map args = [:] args A map of the following parameters - * @param args.bundleManifest - OpenSearch bundle manifest url. + * @param args.bundleManifest - OpenSearch bundle manifest url. + * @param args.distributionUrl - Download link for the OpenSearch bundle tarball. + * @param args.distributionVersion - Provide OpenSearch version if using distributionUrl param * @param args.insecure - Force the security of the cluster to be disabled, default is false. * @param args.workload - Name of the workload that OpenSearch Benchmark should run, default is nyc_taxis. * @param args.singleNode - Create single node OS cluster, default is true. @@ -34,8 +36,12 @@ * @param args.telemetryParams - Allows to set parameters for telemetry devices such as node-stat etc., e.g. {"node-stats-include-indices": "true"} */ void call(Map args = [:]) { - lib = library(identifier: 'jenkins@5.9.2', retriever: legacySCM(scm)) - def buildManifest = lib.jenkins.BuildManifest.new(readYaml(file: args.bundleManifest)) + lib = library(identifier: 'jenkins@main', retriever: legacySCM(scm)) + def buildManifest = null + + if (!isNullOrEmpty(args.bundleManifest as String)){ + buildManifest = lib.jenkins.BuildManifest.new(readYaml(file: args.bundleManifest)) + } config_name = isNullOrEmpty(args.config) ? 'config.yml' : args.config benchmark_config = 'benchmark.ini' @@ -46,9 +52,14 @@ void call(Map args = [:]) { s3Download(file: 'benchmark.ini', bucket: "${ARTIFACT_BUCKET_NAME}", path: "${BENCHMARK_TEST_CONFIG_LOCATION}/${benchmark_config}", force: true) /*Added sleep to let the file get downloaded first before write happens. Without the sleep the write is - happening in parallel to download resulting in file not found error + happening in parallel to download resulting in file not found error. To avoid pip install conflict errors + when runnin with and without security run in parallel add enough gap between execution. */ - sleep(5) + if (args.insecure.toBoolean()) { + sleep(5) + } else { + sleep(15) + } } } editBenchmarkConfig("${WORKSPACE}/benchmark.ini") @@ -57,7 +68,9 @@ void call(Map args = [:]) { sh([ './test.sh', 'benchmark-test', - "--bundle-manifest ${args.bundleManifest}", + isNullOrEmpty(args.bundleManifest.toString()) ? "" : "--bundle-manifest ${args.bundleManifest}", + isNullOrEmpty(args.distributionUrl.toString()) ? "" : "--distribution-url ${args.distributionUrl}", + isNullOrEmpty(args.distributionVersion.toString()) ? "" : "--distribution-version ${args.distributionVersion}", "--config ${WORKSPACE}/config.yml", "--workload ${args.workload}", "--benchmark-config ${WORKSPACE}/benchmark.ini", @@ -98,11 +111,17 @@ void editBenchmarkConfig(String config_file) { } String getMetadataTags(tags, buildManifest) { - def metadataTags = "distribution-build-id:${buildManifest.getArtifactBuildId()},arch:${buildManifest.getArtifactArchitecture()}," + - "os-commit-id:${buildManifest.getCommitId("OpenSearch")}" - if (!isNullOrEmpty(tags)){ + def metadataTags = null + if (buildManifest != null) { + metadataTags = "distribution-build-id:${buildManifest.getArtifactBuildId()},arch:${buildManifest.getArtifactArchitecture()}," + + "os-commit-id:${buildManifest.getCommitId("OpenSearch")}" + } + + if (!isNullOrEmpty(tags) && buildManifest != null){ metadataTags = metadataTags + ',' + tags return metadataTags + } else if (!isNullOrEmpty(tags)) { + return tags } return metadataTags }