Introduce BWC testing infrastructure

.github/workflows/BWC.yml

@@ -0,0 +1,32 @@
+name: Build and Run BWC Tests
+
+on:
+  push:
+    branches:
+      - "*"
+  pull_request:
+    branches:
+      - "*"
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        java: [21]
+    name: Build and Run BWC Tests
+    runs-on: ${{ matrix.os }}
+    permissions:
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v4
+        with:
+          distribution: "temurin"
+          java-version: ${{ matrix.java }}
+
+      - name: Build and Run Tests
+        run: |
+          ./gradlew --info bwcTestSuite -Dtests.security.manager=false

.gitignore

@@ -2,6 +2,9 @@
 # https://discuss.elastic.co/t/leftovers-after-integtestrunner-in-root/152610
 \.local*-integTestRunner-execution-times.log
 
+# This location is used by BWC test clusters to host older and actual versions of plugin ZIP files.
+src/test/resources/org/opensearch/prometheus-exporter/bwc/
+
 # intellij files
 .idea/
 *.iml

README.md

@@ -12,6 +12,7 @@ The [Prometheus® exporter](https://prometheus.io/docs/instrumenting/writing_exp
 - [Usage](#usage)
 - [Build from Source](#build-from-source)
 - [Testing](#testing)
+  - [BWC Testing](#bwc-testing)
 - [License](#license)
 - [Trademarks & Attributions](#trademarks--attributions)
 
@@ -279,6 +280,20 @@ To run individual integration rest test file use:
   -Dtests.method="test {yaml=/20_11_index_level_metrics_disabled/Dynamically disable index level metrics}"
 ```
 
+### BWC Testing
+
+Backward Compatibility (BWC) Testing is run manually using provided shell script:
+
+```
+./bwctest.sh
+```
+
+It is not part of `./gradlew [build|check]` task(s), but it is included in the CI workflow.
+
+OpenSearch versions used during BWC tests use determined by properties located in `gradle.properties` file. Specifically `project.version` and `project.BWCversion`. Version of plugin deployed into `project.BWCversion` cluster is specified by `project.BWCPluginVersion` property.
+
+In the beginning of BWC tests the actual version of plugin (`project.version`) is build using `bundlePlugin` gradle task and the `project.BWCPluginVersion` plugin is downloaded from GitHub releases. Both ZIP files are placed into `src/test/resource/org/opensearch/prometheus-exrpoter/bwc/prometheus-exporter` folder (this folder is ignored by git). 
+
 ## License
 
 Licensed under the Apache License, Version 2.0 (the "License");

build.gradle

@@ -1,5 +1,6 @@
 import org.opensearch.gradle.test.RestIntegTestTask
 
+import java.util.concurrent.Callable
 import java.util.regex.Matcher
 import java.util.regex.Pattern
 
@@ -36,6 +37,12 @@ buildscript {
                 "opensearch": opensearch_version,
                 "prometheus": "0.16.0"
         ]
+
+        bwcPluginDownloadLink = 'https://github.com/Aiven-Open/prometheus-exporter-plugin-for-opensearch/releases/download/' +
+                project.BWCPluginVersion + '/prometheus-exporter-' + project.BWCPluginVersion + '.zip'
+        baseName = "bwcCluster"
+        bwcFilePath = "src/test/resources/org/opensearch/prometheus-exporter/bwc/"
+        bwcPrometheusExporterPath = bwcFilePath + "prometheus-exporter/"
     }
 
     repositories {
@@ -46,7 +53,12 @@ buildscript {
         maven { url "https://plugins.gradle.org/m2/" }
     }
 
+    configurations {
+        zipArchive
+    }
+
     dependencies {
+        zipArchive group: 'org.opensearch.plugin.prometheus', name:'prometheus-exporter', version: "${versions.opensearch}"
         classpath   "org.opensearch.gradle:build-tools:${versions.opensearch}"
     }
 }
@@ -56,6 +68,7 @@ apply plugin: 'idea'
 apply plugin: 'opensearch.opensearchplugin'
 apply plugin: 'opensearch.yaml-rest-test'
 //apply plugin: 'checkstyle'
+apply plugin: 'opensearch.pluginzip'
 
 def pluginName = pluginName
 def pluginDescription = pluginDescription
@@ -118,6 +131,9 @@ task integTest(type: RestIntegTestTask) {
     description = "Run tests against a cluster"
     testClassesDirs = sourceSets.test.output.classesDirs
     classpath = sourceSets.test.runtimeClasspath
+    filter {
+        excludeTestsMatching "org.opensearch.plugin.bwc.*IT"
+    }
 }
 tasks.named("check").configure { dependsOn(integTest) }
 
@@ -146,6 +162,187 @@ testClusters.integTest {
     plugin(project.tasks.bundlePlugin.archiveFile)
 }
 
+task copyZIPBundle {
+    dependsOn(bundlePlugin)
+
+//    doFirst {
+//        if (new File("$bwcPrometheusExporterPath/$project.version").exists()) {
+//            project.delete(files("$bwcPrometheusExporterPath/$project.version"))
+//        }
+//        project.mkdir bwcPrometheusExporterPath + project.version
+//    }
+
+//    from 'build/distributions/prometheus-exporter-' + project.version + '.zip'
+//    into bwcPrometheusExporterPath + project.version
+    doLast {
+        // By using ant.copy we can "hack" around gradle check for other task's resources modification.
+        // It seems like a dirty hack but some official plugins seem to use this practice;
+        // for instance see https://github.com/opensearch-project/anomaly-detection/
+        ant.copy(todir: bwcPrometheusExporterPath + project.version) {
+            ant.fileset(dir: 'build/distributions/', includes: 'prometheus-exporter-' + project.version + '.zip')
+        }
+    }
+}
+
+// Clusters for BWC tests
+2.times { i ->
+    testClusters {
+        "${baseName}$i" {
+            versions = [project.BWCversion, "2.17.1"]
+            numberOfNodes = 3
+            plugin(provider(new Callable<RegularFile>() {
+                @Override
+                RegularFile call() throws Exception {
+                    return new RegularFile() {
+                        @Override
+                        File getAsFile() {
+                            if (new File("$project.rootDir/$bwcFilePath/prometheus-exporter/$project.BWCPluginVersion").exists()) {
+                                project.delete(files("$project.rootDir/$bwcFilePath/prometheus-exporter/$project.BWCPluginVersion"))
+                            }
+                            project.mkdir bwcPrometheusExporterPath + project.BWCPluginVersion
+                            ant.get(src: bwcPluginDownloadLink,
+                                    dest: bwcPrometheusExporterPath + project.BWCPluginVersion,
+                                    httpusecaches: false)
+                            return fileTree(bwcPrometheusExporterPath + project.BWCPluginVersion).getSingleFile()
+                        }
+                    }
+                }
+            }))
+            setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
+            setting 'http.content_type.required', 'true'
+        }
+    }
+}
+
+List<Provider<RegularFile>> plugins = [
+    provider(new Callable<RegularFile>(){
+        @Override
+        RegularFile call() throws Exception {
+            return new RegularFile() {
+                @Override
+                File getAsFile() {
+                    return fileTree(bwcPrometheusExporterPath + project.version).getSingleFile()
+                }
+            }
+        }
+    })
+]
+
+// Creates 2 test clusters with 3 nodes of the old version.
+2.times {i ->
+    task "${baseName}#oldVersionClusterTask$i"(type: RestIntegTestTask) {
+        useCluster testClusters."${baseName}$i"
+        filter {
+            includeTestsMatching "org.opensearch.plugin.bwc.*IT"
+        }
+        systemProperty 'tests.rest.bwcsuite', 'old_cluster'
+        systemProperty 'tests.rest.bwcsuite_round', 'old'
+        systemProperty 'tests.plugin_bwc_version', project.BWCPluginVersion
+        nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}$i".allHttpSocketURI.join(",")}")
+        nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}$i".getName()}")
+    }
+}
+
+// Upgrades one node of the old cluster to new OpenSearch version with upgraded plugin version
+// This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node.
+// This is also used as a one third upgraded cluster for a rolling upgrade.
+task "${baseName}#mixedClusterTask"(type: RestIntegTestTask) {
+    dependsOn tasks.named("copyZIPBundle")
+    useCluster testClusters."${baseName}0"
+    dependsOn "${baseName}#oldVersionClusterTask0"
+    doFirst {
+        testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
+    }
+    filter {
+        includeTestsMatching "org.opensearch.plugin.bwc.*IT"
+    }
+    systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
+    systemProperty 'tests.rest.bwcsuite_round', 'first'
+    systemProperty 'tests.plugin_bwc_version', project.BWCPluginVersion
+    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
+    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
+}
+
+// Upgrades the second node to new OpenSearch version with upgraded plugin version after the first node is upgraded.
+// This results in a mixed cluster with 1 node on the old version and 2 upgraded nodes.
+// This is used for rolling upgrade.
+task "${baseName}#twoThirdsUpgradedClusterTask"(type: RestIntegTestTask) {
+    dependsOn tasks.named("copyZIPBundle")
+    dependsOn "${baseName}#mixedClusterTask"
+    useCluster testClusters."${baseName}0"
+    doFirst {
+        testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
+    }
+    filter {
+        includeTestsMatching "org.opensearch.plugin.bwc.*IT"
+    }
+    systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
+    systemProperty 'tests.rest.bwcsuite_round', 'second'
+    systemProperty 'tests.plugin_bwc_version', project.BWCPluginVersion
+    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
+    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
+}
+
+// Upgrades the third node to new OpenSearch version with upgraded plugin version after the second node is upgraded.
+// This results in a fully upgraded cluster.
+// This is used for rolling upgrade.
+task "${baseName}#rollingUpgradeClusterTask"(type: RestIntegTestTask) {
+    dependsOn tasks.named("copyZIPBundle")
+    dependsOn "${baseName}#twoThirdsUpgradedClusterTask"
+    useCluster testClusters."${baseName}0"
+    doFirst {
+        testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
+    }
+    filter {
+        includeTestsMatching "org.opensearch.plugin.bwc.*IT"
+    }
+    mustRunAfter "${baseName}#mixedClusterTask"
+    systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
+    systemProperty 'tests.rest.bwcsuite_round', 'third'
+    systemProperty 'tests.plugin_bwc_version', project.BWCPluginVersion
+    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
+    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
+}
+
+// Upgrades all the nodes of the old cluster to new OpenSearch version with upgraded plugin version
+// at the same time resulting in a fully upgraded cluster.
+task "${baseName}#fullRestartClusterTask"(type: RestIntegTestTask) {
+    dependsOn tasks.named("copyZIPBundle")
+    dependsOn "${baseName}#oldVersionClusterTask1"
+    useCluster testClusters."${baseName}1"
+    doFirst {
+        testClusters."${baseName}1".upgradeAllNodesAndPluginsToNextVersion(plugins)
+    }
+    filter {
+        includeTestsMatching "org.opensearch.plugin.bwc.*IT"
+    }
+    systemProperty 'tests.rest.bwcsuite', 'upgraded_cluster'
+    systemProperty 'tests.plugin_bwc_version', project.BWCPluginVersion
+    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}1".allHttpSocketURI.join(",")}")
+    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}1".getName()}")
+}
+
+// A BWC test suite which runs all the bwc tasks combined.
+task bwcTestSuite(type: RestIntegTestTask) {
+
+    // Delete all downloaded and built plugin ZIP files.
+    // Again – we are using ant task to workaround gradle resources modification alert.
+    doFirst {
+        ant.delete(includeEmptyDirs: true, verbose: true, removeNotFollowedSymlinks: true) {
+            ant.fileset(
+                    dir: 'src/test/resources/org/opensearch/prometheus-exporter/bwc/prometheus-exporter'
+            )
+        }
+    }
+
+    exclude '**/*Test*'
+    exclude '**/*IT*'
+
+    dependsOn tasks.named("${baseName}#mixedClusterTask")
+    dependsOn tasks.named("${baseName}#rollingUpgradeClusterTask")
+    dependsOn tasks.named("${baseName}#fullRestartClusterTask")
+}
+
 run {
     useCluster testClusters.integTest
 }

bwctest.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -e
+
+function usage() {
+  echo ""
+    echo "This script is used to run Backwards Compatibility tests"
+    echo "--------------------------------------------------------------------------"
+    echo "Usage: $0 [args]"
+    echo ""
+    echo "Required arguments:"
+    echo "None"
+    echo ""
+    echo -e "-h\tPrint this message."
+    echo "--------------------------------------------------------------------------"
+}
+
+while getopts ":h" arg; do
+    case $arg in
+        h)
+            usage
+            exit 1
+            ;;
+        ?)
+            echo "Invalid option: -${OPTARG}"
+            exit 1
+            ;;
+    esac
+done
+
+# Warning!
+# This should be done from gradle, see bwcTestSuite task, but that task is skipped.
+# TODO: Skipping task ':bwcTestSuite' as it has no source files and no previous output files.
+rm -rf src/test/resources/org/opensearch/prometheus-exporter/bwc/prometheus-exporter
+
+./gradlew bwcTestSuite -Dtests.security.manager=false

gradle.properties

@@ -1,7 +1,14 @@
 #group = org.opensearch.plugin.prometheus
 
+# An actual version of plugin
 version = 2.17.1.0
 
+# A version of OpenSearch cluster to run BWC tests against
+BWCversion = 2.17.0
+
+# A version of plugin to deploy to BWC clusters
+BWCPluginVersion = 2.17.0.0
+
 pluginName = prometheus-exporter
 pluginClassname = org.opensearch.plugin.prometheus.PrometheusExporterPlugin
 pluginDescription = Prometheus exporter plugin for OpenSearch