Add backward compatibility regression test suite for multi-stage quer…

…y engine (apache#13193)
soumitra-st · Jun 3, 2024 · 68685dc · 68685dc
1 parent a4950fe
commit 68685dc
Show file tree

Hide file tree

Showing 27 changed files with 1,017 additions and 6 deletions.
diff --git a/.github/workflows/pinot_multi_stage_query_engine_compatibility_tests.yml b/.github/workflows/pinot_multi_stage_query_engine_compatibility_tests.yml
@@ -0,0 +1,84 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+name: Pinot Multi-Stage Query Engine Compatibility Regression Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      oldCommit:
+        description: "Git hash (or tag) for old commit. (required)"
+        required: true
+      newCommit:
+        description: "Git hash (or tag) for new commit. (required)"
+        required: true
+
+jobs:
+  compatibility-verifier:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        test_suite: [ "compatibility-verifier/multi-stage-query-engine-test-suite" ]
+    name: Pinot Multi-Stage Query Engine Compatibility Regression Testing against ${{ github.event.inputs.oldCommit }} and ${{ github.event.inputs.newCommit }} on ${{ matrix.test_suite }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK 11
+        uses: actions/setup-java@v4
+        with:
+          java-version: 11
+          distribution: 'temurin'
+          cache: 'maven'
+      - name: Setup node
+        uses: actions/setup-node@v4
+        with:
+          node-version: v16.15.0
+          cache: 'npm'
+          cache-dependency-path: pinot-controller/src/main/resources/package-lock.json
+      - name: Install npm
+        run: |
+          npm install -g [email protected]
+          npm --version
+      - name: Pinot Multi-Stage Query Engine Compatibility Regression Testing
+        if : ${{github.event_name == 'workflow_dispatch'}}
+        env:
+          OLD_COMMIT: ${{ github.event.inputs.oldCommit }}
+          NEW_COMMIT: ${{ github.event.inputs.newCommit }}
+          WORKING_DIR: /tmp/multi-stage-compatibility-verifier
+          TEST_SUITE: ${{ matrix.test_suite }}
+          MAVEN_OPTS: >
+            -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
+            -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
+            -XX:+IgnoreUnrecognizedVMOptions
+            --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
+            --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
+            --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
+            --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
+            --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+        run: .github/workflows/scripts/.pinot_compatibility_verifier.sh
+      - name: Archive artifacts into zip
+        if: always()
+        run: |
+          zip -1 -r artifacts.zip /tmp/multi-stage-compatibility-verifier/*
+      - uses: actions/upload-artifact@v4
+        name: Store multi-stage compatibility verifier work directory
+        if: always()
+        with:
+          ## TODO: currently matrix.test_suite cannot be used as part of name due to invalid path character.
+          name: multi_stage_compatibility_verifier_work_dir
+          retention-days: 3
+          path: artifacts.zip
diff --git a/.github/workflows/pinot_tests.yml b/.github/workflows/pinot_tests.yml
@@ -314,6 +314,62 @@ jobs:
             --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
         run: .github/workflows/scripts/.pinot_compatibility_verifier.sh
 
+  multi-stage-compatibility-verifier:
+    if: github.repository == 'apache/pinot'
+    runs-on: ubuntu-latest
+    strategy:
+      # Changed to false in order to improve coverage using unsafe buffers
+      fail-fast: false
+      matrix:
+        test_suite: [ "compatibility-verifier/multi-stage-query-engine-test-suite" ]
+        old_commit: [
+          "master"
+        ]
+    name: Pinot Multi-Stage Query Engine Compatibility Regression Testing against ${{ matrix.old_commit }} on ${{ matrix.test_suite }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK 11
+        uses: actions/setup-java@v4
+        with:
+          java-version: 11
+          distribution: 'temurin'
+          cache: 'maven'
+      - name: Setup node
+        uses: actions/setup-node@v4
+        with:
+          node-version: v16.15.0
+          cache: 'npm'
+          cache-dependency-path: pinot-controller/src/main/resources/package-lock.json
+      - name: Install npm
+        run: |
+          npm install -g [email protected]
+          npm --version
+      # Step that does that actual cache save and restore
+      - uses: actions/cache@v4
+        env:
+          SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+      - name: Pinot Multi-Stage Query Engine Compatibility Regression Testing
+        env:
+          OLD_COMMIT: ${{ matrix.old_commit }}
+          WORKING_DIR: /tmp/multi-stage-compatibility-verifier
+          TEST_SUITE: ${{ matrix.test_suite }}
+          MAVEN_OPTS: >
+            -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
+            -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
+            -B -ntp
+            -XX:+IgnoreUnrecognizedVMOptions
+            --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
+            --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
+            --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
+            --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
+            --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+        run: .github/workflows/scripts/.pinot_compatibility_verifier.sh
+
   quickstarts:
     if: github.repository == 'apache/pinot'
     runs-on: ubuntu-latest

diff --git a/.gitignore b/.gitignore
@@ -2,6 +2,7 @@ cscope.*
 .classpath
 .project
 .svn
+.java-version
 .externalToolBuilders/
 maven-eclipse.xml
 target/

diff --git a/compatibility-verifier/multi-stage-query-engine-test-suite/config/BrokerConfig.properties b/compatibility-verifier/multi-stage-query-engine-test-suite/config/BrokerConfig.properties
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+pinot.broker.client.queryPort = 8099
+pinot.zk.server = localhost:2181
+pinot.cluster.name = PinotCluster
+pinot.broker.disable.query.groovy=false
diff --git a/...atibility-verifier/multi-stage-query-engine-test-suite/config/ControllerConfig.properties b/...atibility-verifier/multi-stage-query-engine-test-suite/config/ControllerConfig.properties
@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+controller.host = localhost
+controller.port = 9000
+controller.zk.str = localhost:2181
+controller.data.dir = /tmp/PinotController
+controller.helix.cluster.name = PinotCluster
+controller.disable.ingestion.groovy = false
diff --git a/compatibility-verifier/multi-stage-query-engine-test-suite/config/FeatureTest1-schema.json b/compatibility-verifier/multi-stage-query-engine-test-suite/config/FeatureTest1-schema.json
@@ -0,0 +1,94 @@
+{
+  "dimensionFieldSpecs": [
+    {
+      "dataType": "INT",
+      "name": "generationNumber"
+    },
+    {
+      "dataType": "STRING",
+      "name": "stringDimSV1"
+    },
+    {
+      "dataType": "STRING",
+      "name": "stringDimSV2"
+    },
+    {
+      "dataType": "LONG",
+      "name": "longDimSV1"
+    },
+    {
+      "dataType": "LONG",
+      "name": "longDimSV2"
+    },
+    {
+      "dataType": "STRING",
+      "name": "stringDimMV1",
+      "singleValueField": false
+    },
+    {
+      "dataType": "STRING",
+      "name": "stringDimMV2",
+      "singleValueField": false
+    },
+    {
+      "dataType": "INT",
+      "name": "intDimMV1",
+      "singleValueField": false
+    },
+    {
+      "dataType": "INT",
+      "name": "intDimMV2",
+      "singleValueField": false
+    },
+    {
+      "dataType": "STRING",
+      "maxLength": 1000,
+      "name": "textDim1"
+    },
+    {
+      "dataType": "BYTES",
+      "name": "bytesDimSV1"
+    },
+    {
+      "dataType": "STRING",
+      "name": "mapDim1__KEYS",
+      "singleValueField": false
+    },
+    {
+      "dataType": "INT",
+      "name": "mapDim1__VALUES",
+      "singleValueField": false
+    },
+    {
+      "dataType": "STRING",
+      "name": "mapDim2json"
+    }
+  ],
+  "metricFieldSpecs": [
+    {
+      "dataType": "INT",
+      "name": "intMetric1"
+    },
+    {
+      "dataType": "LONG",
+      "name": "longMetric1"
+    },
+    {
+      "dataType": "FLOAT",
+      "name": "floatMetric1"
+    },
+    {
+      "dataType": "DOUBLE",
+      "name": "doubleMetric1"
+    }
+  ],
+  "dateTimeFieldSpecs" : [
+    {
+      "name" : "HoursSinceEpoch",
+      "dataType" : "INT",
+      "format" : "1:HOURS:EPOCH",
+      "granularity": "1:HOURS"
+    }
+  ],
+  "schemaName": "FeatureTest1"
+}
diff --git a/compatibility-verifier/multi-stage-query-engine-test-suite/config/FeatureTest2-schema.json b/compatibility-verifier/multi-stage-query-engine-test-suite/config/FeatureTest2-schema.json
@@ -0,0 +1,94 @@
+{
+  "dimensionFieldSpecs": [
+    {
+      "dataType": "INT",
+      "name": "generationNumber"
+    },
+    {
+      "dataType": "STRING",
+      "name": "stringDimSV1"
+    },
+    {
+      "dataType": "STRING",
+      "name": "stringDimSV2"
+    },
+    {
+      "dataType": "LONG",
+      "name": "longDimSV1"
+    },
+    {
+      "dataType": "LONG",
+      "name": "longDimSV2"
+    },
+    {
+      "dataType": "STRING",
+      "name": "stringDimMV1",
+      "singleValueField": false
+    },
+    {
+      "dataType": "STRING",
+      "name": "stringDimMV2",
+      "singleValueField": false
+    },
+    {
+      "dataType": "INT",
+      "name": "intDimMV1",
+      "singleValueField": false
+    },
+    {
+      "dataType": "INT",
+      "name": "intDimMV2",
+      "singleValueField": false
+    },
+    {
+      "dataType": "STRING",
+      "maxLength": 1000,
+      "name": "textDim1"
+    },
+    {
+      "dataType": "BYTES",
+      "name": "bytesDimSV1"
+    },
+    {
+      "dataType": "STRING",
+      "name": "mapDim1__KEYS",
+      "singleValueField": false
+    },
+    {
+      "dataType": "INT",
+      "name": "mapDim1__VALUES",
+      "singleValueField": false
+    },
+    {
+      "dataType": "STRING",
+      "name": "mapDim2json"
+    }
+  ],
+  "metricFieldSpecs": [
+    {
+      "dataType": "INT",
+      "name": "intMetric1"
+    },
+    {
+      "dataType": "LONG",
+      "name": "longMetric1"
+    },
+    {
+      "dataType": "FLOAT",
+      "name": "floatMetric1"
+    },
+    {
+      "dataType": "DOUBLE",
+      "name": "doubleMetric1"
+    }
+  ],
+  "dateTimeFieldSpecs" : [
+    {
+      "name" : "HoursSinceEpoch",
+      "dataType" : "INT",
+      "format" : "1:HOURS:EPOCH",
+      "granularity": "1:HOURS"
+    }
+  ],
+  "schemaName": "FeatureTest2"
+}