diff --git a/.github/workflows/pinot_multi_stage_query_engine_compatibility_tests.yml b/.github/workflows/pinot_multi_stage_query_engine_compatibility_tests.yml new file mode 100644 index 000000000000..52b7773033de --- /dev/null +++ b/.github/workflows/pinot_multi_stage_query_engine_compatibility_tests.yml @@ -0,0 +1,84 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +name: Pinot Multi-Stage Query Engine Compatibility Regression Test + +on: + workflow_dispatch: + inputs: + oldCommit: + description: "Git hash (or tag) for old commit. (required)" + required: true + newCommit: + description: "Git hash (or tag) for new commit. (required)" + required: true + +jobs: + compatibility-verifier: + runs-on: ubuntu-latest + strategy: + matrix: + test_suite: [ "compatibility-verifier/multi-stage-query-engine-test-suite" ] + name: Pinot Multi-Stage Query Engine Compatibility Regression Testing against ${{ github.event.inputs.oldCommit }} and ${{ github.event.inputs.newCommit }} on ${{ matrix.test_suite }} + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: 11 + distribution: 'temurin' + cache: 'maven' + - name: Setup node + uses: actions/setup-node@v4 + with: + node-version: v16.15.0 + cache: 'npm' + cache-dependency-path: pinot-controller/src/main/resources/package-lock.json + - name: Install npm + run: | + npm install -g npm@8.5.5 + npm --version + - name: Pinot Multi-Stage Query Engine Compatibility Regression Testing + if : ${{github.event_name == 'workflow_dispatch'}} + env: + OLD_COMMIT: ${{ github.event.inputs.oldCommit }} + NEW_COMMIT: ${{ github.event.inputs.newCommit }} + WORKING_DIR: /tmp/multi-stage-compatibility-verifier + TEST_SUITE: ${{ matrix.test_suite }} + MAVEN_OPTS: > + -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 + -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false + -XX:+IgnoreUnrecognizedVMOptions + --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED + --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED + --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED + --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED + --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED + run: .github/workflows/scripts/.pinot_compatibility_verifier.sh + - name: Archive artifacts into zip + if: always() + run: | + zip -1 -r artifacts.zip /tmp/multi-stage-compatibility-verifier/* + - uses: actions/upload-artifact@v4 + name: Store multi-stage compatibility verifier work directory + if: always() + with: + ## TODO: currently matrix.test_suite cannot be used as part of name due to invalid path character. + name: multi_stage_compatibility_verifier_work_dir + retention-days: 3 + path: artifacts.zip diff --git a/.github/workflows/pinot_tests.yml b/.github/workflows/pinot_tests.yml index 01e77cdcb018..ef2828d549b9 100644 --- a/.github/workflows/pinot_tests.yml +++ b/.github/workflows/pinot_tests.yml @@ -314,6 +314,62 @@ jobs: --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED run: .github/workflows/scripts/.pinot_compatibility_verifier.sh + multi-stage-compatibility-verifier: + if: github.repository == 'apache/pinot' + runs-on: ubuntu-latest + strategy: + # Changed to false in order to improve coverage using unsafe buffers + fail-fast: false + matrix: + test_suite: [ "compatibility-verifier/multi-stage-query-engine-test-suite" ] + old_commit: [ + "master" + ] + name: Pinot Multi-Stage Query Engine Compatibility Regression Testing against ${{ matrix.old_commit }} on ${{ matrix.test_suite }} + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: 11 + distribution: 'temurin' + cache: 'maven' + - name: Setup node + uses: actions/setup-node@v4 + with: + node-version: v16.15.0 + cache: 'npm' + cache-dependency-path: pinot-controller/src/main/resources/package-lock.json + - name: Install npm + run: | + npm install -g npm@8.5.5 + npm --version + # Step that does that actual cache save and restore + - uses: actions/cache@v4 + env: + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + - name: Pinot Multi-Stage Query Engine Compatibility Regression Testing + env: + OLD_COMMIT: ${{ matrix.old_commit }} + WORKING_DIR: /tmp/multi-stage-compatibility-verifier + TEST_SUITE: ${{ matrix.test_suite }} + MAVEN_OPTS: > + -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 + -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false + -B -ntp + -XX:+IgnoreUnrecognizedVMOptions + --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED + --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED + --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED + --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED + --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED + run: .github/workflows/scripts/.pinot_compatibility_verifier.sh + quickstarts: if: github.repository == 'apache/pinot' runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 25c9ab85411d..c3b52d6e047a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ cscope.* .classpath .project .svn +.java-version .externalToolBuilders/ maven-eclipse.xml target/ diff --git a/compatibility-verifier/multi-stage-query-engine-test-suite/config/BrokerConfig.properties b/compatibility-verifier/multi-stage-query-engine-test-suite/config/BrokerConfig.properties new file mode 100644 index 000000000000..e56f23c287b6 --- /dev/null +++ b/compatibility-verifier/multi-stage-query-engine-test-suite/config/BrokerConfig.properties @@ -0,0 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +pinot.broker.client.queryPort = 8099 +pinot.zk.server = localhost:2181 +pinot.cluster.name = PinotCluster +pinot.broker.disable.query.groovy=false diff --git a/compatibility-verifier/multi-stage-query-engine-test-suite/config/ControllerConfig.properties b/compatibility-verifier/multi-stage-query-engine-test-suite/config/ControllerConfig.properties new file mode 100644 index 000000000000..9949b2519ea6 --- /dev/null +++ b/compatibility-verifier/multi-stage-query-engine-test-suite/config/ControllerConfig.properties @@ -0,0 +1,25 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +controller.host = localhost +controller.port = 9000 +controller.zk.str = localhost:2181 +controller.data.dir = /tmp/PinotController +controller.helix.cluster.name = PinotCluster +controller.disable.ingestion.groovy = false diff --git a/compatibility-verifier/multi-stage-query-engine-test-suite/config/FeatureTest1-schema.json b/compatibility-verifier/multi-stage-query-engine-test-suite/config/FeatureTest1-schema.json new file mode 100644 index 000000000000..85378dd3a04d --- /dev/null +++ b/compatibility-verifier/multi-stage-query-engine-test-suite/config/FeatureTest1-schema.json @@ -0,0 +1,94 @@ +{ + "dimensionFieldSpecs": [ + { + "dataType": "INT", + "name": "generationNumber" + }, + { + "dataType": "STRING", + "name": "stringDimSV1" + }, + { + "dataType": "STRING", + "name": "stringDimSV2" + }, + { + "dataType": "LONG", + "name": "longDimSV1" + }, + { + "dataType": "LONG", + "name": "longDimSV2" + }, + { + "dataType": "STRING", + "name": "stringDimMV1", + "singleValueField": false + }, + { + "dataType": "STRING", + "name": "stringDimMV2", + "singleValueField": false + }, + { + "dataType": "INT", + "name": "intDimMV1", + "singleValueField": false + }, + { + "dataType": "INT", + "name": "intDimMV2", + "singleValueField": false + }, + { + "dataType": "STRING", + "maxLength": 1000, + "name": "textDim1" + }, + { + "dataType": "BYTES", + "name": "bytesDimSV1" + }, + { + "dataType": "STRING", + "name": "mapDim1__KEYS", + "singleValueField": false + }, + { + "dataType": "INT", + "name": "mapDim1__VALUES", + "singleValueField": false + }, + { + "dataType": "STRING", + "name": "mapDim2json" + } + ], + "metricFieldSpecs": [ + { + "dataType": "INT", + "name": "intMetric1" + }, + { + "dataType": "LONG", + "name": "longMetric1" + }, + { + "dataType": "FLOAT", + "name": "floatMetric1" + }, + { + "dataType": "DOUBLE", + "name": "doubleMetric1" + } + ], + "dateTimeFieldSpecs" : [ + { + "name" : "HoursSinceEpoch", + "dataType" : "INT", + "format" : "1:HOURS:EPOCH", + "granularity": "1:HOURS" + } + ], + "schemaName": "FeatureTest1" +} diff --git a/compatibility-verifier/multi-stage-query-engine-test-suite/config/FeatureTest2-schema.json b/compatibility-verifier/multi-stage-query-engine-test-suite/config/FeatureTest2-schema.json new file mode 100644 index 000000000000..f53c5a2c863f --- /dev/null +++ b/compatibility-verifier/multi-stage-query-engine-test-suite/config/FeatureTest2-schema.json @@ -0,0 +1,94 @@ +{ + "dimensionFieldSpecs": [ + { + "dataType": "INT", + "name": "generationNumber" + }, + { + "dataType": "STRING", + "name": "stringDimSV1" + }, + { + "dataType": "STRING", + "name": "stringDimSV2" + }, + { + "dataType": "LONG", + "name": "longDimSV1" + }, + { + "dataType": "LONG", + "name": "longDimSV2" + }, + { + "dataType": "STRING", + "name": "stringDimMV1", + "singleValueField": false + }, + { + "dataType": "STRING", + "name": "stringDimMV2", + "singleValueField": false + }, + { + "dataType": "INT", + "name": "intDimMV1", + "singleValueField": false + }, + { + "dataType": "INT", + "name": "intDimMV2", + "singleValueField": false + }, + { + "dataType": "STRING", + "maxLength": 1000, + "name": "textDim1" + }, + { + "dataType": "BYTES", + "name": "bytesDimSV1" + }, + { + "dataType": "STRING", + "name": "mapDim1__KEYS", + "singleValueField": false + }, + { + "dataType": "INT", + "name": "mapDim1__VALUES", + "singleValueField": false + }, + { + "dataType": "STRING", + "name": "mapDim2json" + } + ], + "metricFieldSpecs": [ + { + "dataType": "INT", + "name": "intMetric1" + }, + { + "dataType": "LONG", + "name": "longMetric1" + }, + { + "dataType": "FLOAT", + "name": "floatMetric1" + }, + { + "dataType": "DOUBLE", + "name": "doubleMetric1" + } + ], + "dateTimeFieldSpecs" : [ + { + "name" : "HoursSinceEpoch", + "dataType" : "INT", + "format" : "1:HOURS:EPOCH", + "granularity": "1:HOURS" + } + ], + "schemaName": "FeatureTest2" +} diff --git a/compatibility-verifier/multi-stage-query-engine-test-suite/config/ServerConfig.properties b/compatibility-verifier/multi-stage-query-engine-test-suite/config/ServerConfig.properties new file mode 100644 index 000000000000..0bf3a9b47bb8 --- /dev/null +++ b/compatibility-verifier/multi-stage-query-engine-test-suite/config/ServerConfig.properties @@ -0,0 +1,25 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +pinot.server.adminapi.port = 8097 +pinot.server.netty.port = 8098 +pinot.zk.server = localhost:2181 +pinot.cluster.name = PinotCluster +pinot.server.instance.dataDir = /tmp/PinotServer/data +pinot.server.instance.segmentTarDir = /tmp/PinotServer/segments diff --git a/compatibility-verifier/multi-stage-query-engine-test-suite/config/data/FeatureTest1-data-00.csv b/compatibility-verifier/multi-stage-query-engine-test-suite/config/data/FeatureTest1-data-00.csv new file mode 100644 index 000000000000..bb537d9ba333 --- /dev/null +++ b/compatibility-verifier/multi-stage-query-engine-test-suite/config/data/FeatureTest1-data-00.csv @@ -0,0 +1,12 @@ +# HoursSinceEpoch generationNumber stringDimSV1 stringDimSV2 longDimSV1 longDimSV2 stringDimMV1 stringDimMV2 intDimMV1 intDimMV2 textDim1 mapDim1__KEYS mapDim1__VALUES mapDim2json intMetric1 longMetric1 floatMetric1 doubleMetric1 +# Add some common rows from first segment, and some new rows as well +123456,__GENERATION_NUMBER__,"s1-0",s2-0,1,2,m1-0-0;m1-0-1,m2-0-0;m2-0-1,3;4,6;7,Java C++ Python,01a0bc,k1;k2;k3;k4;k5,1;1;2;2;2,"{""k1"":1,""k2"":1,""k3"":2,""k4"":2,""k5"":2}",10,11,12.1,13.1 +123456,__GENERATION_NUMBER__,"s1-0",s2-0,1,2,m1-0-0;m1-0-1,m2-0-0;m2-0-1,3;4,6;7,Java C++ Python,4877625602,k1;k2;k3;k4;k5,3;3;3;3;3,"{""k1"":3,""k2"":3,""k3"":3,""k4"":3,""k5"":3}",10,11,12.1,13.1, # Dupliate of row 0 1 +123456,__GENERATION_NUMBER__,s1-2,s2-2,11,21,m1-2-0;m1-2-1,m2-2-0;m2-2-1,32;42,62;72,Java C++ golang,13225573e3f5,k1;k2;k3;k4;k5,4;5;6;7;7,"{""k1"":4,""k2"":5,""k3"":6,""k4"":7,""k5"":7}",10,21,22.1,23.10 +123456,__GENERATION_NUMBER__,s1-2,s2-2,11,21,m1-3-0;m1-3-1,m2-3-0;m2-3-1,32;42,62;72,Java C++ golang,deadbeef,k1;k2;k3;k4;k5,7;7;7;7;7,"{""k1"":7,""k2"":7,""k3"":7,""k4"":7,""k5"":7}",10,21,22.1,23.10, # All sv cols same as prev +123456,__GENERATION_NUMBER__,s1-4,s2-4,41,22,m1-2-0;m1-2-1,m2-2-0;m2-2-1,42;52,72;82,Java C++ golang,deed0507,k1;k2;k3;k4;k5,7;7;8;8;8,"{""k1"":7,""k2"":7,""k3"":8,""k4"":8,""k5"":8}",14,24,24.1,24.10, # All mv cols same as row 2 +123456,__GENERATION_NUMBER__,s1-5,,,32,m1-5-0,m2-2-0,,92;22,golang shell bash,,k1;k2;k3;k4;k5,7;7;7;7;7,"{""k1"":7,""k2"":7,""k3"":7,""k4"":7,""k5"":7}",,24,,24.10, # Default values for some columns +123456,__GENERATION_NUMBER__,s1-6,s2-6,7611,7621,m1-5-0;m1-5-1;m1-5-2,m2-2-0,392;462,6662;782,C++ golang python,deed0507,k1;k2;k3;k4;k5,7;8;9;10;20,"{""k1"":7,""k2"":8,""k3"":9,""k4"":10,""k5"":20}",101,251,262.1,263.10, # 3 values in MV +123456,__GENERATION_NUMBER__,s1-6,s2-6,7611,7621,m1-5-0;m1-5-1;m1-5-2,m2-2-0,392;462,6662;782,C++ golang python,deed0507,k1;k2;k3;k4;k5,7;8;9;10;20,"{""k1"":7,""k2"":8,""k3"":9,""k4"":10,""k5"":20}",2147483647,251,262.1,263.10, # MAX_INT in int metric +123456,__GENERATION_NUMBER__,s1-6,s2-6,7611,7621,m1-5-0;m1-5-1;m1-5-2,m2-2-0,392;462,6662;782,C++ golang python,deed0507,k1;k2;k3;k4;k5,7;8;9;10;20,"{""k1"":7,""k2"":8,""k3"":9,""k4"":10,""k5"":20}",2147483647,251,262.1,263.10, # MAX_INT in int metric +123456,__GENERATION_NUMBER__,s1-7,s2-7,6766,6777,m1-6-0;m1-6-1;m1-6-2;m1-6-3,m2-6-0;m2-6-1,392;462,6662;782,golang Java,d54d0507,k1;k2;k3;k4;k5,31;31;32;32;32,"{""k1"":31,""k2"":31,""k3"":32,""k4"":32,""k5"":32}",87,251,262.10,263.10 diff --git a/compatibility-verifier/multi-stage-query-engine-test-suite/config/data/FeatureTest2-data-realtime-00.csv b/compatibility-verifier/multi-stage-query-engine-test-suite/config/data/FeatureTest2-data-realtime-00.csv new file mode 100644 index 000000000000..e4e36e47051f --- /dev/null +++ b/compatibility-verifier/multi-stage-query-engine-test-suite/config/data/FeatureTest2-data-realtime-00.csv @@ -0,0 +1,11 @@ +# HoursSinceEpoch generationNumber stringDimSV1 stringDimSV2 longDimSV1 longDimSV2 stringDimMV1 stringDimMV2 intDimMV1 intDimMV2 textDim1 mapDim1__KEYS mapDim1__VALUES mapDim2json intMetric1 longMetric1 floatMetric1 doubleMetric1 +123456,__GENERATION_NUMBER__,"s1-0",s2-0,1,2,m1-0-0;m1-0-1,m2-0-0;m2-0-1,3;4,6;7,Java C++ Python,01a0bc,k1;k2;k3;k4;k5,1;1;2;2;2,"{""k1"":1,""k2"":1,""k3"":2,""k4"":2,""k5"":2}",10,11,12.1,13.1 +123456,__GENERATION_NUMBER__,"s1-0",s2-0,1,2,m1-0-0;m1-0-1,m2-0-0;m2-0-1,3;4,6;7,Java C++ Python,4877625602,k1;k2;k3;k4;k5,3;3;3;3;3,"{""k1"":3,""k2"":3,""k3"":3,""k4"":3,""k5"":3}",10,11,12.1,13.1, # Dupliate of row 0 1 +123456,__GENERATION_NUMBER__,s1-2,s2-2,11,21,m1-2-0;m1-2-1,m2-2-0;m2-2-1,32;42,62;72,Java C++ golang,13225573e3f5,k1;k2;k3;k4;k5,4;5;6;7;7,"{""k1"":4,""k2"":5,""k3"":6,""k4"":7,""k5"":7}",10,21,22.1,23.10 +123456,__GENERATION_NUMBER__,s1-2,s2-2,11,21,m1-3-0;m1-3-1,m2-3-0;m2-3-1,32;42,62;72,Java C++ golang,deadbeef,k1;k2;k3;k4;k5,7;7;7;7;7,"{""k1"":7,""k2"":7,""k3"":7,""k4"":7,""k5"":7}",10,21,22.1,23.10, # All sv cols same as prev +123456,__GENERATION_NUMBER__,s1-4,s2-4,41,22,m1-2-0;m1-2-1,m2-2-0;m2-2-1,42;52,72;82,Java C++ golang,deed0507,k1;k2;k3;k4;k5,7;7;8;8;8,"{""k1"":7,""k2"":7,""k3"":8,""k4"":8,""k5"":8}",14,24,24.1,24.10, # All mv cols same as row 2 +123456,__GENERATION_NUMBER__,s1-5,,,32,m1-5-0,m2-2-0,,92;22,golang shell bash,,k1;k2;k3;k4;k5,7;7;7;7;7,"{""k1"":7,""k2"":7,""k3"":7,""k4"":7,""k5"":7}",,24,,24.10, # Default values for some columns +123456,__GENERATION_NUMBER__,s1-6,s2-6,7611,7621,m1-5-0;m1-5-1;m1-5-2,m2-2-0,392;462,6662;782,C++ golang python,deed0507,k1;k2;k3;k4;k5,7;8;9;10;20,"{""k1"":7,""k2"":8,""k3"":9,""k4"":10,""k5"":20}",101,251,262.1,263.10, # 3 values in MV +123456,__GENERATION_NUMBER__,s1-6,s2-6,7611,7621,m1-5-0;m1-5-1;m1-5-2,m2-2-0,392;462,6662;782,C++ golang python,deed0507,k1;k2;k3;k4;k5,7;8;9;10;20,"{""k1"":7,""k2"":8,""k3"":9,""k4"":10,""k5"":20}",2147483647,251,262.1,263.10, # MAX_INT in int metric +123456,__GENERATION_NUMBER__,s1-6,s2-6,7611,7621,m1-5-0;m1-5-1;m1-5-2,m2-2-0,392;462,6662;782,C++ golang python,deed0507,k1;k2;k3;k4;k5,7;8;9;10;20,"{""k1"":7,""k2"":8,""k3"":9,""k4"":10,""k5"":20}",2147483647,251,262.1,263.10, # MAX_INT in int metric +123456,__GENERATION_NUMBER__,s1-7,s2-7,6766,6777,m1-6-0;m1-6-1;m1-6-2;m1-6-3,m2-6-0;m2-6-1,392;462,6662;782,golang Java,d54d0507,k1;k2;k3;k4;k5,31;31;32;32;32,"{""k1"":31,""k2"":31,""k3"":32,""k4"":32,""k5"":32}",87,251,262.10,263.10 diff --git a/compatibility-verifier/multi-stage-query-engine-test-suite/config/data/recordReaderConfig.json b/compatibility-verifier/multi-stage-query-engine-test-suite/config/data/recordReaderConfig.json new file mode 100644 index 000000000000..ebaa04e10464 --- /dev/null +++ b/compatibility-verifier/multi-stage-query-engine-test-suite/config/data/recordReaderConfig.json @@ -0,0 +1,5 @@ +{ + "commentMarker" : "#", + "header" : + "HoursSinceEpoch,generationNumber,stringDimSV1,stringDimSV2,longDimSV1,longDimSV2,stringDimMV1,stringDimMV2,intDimMV1,intDimMV2,textDim1,bytesDimSV1,mapDim1__KEYS,mapDim1__VALUES,mapDim2json,intMetric1,longMetric1,floatMetric1,doubleMetric1" +} diff --git a/compatibility-verifier/multi-stage-query-engine-test-suite/config/feature-test-1.json b/compatibility-verifier/multi-stage-query-engine-test-suite/config/feature-test-1.json new file mode 100644 index 000000000000..381e6885d19a --- /dev/null +++ b/compatibility-verifier/multi-stage-query-engine-test-suite/config/feature-test-1.json @@ -0,0 +1,46 @@ +{ + "fieldConfigList": [ + { + "encodingType": "RAW", + "indexType": "TEXT", + "name": "textDim1", + "properties": { + "deriveNumDocsPerChunkForRawIndex": "true", + "rawIndexWriterVersion": "3" + } + } + ], + "metadata": { + "customConfigs": { + "d2Name": "" + } + }, + "segmentsConfig": { + "replication": "1", + "retentionTimeUnit": "", + "retentionTimeValue": "", + "segmentAssignmentStrategy": "BalanceNumSegmentAssignmentStrategy", + "segmentPushFrequency": "daily", + "segmentPushType": "REFRESH", + "timeColumnName": "HoursSinceEpoch", + "timeType": "HOURS" + }, + "tableIndexConfig": { + "aggregateMetrics": false, + "autoGeneratedInvertedIndex": false, + "createInvertedIndexDuringSegmentGeneration": false, + "enableDefaultStarTree": false, + "enableDynamicStarTreeCreation": false, + "loadMode": "MMAP", + "noDictionaryColumns": ["textDim1"], + "nullHandlingEnabled": false, + "sortedColumn": [], + "streamConfigs": {} + }, + "tableName": "FeatureTest1_OFFLINE", + "tableType": "OFFLINE", + "tenants": { + "broker": "DefaultTenant", + "server": "DefaultTenant" + } +} diff --git a/compatibility-verifier/multi-stage-query-engine-test-suite/config/feature-test-2-realtime-stream-config.json b/compatibility-verifier/multi-stage-query-engine-test-suite/config/feature-test-2-realtime-stream-config.json new file mode 100644 index 000000000000..441ce201a903 --- /dev/null +++ b/compatibility-verifier/multi-stage-query-engine-test-suite/config/feature-test-2-realtime-stream-config.json @@ -0,0 +1,8 @@ +{ + "streamType": "kafka", + "stream.kafka.consumer.type": "simple", + "topicName": "PinotRealtimeFeatureTest2Event", + "partitionColumn": "longDimSV1", + "numPartitions": "1", + "stream.kafka.consumer.prop.auto.offset.reset": "smallest" +} diff --git a/compatibility-verifier/multi-stage-query-engine-test-suite/config/feature-test-2-realtime.json b/compatibility-verifier/multi-stage-query-engine-test-suite/config/feature-test-2-realtime.json new file mode 100644 index 000000000000..6df335e48841 --- /dev/null +++ b/compatibility-verifier/multi-stage-query-engine-test-suite/config/feature-test-2-realtime.json @@ -0,0 +1,61 @@ +{ + "fieldConfigList": [ + { + "encodingType": "RAW", + "indexType": "TEXT", + "name": "textDim1", + "properties": { + "deriveNumDocsPerChunkForRawIndex": "true", + "rawIndexWriterVersion": "3" + } + } + ], + "metadata": { + "customConfigs": { + "d2Name": "" + } + }, + "segmentsConfig": { + "replicasPerPartition": "1", + "replication": "1", + "retentionTimeUnit": "", + "retentionTimeValue": "", + "schemaName": "FeatureTest2", + "segmentAssignmentStrategy": "BalanceNumSegmentAssignmentStrategy", + "segmentPushFrequency": "daily", + "segmentPushType": "APPEND", + "timeColumnName": "HoursSinceEpoch", + "timeType": "HOURS" + }, + "tableIndexConfig": { + "aggregateMetrics": false, + "autoGeneratedInvertedIndex": false, + "bloomFilterColumns": [], + "createInvertedIndexDuringSegmentGeneration": false, + "enableDefaultStarTree": false, + "enableDynamicStarTreeCreation": false, + "loadMode": "MMAP", + "noDictionaryColumns": [], + "nullHandlingEnabled": false, + "segmentFormatVersion": "v3", + "sortedColumn": [], + "streamConfigs": { + "realtime.segment.flush.threshold.size": "63", + "realtime.segment.flush.threshold.time": "1h", + "streamType": "kafka", + "stream.kafka.topic.name": "PinotRealtimeFeatureTest2Event", + "stream.kafka.consumer.type": "simple", + "stream.kafka.decoder.class.name": "org.apache.pinot.plugin.stream.kafka.KafkaJSONMessageDecoder", + "stream.kafka.consumer.factory.class.name": "org.apache.pinot.plugin.stream.kafka20.KafkaConsumerFactory", + "stream.kafka.broker.list": "localhost:19092", + "stream.kafka.zk.broker.url": "localhost:2181/kafka", + "stream.kafka.consumer.prop.auto.offset.reset": "largest" + } + }, + "tableName": "FeatureTest2", + "tableType": "REALTIME", + "tenants": { + "broker": "DefaultTenant", + "server": "DefaultTenant" + } +} diff --git a/compatibility-verifier/multi-stage-query-engine-test-suite/config/queries/feature-test-multi-stage.queries b/compatibility-verifier/multi-stage-query-engine-test-suite/config/queries/feature-test-multi-stage.queries new file mode 100644 index 000000000000..00d0c6cb8a74 --- /dev/null +++ b/compatibility-verifier/multi-stage-query-engine-test-suite/config/queries/feature-test-multi-stage.queries @@ -0,0 +1,31 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Joins +SELECT COUNT(*) FROM FeatureTest1 ft1 INNER JOIN FeatureTest2 ft2 ON ft1.stringDimSV1 = ft2.stringDimSV1 WHERE ft1.generationNumber = __GENERATION_NUMBER__ AND ft2.generationNumber = __GENERATION_NUMBER__ +SELECT ft1.stringDimSV1, COUNT(ft1.stringDimSV1) FROM FeatureTest1 ft1 INNER JOIN FeatureTest2 ft2 ON ft1.stringDimSV1 = ft2.stringDimSV1 WHERE ft1.generationNumber = __GENERATION_NUMBER__ AND ft2.generationNumber = __GENERATION_NUMBER__ GROUP BY ft1.stringDimSV1 +SELECT ft1.stringDimSV2, SUM(ft1.floatMetric1) FROM FeatureTest1 ft1 INNER JOIN FeatureTest2 ft2 ON ft1.stringDimSV1 = ft2.stringDimSV1 WHERE ft1.generationNumber = __GENERATION_NUMBER__ AND ft2.generationNumber = __GENERATION_NUMBER__ GROUP BY ft1.stringDimSV2 +SELECT ft1.stringDimSV1 FROM FeatureTest1 ft1 WHERE ft1.generationNumber = __GENERATION_NUMBER__ AND EXISTS (SELECT 1 FROM FeatureTest2 ft2 WHERE ft2.generationNumber = __GENERATION_NUMBER__ AND ft2.stringDimSV2 = ft1.stringDimSV1) + +# Set operations +SELECT * FROM (SELECT stringDimSV1 FROM FeatureTest1 WHERE generationNumber = __GENERATION_NUMBER__) INTERSECT (SELECT stringDimSV1 FROM FeatureTest2 WHERE generationNumber = __GENERATION_NUMBER__) +SELECT * FROM (SELECT stringDimSV1 FROM FeatureTest1 WHERE generationNumber = __GENERATION_NUMBER__) UNION (SELECT stringDimSV1 FROM FeatureTest2 WHERE generationNumber = __GENERATION_NUMBER__) + +# Windows +SELECT stringDimSV1, longMetric1, SUM(longMetric1) OVER (PARTITION BY stringDimSV1) FROM FeatureTest1 WHERE generationNumber = __GENERATION_NUMBER__ diff --git a/compatibility-verifier/multi-stage-query-engine-test-suite/config/query-results/feature-test-multi-stage.results b/compatibility-verifier/multi-stage-query-engine-test-suite/config/query-results/feature-test-multi-stage.results new file mode 100644 index 000000000000..b073dd9e1cbb --- /dev/null +++ b/compatibility-verifier/multi-stage-query-engine-test-suite/config/query-results/feature-test-multi-stage.results @@ -0,0 +1,31 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Joins +{"resultTable":{"dataSchema":{"columnNames":["EXPR$0"],"columnDataTypes":["LONG"]},"rows":[[130]]},"requestId":"11345778000000000","stageStats":{"type":"MAILBOX_RECEIVE","executionTimeMs":36,"emittedRows":1,"fanIn":1,"rawMessages":2,"deserializedBytes":714,"upstreamWaitMs":40,"children":[{"type":"MAILBOX_SEND","executionTimeMs":36,"emittedRows":1,"stage":1,"parallelism":1,"fanOut":1,"rawMessages":2,"serializedBytes":99,"serializationTimeMs":2,"children":[{"type":"AGGREGATE","executionTimeMs":1,"emittedRows":1,"children":[{"type":"MAILBOX_RECEIVE","executionTimeMs":1,"emittedRows":1,"fanIn":1,"inMemoryMessages":2,"upstreamWaitMs":7,"children":[{"type":"MAILBOX_SEND","executionTimeMs":1,"emittedRows":1,"stage":2,"parallelism":1,"fanOut":1,"inMemoryMessages":2,"children":[{"type":"AGGREGATE","executionTimeMs":1,"emittedRows":1,"children":[{"type":"HASH_JOIN","executionTimeMs":1,"emittedRows":130,"timeBuildingHashTableMs":1,"children":[{"type":"MAILBOX_RECEIVE","emittedRows":10,"fanIn":1,"inMemoryMessages":2,"upstreamWaitMs":4,"children":[{"type":"MAILBOX_SEND","executionTimeMs":4,"emittedRows":10,"stage":3,"parallelism":1,"fanOut":1,"inMemoryMessages":2,"children":[{"type":"LEAF","table":"FeatureTest1","executionTimeMs":5,"emittedRows":10,"numDocsScanned":10,"numEntriesScannedPostFilter":10,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"totalDocs":10}]}]},{"type":"MAILBOX_RECEIVE","emittedRows":66,"fanIn":1,"inMemoryMessages":3,"upstreamWaitMs":3,"children":[{"type":"MAILBOX_SEND","executionTimeMs":4,"emittedRows":66,"stage":4,"parallelism":1,"fanOut":1,"inMemoryMessages":3,"children":[{"type":"LEAF","table":"FeatureTest2","executionTimeMs":4,"emittedRows":66,"numDocsScanned":66,"numEntriesScannedPostFilter":66,"numSegmentsQueried":2,"numSegmentsProcessed":2,"numSegmentsMatched":2,"numConsumingSegmentsQueried":1,"minConsumingFreshnessTimeMs":1716274896691,"totalDocs":66}]}]}]}]}]}]}]}]}]},"brokerId":"Broker_192.168.29.25_8099","exceptions":[],"numServersQueried":0,"numServersResponded":0,"numSegmentsQueried":3,"numSegmentsProcessed":3,"numSegmentsMatched":3,"numConsumingSegmentsQueried":1,"numConsumingSegmentsProcessed":0,"numConsumingSegmentsMatched":0,"numDocsScanned":76,"numEntriesScannedInFilter":0,"numEntriesScannedPostFilter":76,"numGroupsLimitReached":false,"maxRowsInJoinReached":false,"totalDocs":76,"timeUsedMs":975,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"offlineSystemActivitiesCpuTimeNs":0,"realtimeSystemActivitiesCpuTimeNs":0,"offlineResponseSerializationCpuTimeNs":0,"realtimeResponseSerializationCpuTimeNs":0,"offlineTotalCpuTimeNs":0,"realtimeTotalCpuTimeNs":0,"brokerReduceTimeMs":56,"partialResult":false,"exceptionsSize":0,"numRowsResultSet":1,"maxRowsInOperator":130,"numSegmentsPrunedByBroker":0,"minConsumingFreshnessTimeMs":1716274896691,"numSegmentsPrunedByServer":0,"numSegmentsPrunedInvalid":0,"numSegmentsPrunedByLimit":0,"numSegmentsPrunedByValue":0,"explainPlanNumEmptyFilterSegments":0,"explainPlanNumMatchAllFilterSegments":0} +{"resultTable":{"dataSchema":{"columnNames":["stringDimSV1","EXPR$1"],"columnDataTypes":["STRING","LONG"]},"rows":[["s1-4",7],["s1-6",54],["s1-2",28],["s1-0",28],["s1-5",7],["s1-7",6]]},"requestId":"11345778000000001","stageStats":{"type":"MAILBOX_RECEIVE","executionTimeMs":3,"emittedRows":6,"fanIn":1,"rawMessages":2,"deserializedBytes":824,"upstreamWaitMs":3,"children":[{"type":"MAILBOX_SEND","executionTimeMs":2,"emittedRows":6,"stage":1,"parallelism":1,"fanOut":1,"rawMessages":2,"serializedBytes":245,"children":[{"type":"AGGREGATE","executionTimeMs":2,"emittedRows":6,"children":[{"type":"MAILBOX_RECEIVE","executionTimeMs":1,"emittedRows":6,"fanIn":1,"inMemoryMessages":2,"upstreamWaitMs":4,"children":[{"type":"MAILBOX_SEND","executionTimeMs":1,"emittedRows":6,"stage":2,"parallelism":1,"fanOut":1,"inMemoryMessages":2,"children":[{"type":"AGGREGATE","executionTimeMs":1,"emittedRows":6,"children":[{"type":"HASH_JOIN","emittedRows":130,"timeBuildingHashTableMs":1,"children":[{"type":"MAILBOX_RECEIVE","emittedRows":10,"fanIn":1,"inMemoryMessages":2,"upstreamWaitMs":3,"children":[{"type":"MAILBOX_SEND","emittedRows":10,"stage":3,"parallelism":1,"fanOut":1,"inMemoryMessages":2,"children":[{"type":"LEAF","table":"FeatureTest1","executionTimeMs":1,"emittedRows":10,"numDocsScanned":10,"numEntriesScannedPostFilter":10,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"totalDocs":10}]}]},{"type":"MAILBOX_RECEIVE","emittedRows":66,"fanIn":1,"inMemoryMessages":3,"upstreamWaitMs":3,"children":[{"type":"MAILBOX_SEND","emittedRows":66,"stage":4,"parallelism":1,"fanOut":1,"inMemoryMessages":3,"children":[{"type":"LEAF","table":"FeatureTest2","executionTimeMs":1,"emittedRows":66,"numDocsScanned":66,"numEntriesScannedPostFilter":66,"numSegmentsQueried":2,"numSegmentsProcessed":2,"numSegmentsMatched":2,"numConsumingSegmentsQueried":1,"minConsumingFreshnessTimeMs":1716276514329,"totalDocs":66}]}]}]}]}]}]}]}]}]},"brokerId":"Broker_192.168.29.25_8099","exceptions":[],"numServersQueried":0,"numServersResponded":0,"numSegmentsQueried":3,"numSegmentsProcessed":3,"numSegmentsMatched":3,"numConsumingSegmentsQueried":1,"numConsumingSegmentsProcessed":0,"numConsumingSegmentsMatched":0,"numDocsScanned":76,"numEntriesScannedInFilter":0,"numEntriesScannedPostFilter":76,"numGroupsLimitReached":false,"maxRowsInJoinReached":false,"totalDocs":76,"timeUsedMs":49,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"offlineSystemActivitiesCpuTimeNs":0,"realtimeSystemActivitiesCpuTimeNs":0,"offlineResponseSerializationCpuTimeNs":0,"realtimeResponseSerializationCpuTimeNs":0,"offlineTotalCpuTimeNs":0,"realtimeTotalCpuTimeNs":0,"brokerReduceTimeMs":3,"partialResult":false,"maxRowsInOperator":130,"numRowsResultSet":6,"exceptionsSize":0,"numSegmentsPrunedByBroker":0,"minConsumingFreshnessTimeMs":1716276514329,"numSegmentsPrunedByServer":0,"numSegmentsPrunedInvalid":0,"numSegmentsPrunedByLimit":0,"numSegmentsPrunedByValue":0,"explainPlanNumEmptyFilterSegments":0,"explainPlanNumMatchAllFilterSegments":0} +{"resultTable":{"dataSchema":{"columnNames":["stringDimSV2","EXPR$1"],"columnDataTypes":["STRING","DOUBLE"]},"rows":[["s2-6",14153.4],["s2-2",618.8],["s2-0",338.8],["s2-4",168.7],["s2-7",1572.6],["null",0.0]]},"requestId":"11345778000000002","stageStats":{"type":"MAILBOX_RECEIVE","executionTimeMs":2,"emittedRows":6,"fanIn":1,"rawMessages":2,"deserializedBytes":826,"upstreamWaitMs":3,"children":[{"type":"MAILBOX_SEND","executionTimeMs":4,"emittedRows":6,"stage":1,"parallelism":1,"fanOut":1,"rawMessages":2,"serializedBytes":247,"children":[{"type":"AGGREGATE","executionTimeMs":3,"emittedRows":6,"children":[{"type":"MAILBOX_RECEIVE","executionTimeMs":3,"emittedRows":6,"fanIn":1,"inMemoryMessages":2,"upstreamWaitMs":4,"children":[{"type":"MAILBOX_SEND","executionTimeMs":2,"emittedRows":6,"stage":2,"parallelism":1,"fanOut":1,"inMemoryMessages":2,"children":[{"type":"AGGREGATE","executionTimeMs":2,"emittedRows":6,"children":[{"type":"HASH_JOIN","executionTimeMs":1,"emittedRows":130,"timeBuildingHashTableMs":1,"children":[{"type":"MAILBOX_RECEIVE","emittedRows":10,"fanIn":1,"inMemoryMessages":2,"upstreamWaitMs":2,"children":[{"type":"MAILBOX_SEND","emittedRows":10,"stage":3,"parallelism":1,"fanOut":1,"inMemoryMessages":2,"children":[{"type":"LEAF","table":"FeatureTest1","emittedRows":10,"numDocsScanned":10,"numEntriesScannedPostFilter":30,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"totalDocs":10}]}]},{"type":"MAILBOX_RECEIVE","executionTimeMs":1,"emittedRows":66,"fanIn":1,"inMemoryMessages":3,"upstreamWaitMs":2,"children":[{"type":"MAILBOX_SEND","emittedRows":66,"stage":4,"parallelism":1,"fanOut":1,"inMemoryMessages":3,"children":[{"type":"LEAF","table":"FeatureTest2","emittedRows":66,"numDocsScanned":66,"numEntriesScannedPostFilter":66,"numSegmentsQueried":2,"numSegmentsProcessed":2,"numSegmentsMatched":2,"numConsumingSegmentsQueried":1,"minConsumingFreshnessTimeMs":1716276645639,"totalDocs":66}]}]}]}]}]}]}]}]}]},"brokerId":"Broker_192.168.29.25_8099","exceptions":[],"numServersQueried":0,"numServersResponded":0,"numSegmentsQueried":3,"numSegmentsProcessed":3,"numSegmentsMatched":3,"numConsumingSegmentsQueried":1,"numConsumingSegmentsProcessed":0,"numConsumingSegmentsMatched":0,"numDocsScanned":76,"numEntriesScannedInFilter":0,"numEntriesScannedPostFilter":96,"numGroupsLimitReached":false,"maxRowsInJoinReached":false,"totalDocs":76,"timeUsedMs":21,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"offlineSystemActivitiesCpuTimeNs":0,"realtimeSystemActivitiesCpuTimeNs":0,"offlineResponseSerializationCpuTimeNs":0,"realtimeResponseSerializationCpuTimeNs":0,"offlineTotalCpuTimeNs":0,"realtimeTotalCpuTimeNs":0,"brokerReduceTimeMs":3,"partialResult":false,"exceptionsSize":0,"numRowsResultSet":6,"maxRowsInOperator":130,"numSegmentsPrunedByBroker":0,"minConsumingFreshnessTimeMs":1716276645639,"numSegmentsPrunedByServer":0,"numSegmentsPrunedInvalid":0,"numSegmentsPrunedByLimit":0,"numSegmentsPrunedByValue":0,"explainPlanNumEmptyFilterSegments":0,"explainPlanNumMatchAllFilterSegments":0} +{"resultTable":{"dataSchema":{"columnNames":["stringDimSV1"],"columnDataTypes":["STRING"]},"rows":[]},"requestId":"11345778000000003","stageStats":{"type":"MAILBOX_RECEIVE","fanIn":1,"rawMessages":1,"deserializedBytes":132,"children":[{"type":"MAILBOX_SEND","stage":1,"parallelism":1,"fanOut":1,"rawMessages":1,"children":[{"type":"LEAF","table":"FeatureTest1","numSegmentsQueried":1,"totalDocs":10,"numSegmentsPrunedByServer":1}]}]},"brokerId":"Broker_192.168.29.25_8099","exceptions":[],"numServersQueried":0,"numServersResponded":0,"numSegmentsQueried":1,"numSegmentsProcessed":0,"numSegmentsMatched":0,"numConsumingSegmentsQueried":0,"numConsumingSegmentsProcessed":0,"numConsumingSegmentsMatched":0,"numDocsScanned":0,"numEntriesScannedInFilter":0,"numEntriesScannedPostFilter":0,"numGroupsLimitReached":false,"maxRowsInJoinReached":false,"totalDocs":10,"timeUsedMs":43,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"offlineSystemActivitiesCpuTimeNs":0,"realtimeSystemActivitiesCpuTimeNs":0,"offlineResponseSerializationCpuTimeNs":0,"realtimeResponseSerializationCpuTimeNs":0,"offlineTotalCpuTimeNs":0,"realtimeTotalCpuTimeNs":0,"brokerReduceTimeMs":0,"partialResult":false,"maxRowsInOperator":0,"exceptionsSize":0,"numRowsResultSet":0,"numSegmentsPrunedByBroker":0,"minConsumingFreshnessTimeMs":0,"numSegmentsPrunedByServer":1,"numSegmentsPrunedInvalid":0,"numSegmentsPrunedByLimit":0,"numSegmentsPrunedByValue":0,"explainPlanNumEmptyFilterSegments":0,"explainPlanNumMatchAllFilterSegments":0} + +# Set operations +{"resultTable":{"dataSchema":{"columnNames":["stringDimSV1"],"columnDataTypes":["STRING"]},"rows":[["s1-0"],["s1-2"],["s1-4"],["s1-5"],["s1-6"],["s1-7"]]},"requestId":"11345778000000004","stageStats":{"type":"MAILBOX_RECEIVE","executionTimeMs":1,"emittedRows":6,"fanIn":1,"rawMessages":2,"deserializedBytes":628,"upstreamWaitMs":2,"children":[{"type":"MAILBOX_SEND","executionTimeMs":1,"emittedRows":6,"stage":1,"parallelism":1,"fanOut":1,"rawMessages":2,"serializedBytes":174,"children":[{"children":[{"type":"MAILBOX_RECEIVE","emittedRows":6,"children":[{"type":"MAILBOX_SEND","emittedRows":10,"stage":2,"parallelism":1,"fanOut":1,"inMemoryMessages":2,"children":[{"type":"LEAF","table":"FeatureTest1","emittedRows":10,"numDocsScanned":10,"numEntriesScannedPostFilter":10,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"totalDocs":10}]}]},{"type":"MAILBOX_RECEIVE","emittedRows":10,"fanIn":1,"inMemoryMessages":2,"upstreamWaitMs":1,"children":[{"type":"MAILBOX_SEND","emittedRows":66,"stage":3,"parallelism":1,"fanOut":1,"inMemoryMessages":3,"children":[{"type":"LEAF","table":"FeatureTest2","emittedRows":66,"numDocsScanned":66,"numEntriesScannedPostFilter":66,"numSegmentsQueried":2,"numSegmentsProcessed":2,"numSegmentsMatched":2,"numConsumingSegmentsQueried":1,"minConsumingFreshnessTimeMs":1716276920346,"totalDocs":66}]}]}]}]}]},"brokerId":"Broker_192.168.29.25_8099","exceptions":[],"numServersQueried":0,"numServersResponded":0,"numSegmentsQueried":3,"numSegmentsProcessed":3,"numSegmentsMatched":3,"numConsumingSegmentsQueried":1,"numConsumingSegmentsProcessed":0,"numConsumingSegmentsMatched":0,"numDocsScanned":76,"numEntriesScannedInFilter":0,"numEntriesScannedPostFilter":76,"numGroupsLimitReached":false,"maxRowsInJoinReached":false,"totalDocs":76,"timeUsedMs":35,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"offlineSystemActivitiesCpuTimeNs":0,"realtimeSystemActivitiesCpuTimeNs":0,"offlineResponseSerializationCpuTimeNs":0,"realtimeResponseSerializationCpuTimeNs":0,"offlineTotalCpuTimeNs":0,"realtimeTotalCpuTimeNs":0,"brokerReduceTimeMs":2,"partialResult":false,"numSegmentsPrunedByBroker":0,"minConsumingFreshnessTimeMs":1716276920346,"numSegmentsPrunedByServer":0,"numSegmentsPrunedInvalid":0,"numSegmentsPrunedByLimit":0,"numSegmentsPrunedByValue":0,"explainPlanNumEmptyFilterSegments":0,"explainPlanNumMatchAllFilterSegments":0,"numRowsResultSet":6,"exceptionsSize":0,"maxRowsInOperator":66} +{"resultTable":{"dataSchema":{"columnNames":["stringDimSV1"],"columnDataTypes":["STRING"]},"rows":[["s1-4"],["s1-6"],["s1-2"],["s1-0"],["s1-5"],["s1-7"]]},"requestId":"11345778000000006","stageStats":{"type":"MAILBOX_RECEIVE","emittedRows":6,"fanIn":1,"rawMessages":2,"deserializedBytes":744,"upstreamWaitMs":1,"children":[{"type":"MAILBOX_SEND","executionTimeMs":2,"emittedRows":6,"stage":1,"parallelism":1,"fanOut":1,"rawMessages":2,"serializedBytes":174,"children":[{"type":"AGGREGATE","executionTimeMs":2,"emittedRows":6,"children":[{"type":"MAILBOX_RECEIVE","executionTimeMs":2,"emittedRows":6,"fanIn":1,"inMemoryMessages":2,"upstreamWaitMs":3,"children":[{"type":"MAILBOX_SEND","executionTimeMs":1,"emittedRows":6,"stage":2,"parallelism":1,"fanOut":1,"inMemoryMessages":2,"children":[{"type":"AGGREGATE","executionTimeMs":1,"emittedRows":6,"children":[{"type":"UNION","executionTimeMs":1,"emittedRows":76,"children":[{"type":"MAILBOX_RECEIVE","executionTimeMs":1,"emittedRows":10,"fanIn":1,"inMemoryMessages":2,"upstreamWaitMs":1,"children":[{"type":"MAILBOX_SEND","emittedRows":10,"stage":3,"parallelism":1,"fanOut":1,"inMemoryMessages":2,"children":[{"type":"LEAF","table":"FeatureTest1","emittedRows":10,"numDocsScanned":10,"numEntriesScannedPostFilter":10,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"totalDocs":10}]}]},{"type":"MAILBOX_RECEIVE","emittedRows":66,"fanIn":1,"inMemoryMessages":3,"upstreamWaitMs":1,"children":[{"type":"MAILBOX_SEND","emittedRows":66,"stage":4,"parallelism":1,"fanOut":1,"inMemoryMessages":3,"children":[{"type":"LEAF","table":"FeatureTest2","emittedRows":66,"numDocsScanned":66,"numEntriesScannedPostFilter":66,"numSegmentsQueried":2,"numSegmentsProcessed":2,"numSegmentsMatched":2,"numConsumingSegmentsQueried":1,"minConsumingFreshnessTimeMs":1716276920346,"totalDocs":66}]}]}]}]}]}]}]}]}]},"brokerId":"Broker_192.168.29.25_8099","exceptions":[],"numServersQueried":0,"numServersResponded":0,"numSegmentsQueried":3,"numSegmentsProcessed":3,"numSegmentsMatched":3,"numConsumingSegmentsQueried":1,"numConsumingSegmentsProcessed":0,"numConsumingSegmentsMatched":0,"numDocsScanned":76,"numEntriesScannedInFilter":0,"numEntriesScannedPostFilter":76,"numGroupsLimitReached":false,"maxRowsInJoinReached":false,"totalDocs":76,"timeUsedMs":13,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"offlineSystemActivitiesCpuTimeNs":0,"realtimeSystemActivitiesCpuTimeNs":0,"offlineResponseSerializationCpuTimeNs":0,"realtimeResponseSerializationCpuTimeNs":0,"offlineTotalCpuTimeNs":0,"realtimeTotalCpuTimeNs":0,"brokerReduceTimeMs":1,"partialResult":false,"numSegmentsPrunedByBroker":0,"minConsumingFreshnessTimeMs":1716276920346,"numSegmentsPrunedByServer":0,"numSegmentsPrunedInvalid":0,"numSegmentsPrunedByLimit":0,"numSegmentsPrunedByValue":0,"explainPlanNumEmptyFilterSegments":0,"explainPlanNumMatchAllFilterSegments":0,"numRowsResultSet":6,"exceptionsSize":0,"maxRowsInOperator":76} + +# Windows +{"resultTable":{"dataSchema":{"columnNames":["stringDimSV1","longMetric1","EXPR$2"],"columnDataTypes":["STRING","LONG","LONG"]},"rows":[["s1-5",24,24],["s1-4",24,24],["s1-7",251,251],["s1-6",251,753],["s1-6",251,753],["s1-6",251,753],["s1-0",11,22],["s1-0",11,22],["s1-2",21,42],["s1-2",21,42]]},"partialResult":false,"exceptions":[],"numGroupsLimitReached":false,"maxRowsInJoinReached":false,"timeUsedMs":34,"stageStats":{"type":"MAILBOX_RECEIVE","executionTimeMs":3,"emittedRows":10,"fanIn":1,"rawMessages":2,"deserializedBytes":717,"upstreamWaitMs":1,"children":[{"type":"MAILBOX_SEND","executionTimeMs":1,"emittedRows":10,"stage":1,"parallelism":1,"fanOut":1,"rawMessages":2,"serializedBytes":407,"children":[{"type":"TRANSFORM","executionTimeMs":1,"emittedRows":10,"children":[{"type":"WINDOW","executionTimeMs":1,"emittedRows":10,"children":[{"type":"MAILBOX_RECEIVE","emittedRows":10,"fanIn":1,"inMemoryMessages":2,"upstreamWaitMs":2,"children":[{"type":"MAILBOX_SEND","emittedRows":10,"stage":2,"parallelism":1,"fanOut":1,"inMemoryMessages":2,"children":[{"type":"LEAF","table":"FeatureTest1","executionTimeMs":1,"emittedRows":10,"numDocsScanned":10,"totalDocs":10,"numEntriesScannedPostFilter":20,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1}]}]}]}]}]}]},"maxRowsInOperator":10,"requestId":"11345778000000006","brokerId":"Broker_192.168.29.25_8099","numDocsScanned":10,"totalDocs":10,"numEntriesScannedInFilter":0,"numEntriesScannedPostFilter":20,"numServersQueried":0,"numServersResponded":0,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"numConsumingSegmentsQueried":0,"numConsumingSegmentsProcessed":0,"numConsumingSegmentsMatched":0,"minConsumingFreshnessTimeMs":0,"numSegmentsPrunedByBroker":0,"numSegmentsPrunedByServer":0,"numSegmentsPrunedInvalid":0,"numSegmentsPrunedByLimit":0,"numSegmentsPrunedByValue":0,"brokerReduceTimeMs":3,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"offlineSystemActivitiesCpuTimeNs":0,"realtimeSystemActivitiesCpuTimeNs":0,"offlineResponseSerializationCpuTimeNs":0,"realtimeResponseSerializationCpuTimeNs":0,"offlineTotalCpuTimeNs":0,"realtimeTotalCpuTimeNs":0,"explainPlanNumEmptyFilterSegments":0,"explainPlanNumMatchAllFilterSegments":0,"traceInfo":{}} diff --git a/compatibility-verifier/multi-stage-query-engine-test-suite/post-broker-rollback.yaml b/compatibility-verifier/multi-stage-query-engine-test-suite/post-broker-rollback.yaml new file mode 100644 index 000000000000..3e927386b107 --- /dev/null +++ b/compatibility-verifier/multi-stage-query-engine-test-suite/post-broker-rollback.yaml @@ -0,0 +1,43 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Operations to be done. +description: Operations to be run after broker rollback +operations: + - type: segmentOp + description: Add segment FeatureTest1_Segment6 to table FeatureTest1 + op: UPLOAD + inputDataFileName: data/FeatureTest1-data-00.csv + schemaFileName: FeatureTest1-schema.json + tableConfigFileName: feature-test-1.json + recordReaderConfigFileName: data/recordReaderConfig.json + segmentName: FeatureTest1_Segment6 + - type: streamOp + description: publish rows to PinotRealtimeFeatureTest2Event + op: PRODUCE + streamConfigFileName: feature-test-2-realtime-stream-config.json + numRows: 66 + inputDataFileName: data/FeatureTest2-data-realtime-00.csv + recordReaderConfigFileName: data/recordReaderConfig.json + tableConfigFileName: feature-test-2-realtime.json + - type: queryOp + description: Run multi-stage queries on FeatureTest1 and FeatureTest2 using SQL + useMultiStageQueryEngine: true + queryFileName: queries/feature-test-multi-stage.queries + expectedResultsFileName: query-results/feature-test-multi-stage.results diff --git a/compatibility-verifier/multi-stage-query-engine-test-suite/post-controller-rollback.yaml b/compatibility-verifier/multi-stage-query-engine-test-suite/post-controller-rollback.yaml new file mode 100644 index 000000000000..c3bcbf631c0d --- /dev/null +++ b/compatibility-verifier/multi-stage-query-engine-test-suite/post-controller-rollback.yaml @@ -0,0 +1,53 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Operations to be done. +description: Operations to be run after controller rollback +operations: + - type: segmentOp + description: Add segment FeatureTest1_Segment7 to table FeatureTest1 + op: UPLOAD + inputDataFileName: data/FeatureTest1-data-00.csv + schemaFileName: FeatureTest1-schema.json + tableConfigFileName: feature-test-1.json + recordReaderConfigFileName: data/recordReaderConfig.json + segmentName: FeatureTest1_Segment7 + - type: streamOp + description: publish rows to PinotRealtimeFeatureTest2Event + op: PRODUCE + streamConfigFileName: feature-test-2-realtime-stream-config.json + numRows: 66 + inputDataFileName: data/FeatureTest2-data-realtime-00.csv + recordReaderConfigFileName: data/recordReaderConfig.json + tableConfigFileName: feature-test-2-realtime.json + - type: queryOp + description: Run multi-stage queries on FeatureTest1 and FeatureTest2 using SQL + useMultiStageQueryEngine: true + queryFileName: queries/feature-test-multi-stage.queries + expectedResultsFileName: query-results/feature-test-multi-stage.results + - type: segmentOp + description: Delete segment FeatureTest1_Segment + op: DELETE + tableConfigFileName: feature-test-1.json + segmentName: FeatureTest1_Segment + - type: tableOp + description: Delete table feature-test-1.json + op: DELETE + schemaFileName: FeatureTest1-schema.json + tableConfigFileName: feature-test-1.json diff --git a/compatibility-verifier/multi-stage-query-engine-test-suite/post-server-rollback.yaml b/compatibility-verifier/multi-stage-query-engine-test-suite/post-server-rollback.yaml new file mode 100644 index 000000000000..c31a41294bb4 --- /dev/null +++ b/compatibility-verifier/multi-stage-query-engine-test-suite/post-server-rollback.yaml @@ -0,0 +1,43 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Operations to be done. +description: Operations to be run after server rollback +operations: + - type: segmentOp + description: Add segment FeatureTest1_Segment5 to table FeatureTest1 + op: UPLOAD + inputDataFileName: data/FeatureTest1-data-00.csv + schemaFileName: FeatureTest1-schema.json + tableConfigFileName: feature-test-1.json + recordReaderConfigFileName: data/recordReaderConfig.json + segmentName: FeatureTest1_Segment5 + - type: streamOp + description: publish rows to PinotRealtimeFeatureTest2Event + op: PRODUCE + streamConfigFileName: feature-test-2-realtime-stream-config.json + numRows: 66 + inputDataFileName: data/FeatureTest2-data-realtime-00.csv + recordReaderConfigFileName: data/recordReaderConfig.json + tableConfigFileName: feature-test-2-realtime.json + - type: queryOp + description: Run multi-stage queries on FeatureTest1 and FeatureTest2 using SQL + useMultiStageQueryEngine: true + queryFileName: queries/feature-test-multi-stage.queries + expectedResultsFileName: query-results/feature-test-multi-stage.results diff --git a/compatibility-verifier/multi-stage-query-engine-test-suite/post-server-upgrade.yaml b/compatibility-verifier/multi-stage-query-engine-test-suite/post-server-upgrade.yaml new file mode 100644 index 000000000000..786cb3fb3981 --- /dev/null +++ b/compatibility-verifier/multi-stage-query-engine-test-suite/post-server-upgrade.yaml @@ -0,0 +1,43 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Operations to be done. +description: Operations to be run after server upgrade +operations: + - type: segmentOp + description: Add segment FeatureTest1_Segment4 to table FeatureTest1 + op: UPLOAD + inputDataFileName: data/FeatureTest1-data-00.csv + schemaFileName: FeatureTest1-schema.json + tableConfigFileName: feature-test-1.json + recordReaderConfigFileName: data/recordReaderConfig.json + segmentName: FeatureTest1_Segment4 + - type: streamOp + description: publish rows to PinotRealtimeFeatureTest2Event + op: PRODUCE + streamConfigFileName: feature-test-2-realtime-stream-config.json + numRows: 66 + inputDataFileName: data/FeatureTest2-data-realtime-00.csv + recordReaderConfigFileName: data/recordReaderConfig.json + tableConfigFileName: feature-test-2-realtime.json + - type: queryOp + description: Run multi-stage queries on FeatureTest1 and FeatureTest2 using SQL + useMultiStageQueryEngine: true + queryFileName: queries/feature-test-multi-stage.queries + expectedResultsFileName: query-results/feature-test-multi-stage.results diff --git a/compatibility-verifier/multi-stage-query-engine-test-suite/pre-broker-upgrade.yaml b/compatibility-verifier/multi-stage-query-engine-test-suite/pre-broker-upgrade.yaml new file mode 100644 index 000000000000..4e255b8cf8a8 --- /dev/null +++ b/compatibility-verifier/multi-stage-query-engine-test-suite/pre-broker-upgrade.yaml @@ -0,0 +1,43 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Operations to be done. +description: Operations to be run before Broker upgrade +operations: + - type: segmentOp + description: Add segment FeatureTest1_Segment2 to table FeatureTest1 + op: UPLOAD + inputDataFileName: data/FeatureTest1-data-00.csv + schemaFileName: FeatureTest1-schema.json + tableConfigFileName: feature-test-1.json + recordReaderConfigFileName: data/recordReaderConfig.json + segmentName: FeatureTest1_Segment2 + - type: streamOp + description: publish rows to PinotRealtimeFeatureTest2Event + op: PRODUCE + streamConfigFileName: feature-test-2-realtime-stream-config.json + numRows: 66 + inputDataFileName: data/FeatureTest2-data-realtime-00.csv + recordReaderConfigFileName: data/recordReaderConfig.json + tableConfigFileName: feature-test-2-realtime.json + - type: queryOp + description: Run multi-stage queries on FeatureTest1 and FeatureTest2 using SQL + useMultiStageQueryEngine: true + queryFileName: queries/feature-test-multi-stage.queries + expectedResultsFileName: query-results/feature-test-multi-stage.results diff --git a/compatibility-verifier/multi-stage-query-engine-test-suite/pre-controller-upgrade.yaml b/compatibility-verifier/multi-stage-query-engine-test-suite/pre-controller-upgrade.yaml new file mode 100644 index 000000000000..39f8bcacdf09 --- /dev/null +++ b/compatibility-verifier/multi-stage-query-engine-test-suite/pre-controller-upgrade.yaml @@ -0,0 +1,58 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Operations to be done. +description: Operations to be run before Controller upgrade +operations: + - type: streamOp + description: create Kafka topic PinotRealtimeFeatureTest2Event + op: CREATE + streamConfigFileName: feature-test-2-realtime-stream-config.json + - type: tableOp + description: Create realtime table FeatureTest2 + op: CREATE + schemaFileName: FeatureTest2-schema.json + tableConfigFileName: feature-test-2-realtime.json + recordReaderConfigFileName: data/recordReaderConfig.json + - type: tableOp + description: Create offline table FeatureTest1 + op: CREATE + schemaFileName: FeatureTest1-schema.json + tableConfigFileName: feature-test-1.json + - type: segmentOp + description: Add segment FeatureTest1_Segment to table FeatureTest1 + op: UPLOAD + inputDataFileName: data/FeatureTest1-data-00.csv + schemaFileName: FeatureTest1-schema.json + tableConfigFileName: feature-test-1.json + recordReaderConfigFileName: data/recordReaderConfig.json + segmentName: FeatureTest1_Segment + - type: streamOp + description: publish rows to PinotRealtimeFeatureTest2Event + op: PRODUCE + streamConfigFileName: feature-test-2-realtime-stream-config.json + numRows: 66 + inputDataFileName: data/FeatureTest2-data-realtime-00.csv + recordReaderConfigFileName: data/recordReaderConfig.json + tableConfigFileName: feature-test-2-realtime.json + - type: queryOp + description: Run multi-stage queries on FeatureTest1 and FeatureTest2 using SQL + useMultiStageQueryEngine: true + queryFileName: queries/feature-test-multi-stage.queries + expectedResultsFileName: query-results/feature-test-multi-stage.results diff --git a/compatibility-verifier/multi-stage-query-engine-test-suite/pre-server-upgrade.yaml b/compatibility-verifier/multi-stage-query-engine-test-suite/pre-server-upgrade.yaml new file mode 100644 index 000000000000..167f71fadf2f --- /dev/null +++ b/compatibility-verifier/multi-stage-query-engine-test-suite/pre-server-upgrade.yaml @@ -0,0 +1,43 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Operations to be done. +description: Operations to be run before server upgrade +operations: + - type: segmentOp + description: Add segment FeatureTest1_Segment3 to table FeatureTest1 + op: UPLOAD + inputDataFileName: data/FeatureTest1-data-00.csv + schemaFileName: FeatureTest1-schema.json + tableConfigFileName: feature-test-1.json + recordReaderConfigFileName: data/recordReaderConfig.json + segmentName: FeatureTest1_Segment3 + - type: streamOp + description: publish rows to PinotRealtimeFeatureTest2Event + op: PRODUCE + streamConfigFileName: feature-test-2-realtime-stream-config.json + numRows: 66 + inputDataFileName: data/FeatureTest2-data-realtime-00.csv + recordReaderConfigFileName: data/recordReaderConfig.json + tableConfigFileName: feature-test-2-realtime.json + - type: queryOp + description: Run multi-stage queries on FeatureTest1 and FeatureTest2 using SQL + useMultiStageQueryEngine: true + queryFileName: queries/feature-test-multi-stage.queries + expectedResultsFileName: query-results/feature-test-multi-stage.results diff --git a/pinot-common/src/main/java/org/apache/pinot/common/utils/SqlResultComparator.java b/pinot-common/src/main/java/org/apache/pinot/common/utils/SqlResultComparator.java index 3120eaea3092..fac24a931a04 100644 --- a/pinot-common/src/main/java/org/apache/pinot/common/utils/SqlResultComparator.java +++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/SqlResultComparator.java @@ -72,6 +72,9 @@ private SqlResultComparator() { private static final String FIELD_NUM_ENTRIES_SCANNED_IN_FILTER = "numEntriesScannedInFilter"; private static final String FIELD_NUM_ENTRIES_SCANNED_POST_FILTER = "numEntriesScannedPostFilter"; private static final String FIELD_NUM_GROUPS_LIMIT_REACHED = "numGroupsLimitReached"; + private static final String FIELD_MULTI_STAGE_STATS = "stageStats"; + private static final String FIELD_MULTI_STAGE_STATS_TYPE = "type"; + private static final String FIELD_MULTI_STAGE_STATS_CHILDREN = "children"; private static final String FIELD_TYPE_INT = "INT"; private static final String FIELD_TYPE_LONG = "LONG"; @@ -163,6 +166,47 @@ public static boolean areEqual(JsonNode actual, JsonNode expected, String query) return areResultsEqual; } + public static boolean areMultiStageQueriesEqual(JsonNode actual, JsonNode expected, String query) + throws IOException { + if (hasExceptions(actual)) { + return false; + } + + if (areEmpty(actual, expected)) { + return true; + } + + if (!areDataSchemaEqual(actual, expected)) { + return false; + } + + ArrayNode actualRows = (ArrayNode) actual.get(FIELD_RESULT_TABLE).get(FIELD_ROWS); + ArrayNode expectedRows = (ArrayNode) expected.get(FIELD_RESULT_TABLE).get(FIELD_ROWS); + ArrayNode columnDataTypes = (ArrayNode) expected.get(FIELD_RESULT_TABLE).get(FIELD_DATA_SCHEMA). + get(FIELD_COLUMN_DATA_TYPES); + + convertNumbersToString(expectedRows, columnDataTypes); + convertNumbersToString(actualRows, columnDataTypes); + + List actualElementsSerialized = new ArrayList<>(); + List expectedElementsSerialized = new ArrayList<>(); + for (int i = 0; i < actualRows.size(); i++) { + actualElementsSerialized.add(actualRows.get(i).toString()); + } + for (int i = 0; i < expectedRows.size(); i++) { + expectedElementsSerialized.add(expectedRows.get(i).toString()); + } + + if (!areLengthsEqual(actual, expected)) { + return false; + } + + // For now, just directly compare elements in result set + return areNonOrderByQueryElementsEqual(actualElementsSerialized, expectedElementsSerialized); + + // Not verifying stage stats for now since they're still subject to change across versions + } + private static boolean areOrderByQueryElementsEqual(ArrayNode actualElements, ArrayNode expectedElements, List actualElementsSerialized, List expectedElementsSerialized, String query) { // Happy path, the results match exactly. diff --git a/pinot-compatibility-verifier/src/main/java/org/apache/pinot/compat/QueryOp.java b/pinot-compatibility-verifier/src/main/java/org/apache/pinot/compat/QueryOp.java index 931ad734d5f7..4b5836e448f5 100644 --- a/pinot-compatibility-verifier/src/main/java/org/apache/pinot/compat/QueryOp.java +++ b/pinot-compatibility-verifier/src/main/java/org/apache/pinot/compat/QueryOp.java @@ -47,6 +47,7 @@ public class QueryOp extends BaseOp { private static final String COMMENT_DELIMITER = "#"; private String _queryFileName; private String _expectedResultsFileName; + private boolean _useMultiStageQueryEngine = false; public QueryOp() { super(OpType.QUERY_OP); @@ -73,6 +74,14 @@ public void setExpectedResultsFileName(String expectedResultsFileName) { _expectedResultsFileName = expectedResultsFileName; } + public boolean getUseMultiStageQueryEngine() { + return _useMultiStageQueryEngine; + } + + public void setUseMultiStageQueryEngine(boolean useMultiStageQueryEngine) { + _useMultiStageQueryEngine = useMultiStageQueryEngine; + } + @Override boolean runOp(int generationNumber) { System.out.println("Verifying queries in " + _queryFileName + " against results in " + _expectedResultsFileName); @@ -125,7 +134,9 @@ boolean verifyQueries(int generationNumber) JsonNode actualJson = null; if (expectedJson != null) { try { - actualJson = Utils.postSqlQuery(query, ClusterDescriptor.getInstance().getBrokerUrl()); + actualJson = _useMultiStageQueryEngine + ? Utils.postMultiStageSqlQuery(query, ClusterDescriptor.getInstance().getBrokerUrl()) + : Utils.postSqlQuery(query, ClusterDescriptor.getInstance().getBrokerUrl()); } catch (Exception e) { LOGGER.error("Comparison FAILED: Line: {} Exception caught while running query: '{}', explain plan: {}", queryLineNum, query, getExplainPlan(query), e); @@ -134,7 +145,9 @@ boolean verifyQueries(int generationNumber) if (expectedJson != null && actualJson != null) { try { - boolean passed = SqlResultComparator.areEqual(actualJson, expectedJson, query); + boolean passed = _useMultiStageQueryEngine + ? SqlResultComparator.areMultiStageQueriesEqual(actualJson, expectedJson, query) + : SqlResultComparator.areEqual(actualJson, expectedJson, query); if (passed) { succeededQueryCount++; LOGGER.debug("Comparison PASSED: Line: {}, query: '{}', actual response: {}, expected response: {}", @@ -163,11 +176,18 @@ boolean verifyQueries(int generationNumber) return testPassed; } - private static String getExplainPlan(String query) { + private String getExplainPlan(String query) { try { - JsonNode explainPlanResponse = - Utils.postSqlQuery("explain plan for " + query, ClusterDescriptor.getInstance().getBrokerUrl()); - return ExplainPlanUtils.formatExplainPlan(explainPlanResponse); + if (!_useMultiStageQueryEngine) { + JsonNode explainPlanResponse = + Utils.postSqlQuery("explain plan for " + query, ClusterDescriptor.getInstance().getBrokerUrl()); + return ExplainPlanUtils.formatExplainPlan(explainPlanResponse); + } else { + JsonNode explainPlanResponse = + Utils.postMultiStageSqlQuery("explain plan for " + query, + ClusterDescriptor.getInstance().getBrokerUrl()); + return ExplainPlanUtils.formatMultiStageExplainPlan(explainPlanResponse); + } } catch (Throwable error) { return error.getMessage(); } diff --git a/pinot-compatibility-verifier/src/main/java/org/apache/pinot/compat/Utils.java b/pinot-compatibility-verifier/src/main/java/org/apache/pinot/compat/Utils.java index 86d85078ffd6..f54b8399cb88 100644 --- a/pinot-compatibility-verifier/src/main/java/org/apache/pinot/compat/Utils.java +++ b/pinot-compatibility-verifier/src/main/java/org/apache/pinot/compat/Utils.java @@ -59,4 +59,13 @@ public static JsonNode postSqlQuery(String query, String brokerBaseApiUrl) return JsonUtils.stringToJsonNode( ControllerTest.sendPostRequest(brokerBaseApiUrl + "/query/sql", payload.toString())); } + + public static JsonNode postMultiStageSqlQuery(String query, String brokerBaseApiUrl) throws Exception { + ObjectNode payload = JsonUtils.newObjectNode(); + payload.put("sql", query); + payload.put("queryOptions", "useMultistageEngine=true"); + + return JsonUtils.stringToJsonNode( + ControllerTest.sendPostRequest(brokerBaseApiUrl + "/query/sql", payload.toString())); + } } diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/utils/ExplainPlanUtils.java b/pinot-tools/src/main/java/org/apache/pinot/tools/utils/ExplainPlanUtils.java index 74da8b45a550..03c9f3b400fc 100644 --- a/pinot-tools/src/main/java/org/apache/pinot/tools/utils/ExplainPlanUtils.java +++ b/pinot-tools/src/main/java/org/apache/pinot/tools/utils/ExplainPlanUtils.java @@ -64,4 +64,9 @@ public static String formatExplainPlan(JsonNode explainPlanJson) { } return explainPlan.toString(); } + + public static String formatMultiStageExplainPlan(JsonNode explainPlanJson) { + JsonNode rows = explainPlanJson.get("resultTable").get("rows"); + return rows.get(0).get(1).asText(); + } }