Merge remote-tracking branch 'apache/master'

.github/scripts/setup_generate_license.sh

@@ -18,6 +18,9 @@
 set -e
 
 sudo apt-get update && sudo apt-get install python3 -y
-curl https://bootstrap.pypa.io/pip/3.5/get-pip.py | sudo -H python3
+# creating python virtual env
+python3 -m venv ~/.python3venv
+source ~/.python3venv/bin/activate
+sudo apt install python3-pip
 pip3 install wheel  # install wheel first explicitly
 pip3 install --upgrade pyyaml

.github/workflows/reusable-revised-its.yml

@@ -57,6 +57,19 @@ on:
       AWS_SECRET_ACCESS_KEY:
         required: false
         type: string
+      BACKWARD_COMPATIBILITY_IT_ENABLED:
+        required: false
+        type: string
+        default: false
+      DRUID_PREVIOUS_VERSION:
+        required: false
+        type: string
+      DRUID_PREVIOUS_VERSION_DOWNLOAD_URL:
+        required: false
+        type: string
+      DRUID_PREVIOUS_IT_IMAGE_NAME:
+        required: false
+        type: string
 
 env:
   MYSQL_DRIVER_CLASSNAME: ${{ inputs.mysql_driver }} # Used by tests to connect to metadata store directly.
@@ -106,6 +119,15 @@ jobs:
             ./druid-container-jdk${{ inputs.build_jdk }}.tar.gz
             ./integration-tests-ex/image/target/env.sh
 
+      - name: Retrieve previous version cached docker image
+        id: docker-restore-previous-version
+        if: ${{ inputs.BACKWARD_COMPATIBILITY_IT_ENABLED == 'true' }}
+        uses: actions/cache/restore@v4
+        with:
+          key: druid-container-jdk${{ inputs.build_jdk }}-version${{ inputs.DRUID_PREVIOUS_VERSION }}.tar.gz-${{ github.sha }}
+          path: |
+            ./druid-container-jdk${{ inputs.build_jdk }}-version${{ inputs.DRUID_PREVIOUS_VERSION }}.tar.gz           
+
       - name: Maven build
         if: steps.maven-restore.outputs.cache-hit != 'true' || ( steps.docker-restore.outputs.cache-hit != 'true' && steps.targets-restore.outputs.cache-hit != 'true' )
         run: |
@@ -115,13 +137,26 @@ jobs:
         if: steps.docker-restore.outputs.cache-hit != 'true' || steps.maven-restore.outputs.cache-hit != 'true'
         env:
           docker-restore: ${{ toJson(steps.docker-restore.outputs) }}
+          BACKWARD_COMPATIBILITY_IT_ENABLED: ${{ inputs.BACKWARD_COMPATIBILITY_IT_ENABLED }}
+          DRUID_PREVIOUS_VERSION: ${{ inputs.DRUID_PREVIOUS_VERSION }}
+          DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ inputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }}
+          DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}
         run: |
           ./it.sh image
           source ./integration-tests-ex/image/target/env.sh
           docker tag $DRUID_IT_IMAGE_NAME $DRUID_IT_IMAGE_NAME-jdk${{ inputs.build_jdk }}
           echo $DRUID_IT_IMAGE_NAME
           docker save "$DRUID_IT_IMAGE_NAME" | gzip > druid-container-jdk${{ inputs.build_jdk }}.tar.gz
 
+      - name: Save previous version docker image
+        if: ${{ inputs.BACKWARD_COMPATIBILITY_IT_ENABLED == 'true' && (steps.docker-restore.outputs.cache-hit != 'true' || steps.maven-restore.outputs.cache-hit != 'true') }}
+        env:
+          docker-restore: ${{ toJson(steps.docker-restore.outputs) }}
+        run: |
+          docker tag ${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }} ${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}-jdk${{ inputs.build_jdk }}-version${{ inputs.DRUID_PREVIOUS_VERSION }}
+          echo ${DRUID_PREVIOUS_IT_IMAGE_NAME}
+          docker save "${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}" | gzip > druid-container-jdk${{ inputs.build_jdk }}-version${{ inputs.DRUID_PREVIOUS_VERSION }}.tar.gz
+
       - name: Stop and remove docker containers
         run: |
           echo "Force stopping all containers and pruning"
@@ -133,9 +168,21 @@ jobs:
           docker load --input druid-container-jdk${{ inputs.build_jdk }}.tar.gz
           docker images
 
+      - name: Load previous version docker image
+        if: ${{ inputs.BACKWARD_COMPATIBILITY_IT_ENABLED == 'true' }}
+        run: |
+          docker load --input druid-container-jdk${{ inputs.build_jdk }}-version${{ inputs.DRUID_PREVIOUS_VERSION }}.tar.gz
+          docker images
+
       - name: Run IT
         id: run-it
-        run: ${{ inputs.script }}
+        env:
+          BACKWARD_COMPATIBILITY_IT_ENABLED: ${{ inputs.BACKWARD_COMPATIBILITY_IT_ENABLED }}
+          DRUID_PREVIOUS_VERSION: ${{ inputs.DRUID_PREVIOUS_VERSION }}
+          DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ inputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }}
+          DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}
+        run: |
+          ${{ inputs.script }}
 
       - name: Collect docker logs on failure
         if: ${{ failure() && steps.run-it.conclusion == 'failure' }}

.github/workflows/reusable-unit-tests.yml

@@ -117,6 +117,31 @@ jobs:
           MAVEN_PROJECTS: ${{ inputs.maven_projects }}
         run: ./.github/scripts/unit_tests_script.sh
 
+      - name: Check for dumps on failure
+        if: ${{ failure() }}
+        id: check_for_dumps
+        run: |
+          if test -n "$(find "${GITHUB_WORKSPACE}" \( -name '*.hprof' -or -name 'hs_err_pid*' -or -name 'replay_pid*' -or -regex '.*/core\.[0-9]*' \))"
+          then
+            echo "found_dumps=true" >> "$GITHUB_ENV"
+          else
+            echo "found_dumps=false" >> "$GITHUB_ENV"
+          fi
+
+      - name: Collect dumps if they exist on failure
+        if: ${{ failure() && env.found_dumps == 'true' }}
+        run: |
+          find "${GITHUB_WORKSPACE}" \
+            \( -name '*.hprof' -or -name 'hs_err_pid*' -or -name 'replay_pid*' -or -regex '.*/core\.[0-9]*' \) \
+            -exec tar -cvzf ${RUNNER_TEMP}/failure-dumps.tar.gz {} +
+
+      - name: Upload dumps to GitHub if they exist on failure
+        if: ${{ failure() && env.found_dumps == 'true' }}
+        uses: actions/upload-artifact@master
+        with:
+          name: Failure-${{ inputs.group }} failure dumps (Compile=jdk${{ inputs.build_jdk }}, Run=jdk${{ inputs.runtime_jdk }})
+          path: ${{ runner.temp }}/failure-dumps.tar.gz
+
       - name: set outputs on failure
         id: set_outputs
         if: ${{ failure() }}

.github/workflows/revised-its.yml

@@ -18,6 +18,24 @@
 name: "Revised ITs workflow"
 on:
   workflow_call:
+    inputs:
+      BACKWARD_COMPATIBILITY_IT_ENABLED:
+        description: "Flag for backward compatibility IT"
+        required: false
+        default: false
+        type: string
+      DRUID_PREVIOUS_VERSION:
+        description: "Previous druid versions to run the test against."
+        required: false
+        type: string
+      DRUID_PREVIOUS_VERSION_DOWNLOAD_URL:
+        description: "URL to download the previous druid version."
+        required: false
+        type: string
+      DRUID_PREVIOUS_IT_IMAGE_NAME:
+        description: "Druid previous version image name."
+        required: false
+        type: string
   workflow_dispatch:
 
 jobs:
@@ -79,3 +97,24 @@ jobs:
       AWS_REGION: us-east-1
       AWS_ACCESS_KEY_ID: admin
       AWS_SECRET_ACCESS_KEY: miniopassword
+
+  backward-compatibility-it:
+    needs: changes
+    uses: ./.github/workflows/reusable-revised-its.yml
+    if: ${{ inputs.BACKWARD_COMPATIBILITY_IT_ENABLED == 'true' && (needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true') }}
+    with:
+      build_jdk: 8
+      runtime_jdk: 8
+      use_indexer: middleManager
+      script: ./it.sh github BackwardCompatibilityMain
+      it: BackwardCompatibilityMain
+      mysql_driver: com.mysql.jdbc.Driver
+      BACKWARD_COMPATIBILITY_IT_ENABLED: ${{ inputs.BACKWARD_COMPATIBILITY_IT_ENABLED }}
+      DRUID_PREVIOUS_VERSION: ${{ inputs.DRUID_PREVIOUS_VERSION }}
+      DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ inputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }}
+      DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}
+      DRUID_CLOUD_BUCKET: druid-qa
+      DRUID_CLOUD_PATH: aws-${{ github.run_id }}-${{ github.run_attempt }}
+      AWS_REGION: us-east-1
+      AWS_ACCESS_KEY_ID: admin
+      AWS_SECRET_ACCESS_KEY: miniopassword

.github/workflows/unit-and-integration-tests-unified.yml

@@ -19,6 +19,7 @@ on:
     paths-ignore:
       - '**/*.md'
       - 'dev/**'
+      - 'distribution/bin/**'
       - 'docs/**'
       - 'examples/**/jupyter-notebooks/**'
       - 'web-console/**'
@@ -31,6 +32,7 @@ on:
     paths-ignore:
       - '**/*.md'
       - 'dev/**'
+      - 'distribution/bin/**'
       - 'docs/**'
       - 'examples/**/jupyter-notebooks/**'
       - 'web-console/**'
@@ -49,7 +51,30 @@ env:
   SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
 
 jobs:
+  set-env-var:
+    name: Set env var
+    runs-on: ubuntu-latest
+    outputs:
+      DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ steps.image_name.outputs.image_name }}
+      BACKWARD_COMPATIBILITY_IT_ENABLED: ${{ steps.it_enabled.outputs.enabled }}
+      DRUID_PREVIOUS_VERSION: ${{ env.DRUID_PREVIOUS_VERSION }}
+      DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ env.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }}
+    steps:
+      - name: Set image name env var
+        id: image_name
+        run: |
+          echo "::set-output name=image_name::org.apache.druid.integration-tests/test:${{ env.DRUID_PREVIOUS_VERSION }}"
+      - name: Set env for enabling backward compatibility it
+        id: it_enabled
+        run: |
+          if [ -n "${{ env.DRUID_PREVIOUS_VERSION }}" ]; then
+            echo "::set-output name=enabled::true"
+          else
+            echo "::set-output name=enabled::false"
+          fi
+
   build:
+    needs: set-env-var
     name: "build (jdk${{ matrix.jdk }})"
     strategy:
       fail-fast: false
@@ -94,12 +119,25 @@ jobs:
             ./druid-container-jdk${{ matrix.jdk }}.tar.gz
             ./integration-tests-ex/image/target/env.sh
 
+      - name: Cache previous version image
+        id: docker_container_previous_version
+        uses: actions/cache@v4
+        with:
+          key: druid-container-jdk${{ matrix.jdk }}-version${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION }}.tar.gz-${{ github.sha }}
+          path: |
+            ./druid-container-jdk${{ matrix.jdk }}-version${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION }}.tar.gz
+
       - name: Maven build
         id: maven_build
         run: |
           ./it.sh ci
 
       - name: Container build
+        env:
+          BACKWARD_COMPATIBILITY_IT_ENABLED: ${{ needs.set-env-var.outputs.BACKWARD_COMPATIBILITY_IT_ENABLED }}
+          DRUID_PREVIOUS_VERSION: ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION }}
+          DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }}
+          DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}
         run: |
           ./it.sh image
           source ./integration-tests-ex/image/target/env.sh
@@ -111,6 +149,13 @@ jobs:
           echo $DRUID_IT_IMAGE_NAME
           docker save "$DRUID_IT_IMAGE_NAME" | gzip > druid-container-jdk${{ matrix.jdk }}.tar.gz
 
+      - name: Save previous version docker image
+        if: ${{ needs.set-env-var.outputs.BACKWARD_COMPATIBILITY_IT_ENABLED == 'true' }}
+        run: |
+          docker tag ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_IT_IMAGE_NAME }} ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}-jdk${{ matrix.jdk }}-version${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION }}
+          echo ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}
+          docker save "${{ needs.set-env-var.outputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}" | gzip > druid-container-jdk${{ matrix.jdk }}-version${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION }}.tar.gz    
+
   unit-tests-phase2:
     strategy:
       fail-fast: false
@@ -142,6 +187,11 @@ jobs:
     uses: ./.github/workflows/standard-its.yml
 
   revised-its:
-    needs: unit-tests
+    needs: [unit-tests, set-env-var]
     if: ${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }}
     uses: ./.github/workflows/revised-its.yml
+    with:
+      BACKWARD_COMPATIBILITY_IT_ENABLED: ${{ needs.set-env-var.outputs.BACKWARD_COMPATIBILITY_IT_ENABLED }}
+      DRUID_PREVIOUS_VERSION: ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION }}
+      DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }}
+      DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}

LABELS

@@ -55,11 +55,6 @@ This product bundles Java Servlet API version 2.5, copyright Oracle and/or its a
   * https://github.com/javaee/servlet-spec
   * javax.servlet:javax.servlet-api
 
-This product bundles JAXB version 2.2.2, copyright Oracle and/or its affiliates.,
- which is available under the CDDL 1.1. For details, see licenses/bin/javax.CDDL11
-  * https://github.com/javaee/jaxb-v2
-  * javax.xml.bind:jaxb-api
-
 This product bundles stax-api version 1.0-2, copyright Oracle and/or its affiliates.,
  which is available under the CDDL 1.1. For details, see licenses/bin/javax.CDDL11
   * https://github.com/javaee/

benchmarks/pom.xml

@@ -27,7 +27,7 @@
   <parent>
     <groupId>org.apache.druid</groupId>
     <artifactId>druid</artifactId>
-    <version>31.0.0-SNAPSHOT</version>
+    <version>32.0.0-SNAPSHOT</version>
   </parent>
 
   <dependencies>
@@ -47,6 +47,16 @@
       <version>${jmh.version}</version>
       <scope>provided</scope>
     </dependency>
+    <dependency>
+      <groupId>org.reflections</groupId>
+      <artifactId>reflections</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-core</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.easymock</groupId>
       <artifactId>easymock</artifactId>
@@ -217,6 +227,13 @@
       <version>${project.parent.version}</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.druid.extensions</groupId>
+      <artifactId>druid-multi-stage-query</artifactId>
+      <version>${project.parent.version}</version>
+      <classifier>tests</classifier>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <properties>

benchmarks/src/test/java/org/apache/druid/benchmark/DataSourcesSnapshotBenchmark.java

@@ -41,9 +41,7 @@
 
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
 @State(Scope.Benchmark)
@@ -54,7 +52,7 @@
 @Measurement(iterations = 50)
 public class DataSourcesSnapshotBenchmark
 {
-  private static Interval TEST_SEGMENT_INTERVAL = Intervals.of("2012-03-15T00:00:00.000/2012-03-16T00:00:00.000");
+  private static final Interval TEST_SEGMENT_INTERVAL = Intervals.of("2012-03-15T00:00:00.000/2012-03-16T00:00:00.000");
 
   @Param({"500", "1000"})
   private int numDataSources;
@@ -69,11 +67,10 @@ public void setUp()
   {
     long start = System.currentTimeMillis();
 
-    Map<String, ImmutableDruidDataSource> dataSources = new HashMap<>();
+    final List<DataSegment> segments = new ArrayList<>();
 
     for (int i = 0; i < numDataSources; i++) {
       String dataSource = StringUtils.format("ds-%d", i);
-      List<DataSegment> segments = new ArrayList<>();
 
       for (int j = 0; j < numSegmentPerDataSource; j++) {
         segments.add(
@@ -90,11 +87,9 @@ public void setUp()
             )
         );
       }
-
-      dataSources.put(dataSource, new ImmutableDruidDataSource(dataSource, Collections.emptyMap(), segments));
     }
 
-    snapshot = new DataSourcesSnapshot(dataSources);
+    snapshot = DataSourcesSnapshot.fromUsedSegments(segments);
 
     System.out.println("Setup Time " + (System.currentTimeMillis() - start) + " ms");
   }

benchmarks/src/test/java/org/apache/druid/benchmark/DelimitedInputFormatBenchmark.java

@@ -132,7 +132,7 @@ public void prepareData() throws Exception
   @Setup(Level.Trial)
   public void prepareFormat()
   {
-    format = new DelimitedInputFormat(fromHeader ? null : COLUMNS, null, "\t", null, fromHeader, fromHeader ? 0 : 1);
+    format = new DelimitedInputFormat(fromHeader ? null : COLUMNS, null, "\t", null, fromHeader, fromHeader ? 0 : 1, null);
   }
 
   @Benchmark