Merge branch 'master' into fix_javascript_j17

.github/dependabot.yml

@@ -4,17 +4,14 @@ updates:
     directory: "/"
     schedule:
       interval: "daily"
+    open-pull-requests-limit: 20
     ignore:
-      - dependency-name: "com.google.guava:guava"
       # pin ZooKeeper dependencies to 3.5.x
       - dependency-name: "org.apache.zookeeper"
         versions: "[3.6,)"
       # pin Jetty dependencies to 9.4.x
       - dependency-name: "org.eclipse.jetty"
         versions: "[9.5,)"
-      - dependency-name: "org.apache.hadoop"
-      # We can't upgrade calcite without shading their guava dependencies. 
-      # We can't upgrade our own guava because we have to support Hadoop 2.
-      # Even then this will involve significant effort.
-      # See https://github.com/apache/druid/pull/12258
-      - dependency-name: "org.apache.calcite"
+      # pin Testng dependencies to 7.3.0
+      - dependency-name: "org.testng"
+        versions: "[7.4.0,)"

.github/labeler.yml

@@ -18,9 +18,79 @@
 #
 
 #
-# Pull Request Labeler Github Action Configuration: https://github.com/marketplace/actions/labeler
+# Pull Request Labeler GitHub Action Configuration: https://github.com/marketplace/actions/labeler
 
-"Area - Documentation":
-  - "docs/**/*"
-  - "website/**"
-  - "examples/quickstart/jupyter-notebooks/*"
+'Area - Batch Ingestion':
+  - changed-files:
+    - any-glob-to-any-file:
+      - 'indexing-hadoop/**'
+      - 'extensions-core/multi-stage-query/**'
+
+'Area - Dependencies':
+  - changed-files:
+    - any-glob-to-any-file:
+      - '**/pom.xml'
+      - 'licenses.yaml'
+
+'Area - Documentation':
+  - changed-files:
+    - any-glob-to-any-file:
+      - 'docs/**/*'
+      - 'website/**'
+      - 'examples/quickstart/jupyter-notebooks/**'
+
+'Area - Ingestion':
+  - changed-files:
+    - any-glob-to-any-file:
+      - 'indexing-service/**'
+
+'Area - Lookups':
+  - changed-files:
+    - any-glob-to-any-file:
+      - 'extensions-core/lookups-cached-global/**'
+      - 'extensions-core/lookups-cached-single/**'
+      - 'extensions-core/kafka-extraction-namespace/**'
+
+'Area - Metrics/Event Emitting':
+  - changed-files:
+    - any-glob-to-any-file:
+      - 'processing/src/main/java/org/apache/druid/java/util/metrics/**'
+      - 'processing/src/main/java/org/apache/druid/java/util/emitter/**'
+      - 'extensions-contrib/*-emitter/**'
+
+'Area - MSQ':
+  - changed-files:
+    - any-glob-to-any-file:
+      - 'extensions-core/multi-stage-query/**'
+
+'Area - Querying':
+  - changed-files:
+    - any-glob-to-any-file:
+      - 'sql/**'
+      - 'extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/**'
+
+'Area - Segment Format and Ser/De':
+  - changed-files:
+    - any-glob-to-any-file:
+      - 'processing/src/main/java/org/apache/druid/segment/**'
+
+'Area - Streaming Ingestion':
+  - changed-files:
+    - any-glob-to-any-file:
+      - 'extensions-core/kafka-indexing-service/**'
+      - 'extensions-core/kinesis-indexing-service/**'
+
+'Area - Web Console':
+  - changed-files:
+    - any-glob-to-any-file:
+      - 'web-console/**'
+
+'Kubernetes':
+  - changed-files:
+    - any-glob-to-any-file:
+      - 'extensions-contrib/kubernetes-overlord-extensions/**'
+
+'GHA':
+  - changed-files:
+      - any-glob-to-any-file:
+          - '.github/**'

.github/scripts/setup_generate_license.sh

@@ -18,6 +18,9 @@
 set -e
 
 sudo apt-get update && sudo apt-get install python3 -y
-curl https://bootstrap.pypa.io/pip/3.5/get-pip.py | sudo -H python3
+# creating python virtual env
+python3 -m venv ~/.python3venv
+source ~/.python3venv/bin/activate
+sudo apt install python3-pip
 pip3 install wheel  # install wheel first explicitly
 pip3 install --upgrade pyyaml

.github/scripts/setup_test_profiling_env.sh

@@ -25,18 +25,26 @@ if [ "$#" -ne 5 ]; then
     echo "usage: $0 <jdk_version> <run_id> <run_number> <run_attempt> <module>"
 fi
 
-if [[ "$1" == "17" ]];
+if [[ "$1" -ge "17" ]];
 then
   curl https://static.imply.io/cp/$JAR_INPUT_FILE -s -o $JAR_OUTPUT_FILE
 
+  # Run 'java -version' and capture the output
+  output=$(java -version 2>&1)
+
+  # Extract the version number using grep and awk
+  jvm_version=$(echo "$output" | grep "version" | awk -F '"' '{print $2}')
+
+
   echo $ENV_VAR=-javaagent:"$PWD"/$JAR_OUTPUT_FILE \
   -Djfr.profiler.http.username=druid-ci \
   -Djfr.profiler.http.password=w3Fb6PW8LIo849mViEkbgA== \
   -Djfr.profiler.tags.project=druid \
   -Djfr.profiler.tags.run_id=$2 \
   -Djfr.profiler.tags.run_number=$3 \
   -Djfr.profiler.tags.run_attempt=$4 \
-  -Djfr.profiler.tags.module=$5
+  -Djfr.profiler.tags.module=$5 \
+  -Djfr.profiler.tags.jvm_version=$jvm_version
 else
   echo $ENV_VAR=\"\"
 fi

.github/scripts/unit_tests_script.sh

@@ -22,7 +22,7 @@ unset _JAVA_OPTIONS
 # Set MAVEN_OPTS for Surefire launcher.
 MAVEN_OPTS='-Xmx2500m' ${MVN} test -pl ${MAVEN_PROJECTS} \
 ${MAVEN_SKIP} -Ddruid.generic.useDefaultValueForNull=${DRUID_USE_DEFAULT_VALUE_FOR_NULL} \
--DjfrProfilerArgLine="${JFR_PROFILER_ARG_LINE}"
+-DjfrProfilerArgLine="${JFR_PROFILER_ARG_LINE}" -Pci
 sh -c "dmesg | egrep -i '(oom|out of memory|kill process|killed).*' -C 1 || exit 0"
 free -m
 ${MVN} -pl ${MAVEN_PROJECTS} jacoco:report || { echo "coverage_failure=false" >> "$GITHUB_ENV" && false; }
@@ -74,5 +74,7 @@ then
   { printf "\n\n****FAILED****\nDiff code coverage check failed. To view coverage report, run 'mvn clean test jacoco:report' and open 'target/site/jacoco/index.html'\nFor more details on how to run code coverage locally, follow instructions here - https://github.com/apache/druid/blob/master/dev/code-review/code-coverage.md#running-code-coverage-locally\n\n" && echo "coverage_failure=true" >> "$GITHUB_ENV" && false; }
 fi
 
-{ for i in 1 2 3; do curl -o codecov.sh -s https://codecov.io/bash && break || sleep 15; done }
-{ for i in 1 2 3; do bash codecov.sh -X gcov && break || sleep 15; done }
+# Commented out as codecov visualizations are currently not being used in the Druid community
+# and cause PR failures due to rate limiting (see https://github.com/apache/druid/pull/16347)
+# { for i in 1 2 3; do curl -o codecov.sh -s https://codecov.io/bash && break || sleep 15; done }
+# { for i in 1 2 3; do bash codecov.sh -X gcov && break || sleep 15; done }

.github/workflows/codeql.yml

@@ -2,8 +2,16 @@ name: "CodeQL"
 
 on:
   push:
+    paths-ignore:
+      - '**/*.md'
+      - 'dev/**'
+      - 'docs/**'
     branches: [ 'master', '0.6.x', '0.7.x', '0.7.1.x', '0.7.2.x', '0.8.0', '0.8.1', '0.8.2', '0.8.3', '0.9.0', '0.9.1' ]
   pull_request:
+    paths-ignore:
+      - '**/*.md'
+      - 'dev/**'
+      - 'docs/**'
     # The branches below must be a subset of the branches above
     branches: [ 'master' ]
   schedule:
@@ -27,22 +35,27 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
+
+    - uses: actions/setup-java@v4
+      with:
+        distribution: 'zulu'
+        java-version: '17'
+        cache: 'maven'
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
-        config-file: ./.github/workflows/codeql-config.yml
+        config-file: ./.github/config/codeql-config.yml
         # If you wish to specify custom queries, you can do so here or in a config file.
         # By default, queries listed here will override any specified in a config file.
         # Prefix the list here with "+" to use these queries and those in the config file.
 
         # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
         queries: +security-and-quality
 
-
     - run: |
         echo "Building using custom commands"
         mvn clean package -f "pom.xml" -B -V -e -Dfindbugs.skip -Dcheckstyle.skip -Dpmd.skip=true -Denforcer.skip -Dmaven.javadoc.skip -DskipTests -Dmaven.test.skip.exec -Dlicense.skip=true -Dweb.console.skip=true -Dcyclonedx.skip=true
@@ -58,6 +71,6 @@ jobs:
     #   ./location_of_script_within_repo/buildscript.sh
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3
       with:
         category: "/language:${{matrix.language}}"

.github/workflows/cron-job-its.yml

@@ -28,18 +28,21 @@ on:
 jobs:
   build:
     if: (github.event_name == 'schedule' && github.repository == 'apache/druid')
-    name: build (jdk8)
+    name: build (jdk17)
     runs-on: ubuntu-latest
     steps:
       - name: Checkout branch
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - name: Setup java
-        run: export JAVA_HOME=$JAVA_HOME_8_X64
+      - name: setup java
+        uses: actions/setup-java@v4
+        with:
+          java-version: '17'
+          distribution: 'zulu'
 
       - name: Cache Maven m2 repository
         id: maven
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.m2/repository
           key: maven-${{ runner.os }}-8-${{ github.sha }}
@@ -57,8 +60,8 @@ jobs:
     uses: ./.github/workflows/reusable-standard-its.yml
     needs: build
     with:
-      build_jdk: 8
-      runtime_jdk: 11
+      build_jdk: 17
+      runtime_jdk: 17
       testing_groups: -Dgroups=${{ matrix.testing_group }}
       use_indexer: middleManager
       group: ${{ matrix.testing_group }}
@@ -71,8 +74,8 @@ jobs:
     uses: ./.github/workflows/reusable-standard-its.yml
     needs: build
     with:
-      build_jdk: 8
-      runtime_jdk: 11
+      build_jdk: 17
+      runtime_jdk: 17
       testing_groups: -Dgroups=${{ matrix.testing_group }}
       use_indexer: indexer
       group: ${{ matrix.testing_group }}
@@ -85,8 +88,8 @@ jobs:
     uses: ./.github/workflows/reusable-standard-its.yml
     needs: build
     with:
-      build_jdk: 8
-      runtime_jdk: 11
+      build_jdk: 17
+      runtime_jdk: 17
       testing_groups: -Dgroups=${{ matrix.testing_group }}
       use_indexer: middleManager
       override_config_path: ./environment-configs/test-groups/prepopulated-data
@@ -100,8 +103,8 @@ jobs:
     uses: ./.github/workflows/reusable-standard-its.yml
     needs: build
     with:
-      build_jdk: 8
-      runtime_jdk: 11
+      build_jdk: 17
+      runtime_jdk: 17
       testing_groups: -DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability,upgrade,shuffle-deep-store,custom-coordinator-duties
       use_indexer: ${{ matrix.indexer }}
       group: other
@@ -114,12 +117,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout branch
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: setup java
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
-          java-version: '8'
+          java-version: '17'
           distribution: 'zulu'
           cache: maven
 

.github/workflows/distribution-checks.yml

@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "Distribution Checks"
+on:
+  push:
+    branches:
+      - master
+      - '[0-9]+.[0-9]+.[0-9]+' # release branches
+      - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches
+    paths:
+      - 'distribution/**'
+      - '**/pom.xml'
+  pull_request:
+    branches:
+      - master
+      - '[0-9]+.[0-9]+.[0-9]+' # release branches
+      - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches
+    paths:
+      - 'distribution/**'
+      - '**/pom.xml'
+
+jobs:
+  docker-build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build the Docker image
+        run: DOCKER_BUILDKIT=1 docker build -t apache/druid:tag -f distribution/docker/Dockerfile .

.github/workflows/labeler.yml

@@ -32,6 +32,6 @@ jobs:
       pull-requests: write
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/labeler@v4
+    - uses: actions/labeler@v5
       with:
         repo-token: "${{ secrets.GITHUB_TOKEN }}"