Merge branch 'master' into pankaj/15054

.github/dependabot.yml

@@ -4,6 +4,7 @@ updates:
     directory: "/"
     schedule:
       interval: "daily"
+    open-pull-requests-limit: 20
     ignore:
       - dependency-name: "com.google.guava:guava"
       # pin ZooKeeper dependencies to 3.5.x
@@ -18,3 +19,6 @@ updates:
       # Even then this will involve significant effort.
       # See https://github.com/apache/druid/pull/12258
       - dependency-name: "org.apache.calcite"
+      # jclouds 2.1 needs Guava 18+
+      - dependency-name: "org.apache.jclouds"
+        versions: "[2.1,)"

.github/labeler.yml

@@ -18,9 +18,50 @@
 #
 
 #
-# Pull Request Labeler Github Action Configuration: https://github.com/marketplace/actions/labeler
+# Pull Request Labeler GitHub Action Configuration: https://github.com/marketplace/actions/labeler
 
-"Area - Documentation":
-  - "docs/**/*"
-  - "website/**"
-  - "examples/quickstart/jupyter-notebooks/*"
+'Area - Batch Ingestion':
+  - 'indexing-hadoop/**'
+  - 'extensions-core/multi-stage-query/**'
+
+'Area - Dependencies':
+  - '**/pom.xml'
+  - 'licenses.yaml'
+
+'Area - Documentation':
+  - 'docs/**/*'
+  - 'website/**'
+  - 'examples/quickstart/jupyter-notebooks/**'
+
+'Area - Ingestion':
+  - 'indexing-service/**'
+
+'Area - Lookups':
+  - 'extensions-core/lookups-cached-global/**'
+  - 'extensions-core/lookups-cached-single/**'
+  - 'extensions-core/kafka-extraction-namespace/**'
+
+'Area - Metrics/Event Emitting':
+  - 'processing/src/main/java/org/apache/druid/java/util/metrics/**'
+  - 'processing/src/main/java/org/apache/druid/java/util/emitter/**'
+  - 'extensions-contrib/*-emitter/**'
+
+'Area - MSQ':
+  - 'extensions-core/multi-stage-query/**'
+
+'Area - Querying':
+  - 'sql/**'
+  - 'extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/**'
+
+'Area - Segment Format and Ser/De':
+  - 'processing/src/main/java/org/apache/druid/segment/**'
+
+'Area - Streaming Ingestion':
+  - 'extensions-core/kafka-indexing-service/**'
+  - 'extensions-core/kinesis-indexing-service/**'
+
+'Area - Web Console':
+  - 'web-console/**'
+
+'Kubernetes':
+  - 'extensions-contrib/kubernetes-overlord-extensions/**'

.github/scripts/analyze_dependencies_script.sh

@@ -15,7 +15,7 @@
 
 #!bin/bash
 
-${MVN} ${MAVEN_SKIP} dependency:analyze -DoutputXML=true -DignoreNonCompile=true -DfailOnWarning=true ${HADOOP_PROFILE} ||
+${MVN} ${MAVEN_SKIP} dependency:analyze -DoutputXML=true -DignoreNonCompile=true -DfailOnWarning=true ||
 { echo "
     The dependency analysis has found a dependency that is either:
 

.github/scripts/license_checks_script.sh

@@ -20,7 +20,7 @@ set -e
 ./.github/scripts/setup_generate_license.sh
 ${MVN} apache-rat:check -Prat --fail-at-end \
 -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
--Drat.consoleOutput=true ${HADOOP_PROFILE}
+-Drat.consoleOutput=true
 # Generate dependency reports and checks they are valid.
 mkdir -p target
 distribution/bin/generate-license-dependency-reports.py . target --clean-maven-artifact-transfer --parallel 2

.github/scripts/setup_test_profiling_env.sh

@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+
+set -e
+
+JAR_INPUT_FILE="jfr-profiler-1.0.0.jar"
+JAR_OUTPUT_FILE="jfr-profiler.jar"
+ENV_VAR="JFR_PROFILER_ARG_LINE"
+
+if [ "$#" -ne 5 ]; then
+    echo "usage: $0 <jdk_version> <run_id> <run_number> <run_attempt> <module>"
+fi
+
+if [[ "$1" == "17" ]];
+then
+  curl https://static.imply.io/cp/$JAR_INPUT_FILE -s -o $JAR_OUTPUT_FILE
+
+  echo $ENV_VAR=-javaagent:"$PWD"/$JAR_OUTPUT_FILE \
+  -Djfr.profiler.http.username=druid-ci \
+  -Djfr.profiler.http.password=w3Fb6PW8LIo849mViEkbgA== \
+  -Djfr.profiler.tags.project=druid \
+  -Djfr.profiler.tags.run_id=$2 \
+  -Djfr.profiler.tags.run_number=$3 \
+  -Djfr.profiler.tags.run_attempt=$4 \
+  -Djfr.profiler.tags.module=$5
+else
+  echo $ENV_VAR=\"\"
+fi
+
+

.github/scripts/unit_tests_script.sh

@@ -21,7 +21,8 @@ unset _JAVA_OPTIONS
 
 # Set MAVEN_OPTS for Surefire launcher.
 MAVEN_OPTS='-Xmx2500m' ${MVN} test -pl ${MAVEN_PROJECTS} \
-${MAVEN_SKIP} -Ddruid.generic.useDefaultValueForNull=${DRUID_USE_DEFAULT_VALUE_FOR_NULL}
+${MAVEN_SKIP} -Ddruid.generic.useDefaultValueForNull=${DRUID_USE_DEFAULT_VALUE_FOR_NULL} \
+-DjfrProfilerArgLine="${JFR_PROFILER_ARG_LINE}"
 sh -c "dmesg | egrep -i '(oom|out of memory|kill process|killed).*' -C 1 || exit 0"
 free -m
 ${MVN} -pl ${MAVEN_PROJECTS} jacoco:report || { echo "coverage_failure=false" >> "$GITHUB_ENV" && false; }

.github/workflows/codeql.yml

@@ -29,6 +29,12 @@ jobs:
     - name: Checkout repository
       uses: actions/checkout@v3
 
+    - uses: actions/setup-java@v3
+      with:
+        distribution: 'zulu'
+        java-version: '8'
+        cache: 'maven'
+
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v2
@@ -42,7 +48,6 @@ jobs:
         # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
         queries: +security-and-quality
 
-
     - run: |
         echo "Building using custom commands"
         mvn clean package -f "pom.xml" -B -V -e -Dfindbugs.skip -Dcheckstyle.skip -Dpmd.skip=true -Denforcer.skip -Dmaven.javadoc.skip -DskipTests -Dmaven.test.skip.exec -Dlicense.skip=true -Dweb.console.skip=true -Dcyclonedx.skip=true

.github/workflows/cron-job-its.yml

@@ -27,7 +27,7 @@ on:
 
 jobs:
   build:
-    if: github.event_name == 'schedule'
+    if: (github.event_name == 'schedule' && github.repository == 'apache/druid')
     name: build (jdk8)
     runs-on: ubuntu-latest
     steps:
@@ -107,10 +107,10 @@ jobs:
       group: other
 
   security_vulnerabilities:
+    if: github.repository == 'apache/druid'
     name: security vulnerabilities
     strategy:
-      matrix:
-        HADOOP_PROFILE: [ '', '-Phadoop3' ]
+      fail-fast: false
     runs-on: ubuntu-latest
     steps:
       - name: Checkout branch
@@ -123,12 +123,12 @@ jobs:
           distribution: 'zulu'
           cache: maven
 
+      - name: maven build # needed to rebuild incase of maven snapshot resolution fails
+        run: mvn clean install dependency:go-offline -P dist -P skip-static-checks,skip-tests -Dmaven.javadoc.skip=true -Dcyclonedx.skip=true -Dweb.console.skip=true
+
       - name: security vulnerabilities check
-        env:
-          MVN: mvn --no-snapshot-updates
-          HADOOP_PROFILE: ${{ matrix.HADOOP_PROFILE }}
         run: |
-          mvn dependency-check:purge dependency-check:check ${HADOOP_PROFILE} || { echo "
+          mvn dependency-check:purge dependency-check:check || { echo "
           The OWASP dependency check has found security vulnerabilities. Please use a newer version
           of the dependency that does not have vulnerabilities. To see a report run
           `mvn dependency-check:check`

.github/workflows/reusable-revised-its.yml

@@ -133,3 +133,35 @@ jobs:
 
       - name: Run IT
         run: ${{ inputs.script }}
+
+      - name: Collect docker logs on failure
+        if: ${{ failure() && steps.run-it.conclusion == 'failure' }}
+        run: |
+          mkdir docker-logs
+          for c in $(docker ps -a --format="{{.Names}}")
+          do
+            docker logs $c > ./docker-logs/$c.log
+          done
+
+      - name: Tar docker logs
+        if: ${{ failure() && steps.run-it.conclusion == 'failure' }}
+        run: tar cvzf ./docker-logs.tgz ./docker-logs
+
+      - name: Upload docker logs to GitHub
+        if: ${{ failure() && steps.run-it.conclusion == 'failure' }}
+        uses: actions/upload-artifact@master
+        with:
+          name: IT-${{ inputs.it }} docker logs (Compile=jdk${{ inputs.build_jdk }}, Run=jdk${{ inputs.runtime_jdk }}, Indexer=${{ inputs.use_indexer }}, Mysql=${{ inputs.mysql_driver }})
+          path: docker-logs.tgz
+
+      - name: Collect service logs on failure
+        if: ${{ failure() && steps.run-it.conclusion == 'failure' }}
+        run: |
+          tar cvzf ./service-logs.tgz ~/shared/logs
+
+      - name: Upload Druid service logs to GitHub
+        if: ${{ failure() && steps.run-it.conclusion == 'failure' }}
+        uses: actions/upload-artifact@master
+        with:
+          name: IT-${{ inputs.it }} service logs (Compile=jdk${{ inputs.build_jdk }}, Run=jdk${{ inputs.runtime_jdk }}, Indexer=${{ inputs.use_indexer }}, Mysql=${{ inputs.mysql_driver }})
+          path: service-logs.tgz

.github/workflows/reusable-standard-its.yml

@@ -88,10 +88,34 @@ jobs:
           echo "${MVN} verify -pl integration-tests -P integration-tests ${{ inputs.testing_groups }} -Djvm.runtime=${{ inputs.runtime_jdk }} -Dit.indexer=${{ inputs.use_indexer }} ${MAVEN_SKIP} -Doverride.config.path=${{ inputs.override_config_path }}"
           ${MVN} verify -pl integration-tests -P integration-tests ${{ inputs.testing_groups }} -Djvm.runtime=${{ inputs.runtime_jdk }} -Dit.indexer=${{ inputs.use_indexer }} ${MAVEN_SKIP} -Doverride.config.path=${{ inputs.override_config_path }}
 
-      - name: Debug IT
+      - name: Collect docker logs on failure
         if: ${{ failure() && steps.run-it.conclusion == 'failure' }}
         run: |
-          for v in broker router ${{ inputs.use_indexer }} historical coordinator overlord; do
-            echo "------------------------druid-"$v"-------------------------";
-            sudo docker exec druid-"$v" tail -1000 /shared/logs/"$v".log;
+          mkdir docker-logs
+          for c in $(docker ps -a --format="{{.Names}}")
+          do
+            docker logs $c > ./docker-logs/$c.log
           done
+
+      - name: Tar docker logs
+        if: ${{ failure() && steps.run-it.conclusion == 'failure' }}
+        run: tar cvzf ./docker-logs.tgz ./docker-logs
+
+      - name: Upload docker logs to GitHub
+        if: ${{ failure() && steps.run-it.conclusion == 'failure' }}
+        uses: actions/upload-artifact@master
+        with:
+          name: IT-${{ inputs.group }} docker logs (Compile=jdk${{ inputs.build_jdk }}, Run=jdk${{ inputs.runtime_jdk }}, Indexer=${{ inputs.use_indexer }}, Mysql=${{ inputs.mysql_driver }})
+          path: docker-logs.tgz
+
+      - name: Collect service logs on failure
+        if: ${{ failure() && steps.run-it.conclusion == 'failure' }}
+        run: |
+          tar cvzf ./service-logs.tgz ~/shared/logs
+
+      - name: Upload Druid service logs to GitHub
+        if: ${{ failure() && steps.run-it.conclusion == 'failure' }}
+        uses: actions/upload-artifact@master
+        with:
+          name: IT-${{ inputs.group }} service logs (Compile=jdk${{ inputs.build_jdk }}, Run=jdk${{ inputs.runtime_jdk }}, Indexer=${{ inputs.use_indexer }}, Mysql=${{ inputs.mysql_driver }})
+          path: service-logs.tgz

.github/workflows/reusable-unit-tests.yml

@@ -59,9 +59,15 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: setup jdk${{ inputs.jdk }}
-        run: echo "JAVA_HOME=$JAVA_HOME_${{ inputs.jdk }}_X64" >> $GITHUB_ENV
+      # skip the "cache: maven" step from setup-java. We explicitly use a
+      # different cache key since we cannot reuse it across commits.
+      - uses: actions/setup-java@v3
+        with:
+          distribution: 'zulu'
+          java-version: ${{ inputs.jdk }}
 
+      # the build step produces SNAPSHOT artifacts into the local maven repository,
+      # we include github.sha in the cache key to make it specific to that build/jdk
       - name: Restore Maven repository
         id: maven-restore
         uses: actions/cache/restore@v3
@@ -86,6 +92,11 @@ jobs:
             echo "DRUID_USE_DEFAULT_VALUE_FOR_NULL=true" >> $GITHUB_ENV
           fi
 
+      - name: test profiling
+        run: | 
+          ./.github/scripts/setup_test_profiling_env.sh ${{ inputs.jdk }} ${{ github.run_id }} \
+          ${{ github.run_number }} ${{ github.run_attempt }} ${{ inputs.module }} >> $GITHUB_ENV
+
       - name: fetch base branch for test coverage
         if: ${{ github.base_ref != '' }}
         run: |

.github/workflows/stale.yml

@@ -51,47 +51,8 @@ jobs:
           days-before-close: 28
           stale-issue-label: stale
           stale-pr-label: stale
-          exempt-issue-labels: |
-            Security
-            Bug
-            Proposal
-            Design Review
-            Improvement
-            Performance
-            Refactoring
-            Apache
-            Area - Automation/Static Analysis
-            Area - Batch Indexing
-            Area - Cache
-            Area - Deep Storage
-            Area - Dependencies
-            Area - Dependency Injection
-            Area - Dev
-            Area - Documentation
-            Area - Extension
-            Area - Kafka/Kinesis Indexing
-            Area - Lookups
-            Area - Metadata
-            Area - Metrics/Event Emitting
-            Area - Null Handling
-            Area - Operations
-            Area - Query UI
-            Area - Querying
-            Area - Router
-            Area - Segment Balancing/Coordination
-            Area - Segment Format and Ser/De
-            Area - SQL
-            Area - Testing
-            Area - Web Console
-            Area - Zookeeper/Curator
-            Compatibility
-            Contributions Welcome
-            Development Blocker
-            Ease of Use
-            Error handling
-            HTTP
-            Incompatible
-            Stable API
+          exempt-issue-labels: 'Evergreen,Security,Bug,Proposal,Design Review,Improvement,Performance,Refactoring,Apache,Area - Automation/Static Analysis,Area - Batch Indexing,Area - Cache,Area - Deep Storage,Area - Dependencies,Area - Dependency Injection,Area - Dev,Area - Documentation,Area - Extension,Area - Kafka/Kinesis Indexing,Area - Lookups,Area - Metadata,Area - Metrics/Event Emitting,Area - Null Handling,Area - Operations,Area - Query UI,Area - Querying,Area - Router,Area - Segment Balancing/Coordination,Area - Segment Format and Ser/De,Area - SQL,Area - Testing,Area - Web Console,Area - Zookeeper/Curator,Compatibility,Contributions Welcome,Development Blocker,Ease of Use,Error handling,HTTP,Incompatible,Stable API'
+          exempt-pr-labels: 'Evergreen'
           exempt-milestones: true
           exempt-assignees: true
           ascending: true

.github/workflows/standard-its.yml

@@ -93,11 +93,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        jdk: [8, 11]
+        jdk: [8, 17]
     uses: ./.github/workflows/reusable-standard-its.yml
     if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }}
     with:
-      build_jdk: 8
+      build_jdk: ${{ matrix.jdk }}
       runtime_jdk: ${{ matrix.jdk }}
       testing_groups: -Dgroups=query
       use_indexer: middleManager
@@ -153,12 +153,15 @@ jobs:
       - name: Setup java
         run: export JAVA_HOME=$JAVA_HOME_8_X64
 
+      # the build step produces SNAPSHOT artifacts into the local maven repository,
+      # we include github.sha in the cache key to make it specific to that build/jdk
       - name: Restore Maven repository
         id: maven-restore
         uses: actions/cache/restore@v3
         with:
           path: ~/.m2/repository
           key: maven-${{ runner.os }}-8-${{ github.sha }}
+          restore-keys: setup-java-Linux-maven-${{ hashFiles('**/pom.xml') }}
 
       - name: Maven build
         if: steps.maven-restore.outputs.cache-hit != 'true'
@@ -177,7 +180,7 @@ jobs:
         run: |
           for v in broker middlemanager router coordinator historical ; do
           echo "------------------------druid-tiny-cluster-"$v"s-0-------------------------";
-          sudo /usr/local/bin/kubectl logs --tail 1000 druid-tiny-cluster-"$v"s-0;
+          sudo /usr/local/bin/kubectl logs --tail 1000 druid-tiny-cluster-"$v"s-0 ||:;
           done
 
   integration-other-tests: