From 721532c051ba02e72363f3de43c9de5b59ef4342 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 22 Jul 2024 21:04:24 +0530 Subject: [PATCH] Cleanup test changes --- .github/workflows/codeql.yml | 33 ++++++++---------- .github/workflows/distribution-checks.yml | 34 +++++++++---------- .github/workflows/revised-its.yml | 8 ----- .github/workflows/static-checks.yml | 22 ++++++------ .../unit-and-integration-tests-unified.yml | 14 +++----- integration-tests-ex/image/docker-build.sh | 7 ---- integration-tests-ex/image/docker/Dockerfile | 1 - .../image/docker/test-setup.sh | 5 ++- integration-tests-ex/image/pom.xml | 8 ----- it.sh | 16 --------- 10 files changed, 48 insertions(+), 100 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index e4cd90de85e5..7cfeb63cbff0 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -1,24 +1,21 @@ name: "CodeQL" on: - workflow_call: - -#on: -# push: -# paths-ignore: -# - '**/*.md' -# - 'dev/**' -# - 'docs/**' -# branches: [ 'master', '0.6.x', '0.7.x', '0.7.1.x', '0.7.2.x', '0.8.0', '0.8.1', '0.8.2', '0.8.3', '0.9.0', '0.9.1' ] -# pull_request: -# paths-ignore: -# - '**/*.md' -# - 'dev/**' -# - 'docs/**' -# # The branches below must be a subset of the branches above -# branches: [ 'master' ] -# schedule: -# - cron: '18 15 * * 4' + push: + paths-ignore: + - '**/*.md' + - 'dev/**' + - 'docs/**' + branches: [ 'master', '0.6.x', '0.7.x', '0.7.1.x', '0.7.2.x', '0.8.0', '0.8.1', '0.8.2', '0.8.3', '0.9.0', '0.9.1' ] + pull_request: + paths-ignore: + - '**/*.md' + - 'dev/**' + - 'docs/**' + # The branches below must be a subset of the branches above + branches: [ 'master' ] + schedule: + - cron: '18 15 * * 4' jobs: analyze: diff --git a/.github/workflows/distribution-checks.yml b/.github/workflows/distribution-checks.yml index feb188eb3d04..caadb16b608e 100644 --- a/.github/workflows/distribution-checks.yml +++ b/.github/workflows/distribution-checks.yml @@ -15,24 +15,22 @@ name: "Distribution Checks" on: - workflow_call: - -# push: -# branches: -# - master -# - '[0-9]+.[0-9]+.[0-9]+' # release branches -# - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches -# paths: -# - 'distribution/**' -# - '**/pom.xml' -# pull_request: -# branches: -# - master -# - '[0-9]+.[0-9]+.[0-9]+' # release branches -# - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches -# paths: -# - 'distribution/**' -# - '**/pom.xml' + push: + branches: + - master + - '[0-9]+.[0-9]+.[0-9]+' # release branches + - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches + paths: + - 'distribution/**' + - '**/pom.xml' + pull_request: + branches: + - master + - '[0-9]+.[0-9]+.[0-9]+' # release branches + - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches + paths: + - 'distribution/**' + - '**/pom.xml' jobs: docker-build: diff --git a/.github/workflows/revised-its.yml b/.github/workflows/revised-its.yml index 1cd144a55135..c79a6479f6d6 100644 --- a/.github/workflows/revised-its.yml +++ b/.github/workflows/revised-its.yml @@ -80,10 +80,6 @@ jobs: script: ./it.sh github ${{ matrix.it }} it: ${{ matrix.it }} mysql_driver: com.mysql.jdbc.Driver - BACKWARD_INCOMPATIBILITY_IT_ENABLED: ${{ inputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED }} - DRUID_PREVIOUS_VERSION: ${{ inputs.DRUID_PREVIOUS_VERSION }} - DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ inputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }} - DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }} s3-deep-storage-minio: needs: changes @@ -101,10 +97,6 @@ jobs: AWS_REGION: us-east-1 AWS_ACCESS_KEY_ID: admin AWS_SECRET_ACCESS_KEY: miniopassword - BACKWARD_INCOMPATIBILITY_IT_ENABLED: ${{ inputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED }} - DRUID_PREVIOUS_VERSION: ${{ inputs.DRUID_PREVIOUS_VERSION }} - DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ inputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }} - DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }} backward-compatibility-it: needs: changes diff --git a/.github/workflows/static-checks.yml b/.github/workflows/static-checks.yml index 82bef910cde8..a374cf72ccfe 100644 --- a/.github/workflows/static-checks.yml +++ b/.github/workflows/static-checks.yml @@ -15,18 +15,16 @@ name: "Static Checks CI" on: - workflow_call: - -# push: -# branches: -# - master -# - '[0-9]+.[0-9]+.[0-9]+' # release branches -# - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches -# pull_request: -# branches: -# - master -# - '[0-9]+.[0-9]+.[0-9]+' # release branches -# - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches + push: + branches: + - master + - '[0-9]+.[0-9]+.[0-9]+' # release branches + - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches + pull_request: + branches: + - master + - '[0-9]+.[0-9]+.[0-9]+' # release branches + - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches concurrency: group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.run_id }}' # group workflows only on pull_requests and not on branch commits diff --git a/.github/workflows/unit-and-integration-tests-unified.yml b/.github/workflows/unit-and-integration-tests-unified.yml index db07902feaa9..b05052f0a81c 100644 --- a/.github/workflows/unit-and-integration-tests-unified.yml +++ b/.github/workflows/unit-and-integration-tests-unified.yml @@ -58,8 +58,7 @@ jobs: strategy: fail-fast: false matrix: - #jdk: [ '8', '11', '17', '21' ] - jdK: [ '8' ] + jdk: [ '8', '11', '17', '21' ] runs-on: ubuntu-latest steps: - name: Checkout branch @@ -144,13 +143,12 @@ jobs: name: "unit tests (jdk${{ matrix.jdk }}, sql-compat=true)" uses: ./.github/workflows/unit-tests.yml needs: unit-tests - if: false #${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} + if: ${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} with: jdk: ${{ matrix.jdk }} sql_compatibility: true unit-tests: - if: false strategy: fail-fast: false matrix: @@ -164,12 +162,10 @@ jobs: standard-its: needs: unit-tests - if: false #${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} + if: ${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} uses: ./.github/workflows/standard-its.yml expose-vars: - #needs: unit-tests - needs: build runs-on: ubuntu-latest outputs: BACKWARD_INCOMPATIBILITY_IT_ENABLED: ${{ env.BACKWARD_INCOMPATIBILITY_IT_ENABLED }} @@ -180,8 +176,8 @@ jobs: - run: echo "Exposing env vars" revised-its: - needs: expose-vars - if: true #${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} + needs: [unit-tests, expose-vars] + if: ${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} uses: ./.github/workflows/revised-its.yml with: BACKWARD_INCOMPATIBILITY_IT_ENABLED: ${{ needs.expose-vars.outputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED }} diff --git a/integration-tests-ex/image/docker-build.sh b/integration-tests-ex/image/docker-build.sh index 5a4ba4ce866b..8afee10e440b 100755 --- a/integration-tests-ex/image/docker-build.sh +++ b/integration-tests-ex/image/docker-build.sh @@ -56,21 +56,14 @@ docker build -t $DRUID_IT_IMAGE_NAME \ --build-arg DRUID_TESTING_TOOLS_VERSION=$DRUID_VERSION \ . -echo "val is $BACKWARD_INCOMPATIBILITY_IT_ENABLED" if [[ -z "${BACKWARD_INCOMPATIBILITY_IT_ENABLED:-""}" || $BACKWARD_INCOMPATIBILITY_IT_ENABLED != "true" ]]; then echo "Not building previous version image." exit 0 fi -echo $DRUID_PREVIOUS_VERSION_DOWNLOAD_URL -echo $DRUID_PREVIOUS_VERSION - # Download the previous druid tar curl -L $DRUID_PREVIOUS_VERSION_DOWNLOAD_URL --output apache-druid-$DRUID_PREVIOUS_VERSION-bin.tar.gz -echo "untarring " -tar -xzf apache-druid-$DRUID_PREVIOUS_VERSION-bin.tar.gz - docker build -t $DRUID_PREVIOUS_IT_IMAGE_NAME \ --build-arg DRUID_VERSION=$DRUID_PREVIOUS_VERSION \ --build-arg MYSQL_VERSION=$MYSQL_VERSION \ diff --git a/integration-tests-ex/image/docker/Dockerfile b/integration-tests-ex/image/docker/Dockerfile index 99b2da9476b2..d76509ee406f 100644 --- a/integration-tests-ex/image/docker/Dockerfile +++ b/integration-tests-ex/image/docker/Dockerfile @@ -73,4 +73,3 @@ RUN bash /test-setup.sh USER druid:druid WORKDIR / ENTRYPOINT [ "bash", "/launch.sh" ] - diff --git a/integration-tests-ex/image/docker/test-setup.sh b/integration-tests-ex/image/docker/test-setup.sh index 9e7b0d6849da..96488ad58531 100644 --- a/integration-tests-ex/image/docker/test-setup.sh +++ b/integration-tests-ex/image/docker/test-setup.sh @@ -24,10 +24,10 @@ set -e set -u # Enable for tracing -set -x +#set -x # For debugging: verify environment -env +#env # Druid system user adduser --system --group --no-create-home druid @@ -58,7 +58,6 @@ chmod a+x /run-druid.sh # should be added in this step. cd /usr/local/ -echo "untar the zip" tar -xzf apache-druid-${DRUID_VERSION}-bin.tar.gz rm apache-druid-${DRUID_VERSION}-bin.tar.gz diff --git a/integration-tests-ex/image/pom.xml b/integration-tests-ex/image/pom.xml index 6467152819c4..600501edb92b 100644 --- a/integration-tests-ex/image/pom.xml +++ b/integration-tests-ex/image/pom.xml @@ -55,10 +55,6 @@ Reference: https://dzone.com/articles/build-docker-image-from-maven image name will typically be provided by the build enfironment, and will override this default name. --> ${project.groupId}/test:${project.version} - 5.5.1 2.7.3 5.7-debian @@ -213,10 +209,6 @@ Reference: https://dzone.com/articles/build-docker-image-from-maven ${hadoop.compile.version} ${project.version} ${druid.it.image-name} - ${project.build.directory} diff --git a/it.sh b/it.sh index e5bb35de62c8..12519c24b54d 100755 --- a/it.sh +++ b/it.sh @@ -241,15 +241,6 @@ case $CMD in mvn -B install -pl :druid-it-tools ;; "image" ) - echo "it.sh printing env vars" - #export DRUID_PREVIOUS_VERSION=30.0.0 - #export BACKWARD_INCOMPATIBILITY_IT_ENABLED=true - #export DRUID_PREVIOUS_VERSION_DOWNLOAD_URL=https://dlcdn.apache.org/druid/30.0.0/apache-druid-30.0.0-bin.tar.gz - #export DRUID_PREVIOUS_IT_IMAGE_NAME=org.apache.druid.integration-tests/test:30.0.0 - echo $BACKWARD_INCOMPATIBILITY_IT_ENABLED - echo $DRUID_PREVIOUS_VERSION - echo $DRUID_PREVIOUS_VERSION_DOWNLOAD_URL - echo $DRUID_PREVIOUS_IT_IMAGE_NAME cd $DRUID_DEV/integration-tests-ex/image mvn -B install -P test-image $MAVEN_IGNORE ;; @@ -295,17 +286,10 @@ case $CMD in tail_logs ;; "github" ) - echo "it.sh printing env vars" - echo $BACKWARD_INCOMPATIBILITY_IT_ENABLED - echo $DRUID_PREVIOUS_VERSION - echo $DRUID_PREVIOUS_VERSION_DOWNLOAD_URL - echo $DRUID_PREVIOUS_IT_IMAGE_NAME - set +e $0 test $CATEGORY RESULT=$? - # Include logs, but only for failures. if [ $RESULT -ne 0 ]; then $0 tail $CATEGORY