From 29f2d99bd7bb6a14c68eba63f8df3777059ef055 Mon Sep 17 00:00:00 2001 From: YanxuanLiu Date: Fri, 10 Nov 2023 16:01:58 +0800 Subject: [PATCH 1/7] add cache for testing Signed-off-by: YanxuanLiu --- .github/workflows/mvn-verify-check.yml | 270 +++++++++++++------------ 1 file changed, 139 insertions(+), 131 deletions(-) diff --git a/.github/workflows/mvn-verify-check.yml b/.github/workflows/mvn-verify-check.yml index 0e9d7899502..19791e63573 100644 --- a/.github/workflows/mvn-verify-check.yml +++ b/.github/workflows/mvn-verify-check.yml @@ -117,6 +117,14 @@ jobs: env | grep JAVA java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH" + - name: Cache local Maven repository + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + - name: package tests check run: | # https://github.com/NVIDIA/spark-rapids/issues/8847 @@ -141,147 +149,147 @@ jobs: } done - package-tests-scala213: - needs: get-shim-versions-from-dist - continue-on-error: ${{ matrix.isSnapshot }} - strategy: - matrix: ${{ fromJSON(needs.get-shim-versions-from-dist.outputs.scala213Versions) }} - fail-fast: false - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 # refs/pull/:prNumber/merge + # package-tests-scala213: + # needs: get-shim-versions-from-dist + # continue-on-error: ${{ matrix.isSnapshot }} + # strategy: + # matrix: ${{ fromJSON(needs.get-shim-versions-from-dist.outputs.scala213Versions) }} + # fail-fast: false + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v3 # refs/pull/:prNumber/merge - - name: Setup Java and Maven Env - uses: actions/setup-java@v3 - with: - distribution: adopt - java-version: 8 + # - name: Setup Java and Maven Env + # uses: actions/setup-java@v3 + # with: + # distribution: adopt + # java-version: 8 - - name: check runtime before tests - run: | - env | grep JAVA - java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH" + # - name: check runtime before tests + # run: | + # env | grep JAVA + # java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH" - - name: package tests check - run: | - # https://github.com/NVIDIA/spark-rapids/issues/8847 - # specify expected versions - export JAVA_HOME=${JAVA_HOME_8_X64} - export PATH=${JAVA_HOME}/bin:${PATH} - java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH" - # verify Scala 2.13 build files - ./build/make-scala-version-build-files.sh 2.13 - # verify git status - if [ -n "$(echo -n $(git status -s | grep 'scala2.13'))" ]; then - git add -N scala2.13/* && git diff 'scala2.13/*' - echo "Generated Scala 2.13 build files don't match what's in repository" - exit 1 - fi - # change to Scala 2.13 Directory - cd scala2.13 - # test command, will retry for 3 times if failed. - max_retry=3; delay=30; i=1 - while true; do - mvn -Dmaven.wagon.http.retryHandler.count=3 \ - -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 \ - -Daether.connector.http.connectionMaxTtl=30 -B package \ - -pl integration_tests,tests -am -P 'individual,pre-merge' \ - -Dbuildver=${{ matrix.spark-version }} -Dmaven.scalastyle.skip=true \ - -Drat.skip=true ${{ env.COMMON_MVN_FLAGS }} && break || { - if [[ $i -le $max_retry ]]; then - echo "mvn command failed. Retry $i/$max_retry."; ((i++)); sleep $delay; ((delay=delay*2)) - else - echo "mvn command failed. Exit 1"; exit 1 - fi - } - done + # - name: package tests check + # run: | + # # https://github.com/NVIDIA/spark-rapids/issues/8847 + # # specify expected versions + # export JAVA_HOME=${JAVA_HOME_8_X64} + # export PATH=${JAVA_HOME}/bin:${PATH} + # java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH" + # # verify Scala 2.13 build files + # ./build/make-scala-version-build-files.sh 2.13 + # # verify git status + # if [ -n "$(echo -n $(git status -s | grep 'scala2.13'))" ]; then + # git add -N scala2.13/* && git diff 'scala2.13/*' + # echo "Generated Scala 2.13 build files don't match what's in repository" + # exit 1 + # fi + # # change to Scala 2.13 Directory + # cd scala2.13 + # # test command, will retry for 3 times if failed. + # max_retry=3; delay=30; i=1 + # while true; do + # mvn -Dmaven.wagon.http.retryHandler.count=3 \ + # -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 \ + # -Daether.connector.http.connectionMaxTtl=30 -B package \ + # -pl integration_tests,tests -am -P 'individual,pre-merge' \ + # -Dbuildver=${{ matrix.spark-version }} -Dmaven.scalastyle.skip=true \ + # -Drat.skip=true ${{ env.COMMON_MVN_FLAGS }} && break || { + # if [[ $i -le $max_retry ]]; then + # echo "mvn command failed. Retry $i/$max_retry."; ((i++)); sleep $delay; ((delay=delay*2)) + # else + # echo "mvn command failed. Exit 1"; exit 1 + # fi + # } + # done - verify-all-modules: - needs: get-shim-versions-from-dist - runs-on: ubuntu-latest - strategy: - matrix: ${{ fromJSON(needs.get-shim-versions-from-dist.outputs.sparkJDKVersions) }} - steps: - - uses: actions/checkout@v3 # refs/pull/:prNumber/merge + # verify-all-modules: + # needs: get-shim-versions-from-dist + # runs-on: ubuntu-latest + # strategy: + # matrix: ${{ fromJSON(needs.get-shim-versions-from-dist.outputs.sparkJDKVersions) }} + # steps: + # - uses: actions/checkout@v3 # refs/pull/:prNumber/merge - - name: Setup Java and Maven Env - uses: actions/setup-java@v3 - with: - distribution: adopt - java-version: ${{ matrix.java-version }} + # - name: Setup Java and Maven Env + # uses: actions/setup-java@v3 + # with: + # distribution: adopt + # java-version: ${{ matrix.java-version }} - - name: check runtime before tests - run: | - env | grep JAVA - java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH" + # - name: check runtime before tests + # run: | + # env | grep JAVA + # java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH" - - name: Build JDK - run: | - # https://github.com/NVIDIA/spark-rapids/issues/8847 - # specify expected versions - export JAVA_HOME=${JAVA_HOME_${{ matrix.java-version }}_X64} - export PATH=${JAVA_HOME}/bin:${PATH} - java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH" - # test command, will retry for 3 times if failed. - max_retry=3; delay=30; i=1 - while true; do - mvn -Dmaven.wagon.http.retryHandler.count=$max_retry \ - -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 \ - -Daether.connector.http.connectionMaxTtl=30 -B verify \ - -P "individual,pre-merge" -Dbuildver=${{ matrix.spark-version }} \ - ${{ env.COMMON_MVN_FLAGS }} && break || { - if [[ $i -le $max_retry ]]; then - echo "mvn command failed. Retry $i/$max_retry."; ((i++)); sleep $delay; ((delay=delay*2)) - else - echo "mvn command failed. Exit 1"; exit 1 - fi - } - done + # - name: Build JDK + # run: | + # # https://github.com/NVIDIA/spark-rapids/issues/8847 + # # specify expected versions + # export JAVA_HOME=${JAVA_HOME_${{ matrix.java-version }}_X64} + # export PATH=${JAVA_HOME}/bin:${PATH} + # java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH" + # # test command, will retry for 3 times if failed. + # max_retry=3; delay=30; i=1 + # while true; do + # mvn -Dmaven.wagon.http.retryHandler.count=$max_retry \ + # -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 \ + # -Daether.connector.http.connectionMaxTtl=30 -B verify \ + # -P "individual,pre-merge" -Dbuildver=${{ matrix.spark-version }} \ + # ${{ env.COMMON_MVN_FLAGS }} && break || { + # if [[ $i -le $max_retry ]]; then + # echo "mvn command failed. Retry $i/$max_retry."; ((i++)); sleep $delay; ((delay=delay*2)) + # else + # echo "mvn command failed. Exit 1"; exit 1 + # fi + # } + # done - install-modules: - needs: get-shim-versions-from-dist - runs-on: ubuntu-latest - strategy: - matrix: - maven-version: [3.6.3, 3.8.8, 3.9.3] - steps: - - uses: actions/checkout@v3 # refs/pull/:prNumber/merge + # install-modules: + # needs: get-shim-versions-from-dist + # runs-on: ubuntu-latest + # strategy: + # matrix: + # maven-version: [3.6.3, 3.8.8, 3.9.3] + # steps: + # - uses: actions/checkout@v3 # refs/pull/:prNumber/merge - - name: Setup Java - uses: actions/setup-java@v3 - with: - distribution: adopt - java-version: 11 + # - name: Setup Java + # uses: actions/setup-java@v3 + # with: + # distribution: adopt + # java-version: 11 - - name: Setup Maven Wrapper - run: mvn wrapper:wrapper -Dmaven=${{ matrix.maven-version }} + # - name: Setup Maven Wrapper + # run: mvn wrapper:wrapper -Dmaven=${{ matrix.maven-version }} - - name: check runtime before tests - run: | - env | grep JAVA - java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH" + # - name: check runtime before tests + # run: | + # env | grep JAVA + # java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH" - - name: Install with Maven ${{ matrix.maven-version }} - run: | - # https://github.com/NVIDIA/spark-rapids/issues/8847 - # specify expected versions - export JAVA_HOME=${JAVA_HOME_11_X64} - export PATH=${JAVA_HOME}/bin:${PATH} - java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH" - # test command, will retry for 3 times if failed. - max_retry=3; delay=30; i=1 - while true; do - ./mvnw -Dmaven.wagon.http.retryHandler.count=3 \ - -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 \ - -Daether.connector.http.connectionMaxTtl=30 -B install \ - -P "individual,pre-merge" \ - -Dbuildver=${{ needs.get-shim-versions-from-dist.outputs.defaultSparkVersion }} \ - ${{ env.COMMON_MVN_FLAGS }} && break || { - if [[ $i -le $max_retry ]]; then - echo "mvn command failed. Retry $i/$max_retry."; ((i++)); sleep $delay; ((delay=delay*2)) - else - echo "mvn command failed. Exit 1"; exit 1 - fi - } - done + # - name: Install with Maven ${{ matrix.maven-version }} + # run: | + # # https://github.com/NVIDIA/spark-rapids/issues/8847 + # # specify expected versions + # export JAVA_HOME=${JAVA_HOME_11_X64} + # export PATH=${JAVA_HOME}/bin:${PATH} + # java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH" + # # test command, will retry for 3 times if failed. + # max_retry=3; delay=30; i=1 + # while true; do + # ./mvnw -Dmaven.wagon.http.retryHandler.count=3 \ + # -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 \ + # -Daether.connector.http.connectionMaxTtl=30 -B install \ + # -P "individual,pre-merge" \ + # -Dbuildver=${{ needs.get-shim-versions-from-dist.outputs.defaultSparkVersion }} \ + # ${{ env.COMMON_MVN_FLAGS }} && break || { + # if [[ $i -le $max_retry ]]; then + # echo "mvn command failed. Retry $i/$max_retry."; ((i++)); sleep $delay; ((delay=delay*2)) + # else + # echo "mvn command failed. Exit 1"; exit 1 + # fi + # } + # done From 40e34ed90fec4c30c62226fc3ae08ad927b1205d Mon Sep 17 00:00:00 2001 From: YanxuanLiu Date: Fri, 10 Nov 2023 16:34:34 +0800 Subject: [PATCH 2/7] change cache key file Signed-off-by: YanxuanLiu --- .github/workflows/mvn-verify-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mvn-verify-check.yml b/.github/workflows/mvn-verify-check.yml index 19791e63573..66166138ae2 100644 --- a/.github/workflows/mvn-verify-check.yml +++ b/.github/workflows/mvn-verify-check.yml @@ -121,7 +121,7 @@ jobs: uses: actions/cache@v3 with: path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + key: ${{ runner.os }}-maven-${{ hashFiles('pom.xml') }} restore-keys: | ${{ runner.os }}-maven- From e472cbc4f9b659c1d69e7bfae79ac779042557d1 Mon Sep 17 00:00:00 2001 From: YanxuanLiu Date: Fri, 10 Nov 2023 16:54:10 +0800 Subject: [PATCH 3/7] add cache for all stages Signed-off-by: YanxuanLiu --- .github/workflows/mvn-verify-check.yml | 312 +++++++++++++------------ 1 file changed, 168 insertions(+), 144 deletions(-) diff --git a/.github/workflows/mvn-verify-check.yml b/.github/workflows/mvn-verify-check.yml index 66166138ae2..e10d9d1d195 100644 --- a/.github/workflows/mvn-verify-check.yml +++ b/.github/workflows/mvn-verify-check.yml @@ -149,147 +149,171 @@ jobs: } done - # package-tests-scala213: - # needs: get-shim-versions-from-dist - # continue-on-error: ${{ matrix.isSnapshot }} - # strategy: - # matrix: ${{ fromJSON(needs.get-shim-versions-from-dist.outputs.scala213Versions) }} - # fail-fast: false - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v3 # refs/pull/:prNumber/merge - - # - name: Setup Java and Maven Env - # uses: actions/setup-java@v3 - # with: - # distribution: adopt - # java-version: 8 - - # - name: check runtime before tests - # run: | - # env | grep JAVA - # java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH" - - # - name: package tests check - # run: | - # # https://github.com/NVIDIA/spark-rapids/issues/8847 - # # specify expected versions - # export JAVA_HOME=${JAVA_HOME_8_X64} - # export PATH=${JAVA_HOME}/bin:${PATH} - # java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH" - # # verify Scala 2.13 build files - # ./build/make-scala-version-build-files.sh 2.13 - # # verify git status - # if [ -n "$(echo -n $(git status -s | grep 'scala2.13'))" ]; then - # git add -N scala2.13/* && git diff 'scala2.13/*' - # echo "Generated Scala 2.13 build files don't match what's in repository" - # exit 1 - # fi - # # change to Scala 2.13 Directory - # cd scala2.13 - # # test command, will retry for 3 times if failed. - # max_retry=3; delay=30; i=1 - # while true; do - # mvn -Dmaven.wagon.http.retryHandler.count=3 \ - # -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 \ - # -Daether.connector.http.connectionMaxTtl=30 -B package \ - # -pl integration_tests,tests -am -P 'individual,pre-merge' \ - # -Dbuildver=${{ matrix.spark-version }} -Dmaven.scalastyle.skip=true \ - # -Drat.skip=true ${{ env.COMMON_MVN_FLAGS }} && break || { - # if [[ $i -le $max_retry ]]; then - # echo "mvn command failed. Retry $i/$max_retry."; ((i++)); sleep $delay; ((delay=delay*2)) - # else - # echo "mvn command failed. Exit 1"; exit 1 - # fi - # } - # done - - - # verify-all-modules: - # needs: get-shim-versions-from-dist - # runs-on: ubuntu-latest - # strategy: - # matrix: ${{ fromJSON(needs.get-shim-versions-from-dist.outputs.sparkJDKVersions) }} - # steps: - # - uses: actions/checkout@v3 # refs/pull/:prNumber/merge - - # - name: Setup Java and Maven Env - # uses: actions/setup-java@v3 - # with: - # distribution: adopt - # java-version: ${{ matrix.java-version }} - - # - name: check runtime before tests - # run: | - # env | grep JAVA - # java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH" - - # - name: Build JDK - # run: | - # # https://github.com/NVIDIA/spark-rapids/issues/8847 - # # specify expected versions - # export JAVA_HOME=${JAVA_HOME_${{ matrix.java-version }}_X64} - # export PATH=${JAVA_HOME}/bin:${PATH} - # java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH" - # # test command, will retry for 3 times if failed. - # max_retry=3; delay=30; i=1 - # while true; do - # mvn -Dmaven.wagon.http.retryHandler.count=$max_retry \ - # -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 \ - # -Daether.connector.http.connectionMaxTtl=30 -B verify \ - # -P "individual,pre-merge" -Dbuildver=${{ matrix.spark-version }} \ - # ${{ env.COMMON_MVN_FLAGS }} && break || { - # if [[ $i -le $max_retry ]]; then - # echo "mvn command failed. Retry $i/$max_retry."; ((i++)); sleep $delay; ((delay=delay*2)) - # else - # echo "mvn command failed. Exit 1"; exit 1 - # fi - # } - # done - - # install-modules: - # needs: get-shim-versions-from-dist - # runs-on: ubuntu-latest - # strategy: - # matrix: - # maven-version: [3.6.3, 3.8.8, 3.9.3] - # steps: - # - uses: actions/checkout@v3 # refs/pull/:prNumber/merge - - # - name: Setup Java - # uses: actions/setup-java@v3 - # with: - # distribution: adopt - # java-version: 11 - - # - name: Setup Maven Wrapper - # run: mvn wrapper:wrapper -Dmaven=${{ matrix.maven-version }} - - # - name: check runtime before tests - # run: | - # env | grep JAVA - # java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH" - - # - name: Install with Maven ${{ matrix.maven-version }} - # run: | - # # https://github.com/NVIDIA/spark-rapids/issues/8847 - # # specify expected versions - # export JAVA_HOME=${JAVA_HOME_11_X64} - # export PATH=${JAVA_HOME}/bin:${PATH} - # java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH" - # # test command, will retry for 3 times if failed. - # max_retry=3; delay=30; i=1 - # while true; do - # ./mvnw -Dmaven.wagon.http.retryHandler.count=3 \ - # -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 \ - # -Daether.connector.http.connectionMaxTtl=30 -B install \ - # -P "individual,pre-merge" \ - # -Dbuildver=${{ needs.get-shim-versions-from-dist.outputs.defaultSparkVersion }} \ - # ${{ env.COMMON_MVN_FLAGS }} && break || { - # if [[ $i -le $max_retry ]]; then - # echo "mvn command failed. Retry $i/$max_retry."; ((i++)); sleep $delay; ((delay=delay*2)) - # else - # echo "mvn command failed. Exit 1"; exit 1 - # fi - # } - # done + package-tests-scala213: + needs: get-shim-versions-from-dist + continue-on-error: ${{ matrix.isSnapshot }} + strategy: + matrix: ${{ fromJSON(needs.get-shim-versions-from-dist.outputs.scala213Versions) }} + fail-fast: false + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 # refs/pull/:prNumber/merge + + - name: Setup Java and Maven Env + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: 8 + + - name: check runtime before tests + run: | + env | grep JAVA + java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH" + + - name: Cache local Maven repository + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: package tests check + run: | + # https://github.com/NVIDIA/spark-rapids/issues/8847 + # specify expected versions + export JAVA_HOME=${JAVA_HOME_8_X64} + export PATH=${JAVA_HOME}/bin:${PATH} + java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH" + # verify Scala 2.13 build files + ./build/make-scala-version-build-files.sh 2.13 + # verify git status + if [ -n "$(echo -n $(git status -s | grep 'scala2.13'))" ]; then + git add -N scala2.13/* && git diff 'scala2.13/*' + echo "Generated Scala 2.13 build files don't match what's in repository" + exit 1 + fi + # change to Scala 2.13 Directory + cd scala2.13 + # test command, will retry for 3 times if failed. + max_retry=3; delay=30; i=1 + while true; do + mvn -Dmaven.wagon.http.retryHandler.count=3 \ + -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 \ + -Daether.connector.http.connectionMaxTtl=30 -B package \ + -pl integration_tests,tests -am -P 'individual,pre-merge' \ + -Dbuildver=${{ matrix.spark-version }} -Dmaven.scalastyle.skip=true \ + -Drat.skip=true ${{ env.COMMON_MVN_FLAGS }} && break || { + if [[ $i -le $max_retry ]]; then + echo "mvn command failed. Retry $i/$max_retry."; ((i++)); sleep $delay; ((delay=delay*2)) + else + echo "mvn command failed. Exit 1"; exit 1 + fi + } + done + + + verify-all-modules: + needs: get-shim-versions-from-dist + runs-on: ubuntu-latest + strategy: + matrix: ${{ fromJSON(needs.get-shim-versions-from-dist.outputs.sparkJDKVersions) }} + steps: + - uses: actions/checkout@v3 # refs/pull/:prNumber/merge + + - name: Setup Java and Maven Env + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: ${{ matrix.java-version }} + + - name: check runtime before tests + run: | + env | grep JAVA + java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH" + + - name: Cache local Maven repository + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Build JDK + run: | + # https://github.com/NVIDIA/spark-rapids/issues/8847 + # specify expected versions + export JAVA_HOME=${JAVA_HOME_${{ matrix.java-version }}_X64} + export PATH=${JAVA_HOME}/bin:${PATH} + java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH" + # test command, will retry for 3 times if failed. + max_retry=3; delay=30; i=1 + while true; do + mvn -Dmaven.wagon.http.retryHandler.count=$max_retry \ + -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 \ + -Daether.connector.http.connectionMaxTtl=30 -B verify \ + -P "individual,pre-merge" -Dbuildver=${{ matrix.spark-version }} \ + ${{ env.COMMON_MVN_FLAGS }} && break || { + if [[ $i -le $max_retry ]]; then + echo "mvn command failed. Retry $i/$max_retry."; ((i++)); sleep $delay; ((delay=delay*2)) + else + echo "mvn command failed. Exit 1"; exit 1 + fi + } + done + + install-modules: + needs: get-shim-versions-from-dist + runs-on: ubuntu-latest + strategy: + matrix: + maven-version: [3.6.3, 3.8.8, 3.9.3] + steps: + - uses: actions/checkout@v3 # refs/pull/:prNumber/merge + + - name: Setup Java + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: 11 + + - name: Setup Maven Wrapper + run: mvn wrapper:wrapper -Dmaven=${{ matrix.maven-version }} + + - name: check runtime before tests + run: | + env | grep JAVA + java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH" + + - name: Cache local Maven repository + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Install with Maven ${{ matrix.maven-version }} + run: | + # https://github.com/NVIDIA/spark-rapids/issues/8847 + # specify expected versions + export JAVA_HOME=${JAVA_HOME_11_X64} + export PATH=${JAVA_HOME}/bin:${PATH} + java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH" + # test command, will retry for 3 times if failed. + max_retry=3; delay=30; i=1 + while true; do + ./mvnw -Dmaven.wagon.http.retryHandler.count=3 \ + -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 \ + -Daether.connector.http.connectionMaxTtl=30 -B install \ + -P "individual,pre-merge" \ + -Dbuildver=${{ needs.get-shim-versions-from-dist.outputs.defaultSparkVersion }} \ + ${{ env.COMMON_MVN_FLAGS }} && break || { + if [[ $i -le $max_retry ]]; then + echo "mvn command failed. Retry $i/$max_retry."; ((i++)); sleep $delay; ((delay=delay*2)) + else + echo "mvn command failed. Exit 1"; exit 1 + fi + } + done From 3a39fb7e61c6535a81dabc29f2fe8ed8a0ec19ce Mon Sep 17 00:00:00 2001 From: YanxuanLiu Date: Fri, 10 Nov 2023 17:47:46 +0800 Subject: [PATCH 4/7] change key path Signed-off-by: YanxuanLiu --- .github/workflows/mvn-verify-check.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/mvn-verify-check.yml b/.github/workflows/mvn-verify-check.yml index e10d9d1d195..83561115108 100644 --- a/.github/workflows/mvn-verify-check.yml +++ b/.github/workflows/mvn-verify-check.yml @@ -121,7 +121,7 @@ jobs: uses: actions/cache@v3 with: path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('pom.xml') }} + key: ${{ runner.os }}-maven-${{ hashFiles('**/**/pom.xml') }} restore-keys: | ${{ runner.os }}-maven- @@ -174,7 +174,7 @@ jobs: uses: actions/cache@v3 with: path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('pom.xml') }} + key: ${{ runner.os }}-maven-${{ hashFiles('**/**/pom.xml') }} restore-keys: | ${{ runner.os }}-maven- @@ -236,7 +236,7 @@ jobs: uses: actions/cache@v3 with: path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('pom.xml') }} + key: ${{ runner.os }}-maven-${{ hashFiles('**/**/pom.xml') }} restore-keys: | ${{ runner.os }}-maven- @@ -290,7 +290,7 @@ jobs: uses: actions/cache@v3 with: path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('pom.xml') }} + key: ${{ runner.os }}-maven-${{ hashFiles('**/**/pom.xml') }} restore-keys: | ${{ runner.os }}-maven- From 64acb4844c24c8f93e5e8cbf9777242b2fddb90f Mon Sep 17 00:00:00 2001 From: YanxuanLiu Date: Mon, 13 Nov 2023 23:40:45 +0800 Subject: [PATCH 5/7] add mvn help to test Signed-off-by: YanxuanLiu --- .github/workflows/mvn-verify-check.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/mvn-verify-check.yml b/.github/workflows/mvn-verify-check.yml index 83561115108..e055725c077 100644 --- a/.github/workflows/mvn-verify-check.yml +++ b/.github/workflows/mvn-verify-check.yml @@ -133,6 +133,9 @@ jobs: export PATH=${JAVA_HOME}/bin:${PATH} java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH" # test command, will retry for 3 times if failed. + + mvn help:effective-settings + max_retry=3; delay=30; i=1 while true; do mvn -Dmaven.wagon.http.retryHandler.count=3 \ From c90f229107737c7c19277262fd35b1891f47ce36 Mon Sep 17 00:00:00 2001 From: YanxuanLiu Date: Tue, 14 Nov 2023 13:29:06 +0800 Subject: [PATCH 6/7] add cache for testing Signed-off-by: YanxuanLiu --- .github/workflows/mvn-verify-check.yml | 42 +++++++++++++------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/mvn-verify-check.yml b/.github/workflows/mvn-verify-check.yml index e055725c077..e5ee206cc9b 100644 --- a/.github/workflows/mvn-verify-check.yml +++ b/.github/workflows/mvn-verify-check.yml @@ -173,13 +173,13 @@ jobs: env | grep JAVA java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH" - - name: Cache local Maven repository - uses: actions/cache@v3 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- + # - name: Cache local Maven repository + # uses: actions/cache@v3 + # with: + # path: ~/.m2/repository + # key: ${{ runner.os }}-maven-${{ hashFiles('**/**/pom.xml') }} + # restore-keys: | + # ${{ runner.os }}-maven- - name: package tests check run: | @@ -235,13 +235,13 @@ jobs: env | grep JAVA java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH" - - name: Cache local Maven repository - uses: actions/cache@v3 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- + # - name: Cache local Maven repository + # uses: actions/cache@v3 + # with: + # path: ~/.m2/repository + # key: ${{ runner.os }}-maven-${{ hashFiles('**/**/pom.xml') }} + # restore-keys: | + # ${{ runner.os }}-maven- - name: Build JDK run: | @@ -289,13 +289,13 @@ jobs: env | grep JAVA java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH" - - name: Cache local Maven repository - uses: actions/cache@v3 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- + # - name: Cache local Maven repository + # uses: actions/cache@v3 + # with: + # path: ~/.m2/repository + # key: ${{ runner.os }}-maven-${{ hashFiles('**/**/pom.xml') }} + # restore-keys: | + # ${{ runner.os }}-maven- - name: Install with Maven ${{ matrix.maven-version }} run: | From d09773f9ae105ca005bd9e0c931717ccad7ad674 Mon Sep 17 00:00:00 2001 From: YanxuanLiu Date: Tue, 14 Nov 2023 14:50:51 +0800 Subject: [PATCH 7/7] remove cache for testing Signed-off-by: YanxuanLiu --- .github/workflows/mvn-verify-check.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/mvn-verify-check.yml b/.github/workflows/mvn-verify-check.yml index e5ee206cc9b..8b58031e3f0 100644 --- a/.github/workflows/mvn-verify-check.yml +++ b/.github/workflows/mvn-verify-check.yml @@ -117,13 +117,13 @@ jobs: env | grep JAVA java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH" - - name: Cache local Maven repository - uses: actions/cache@v3 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- + # - name: Cache local Maven repository + # uses: actions/cache@v3 + # with: + # path: ~/.m2/repository + # key: ${{ runner.os }}-maven-${{ hashFiles('**/**/pom.xml') }} + # restore-keys: | + # ${{ runner.os }}-maven- - name: package tests check run: |