Add cache action to speed up mvn workflow [skip ci] (#9674)

* add cache for testing Signed-off-by: YanxuanLiu <[email protected]> * change cache key file Signed-off-by: YanxuanLiu <[email protected]> * add cache for all stages Signed-off-by: YanxuanLiu <[email protected]> * change key path Signed-off-by: YanxuanLiu <[email protected]> * add mvn help to test Signed-off-by: YanxuanLiu <[email protected]> * add isolated cache job Signed-off-by: YanxuanLiu <[email protected]> * add checkout Signed-off-by: YanxuanLiu <[email protected]> * add java setup Signed-off-by: YanxuanLiu <[email protected]> * remove distribution Signed-off-by: YanxuanLiu <[email protected]> * add distribution Signed-off-by: YanxuanLiu <[email protected]> * add retry for cache Signed-off-by: YanxuanLiu <[email protected]> * add project param Signed-off-by: YanxuanLiu <[email protected]> * make sure cache finished before shim version matrix Signed-off-by: YanxuanLiu <[email protected]> * add cache Signed-off-by: YanxuanLiu <[email protected]> * add cache Signed-off-by: YanxuanLiu <[email protected]> * add cache Signed-off-by: YanxuanLiu <[email protected]> * add check Signed-off-by: YanxuanLiu <[email protected]> * remove test step Signed-off-by: YanxuanLiu <[email protected]> * add ls before cache Signed-off-by: YanxuanLiu <[email protected]> * add check Signed-off-by: YanxuanLiu <[email protected]> * removed test steps Signed-off-by: YanxuanLiu <[email protected]> * add new mvn command, and changed key Signed-off-by: YanxuanLiu <[email protected]> * change mvn command Signed-off-by: YanxuanLiu <[email protected]> * change mvn command Signed-off-by: YanxuanLiu <[email protected]> * Update .github/workflows/mvn-verify-check.yml Co-authored-by: Gera Shegalov <[email protected]> * Update .github/workflows/mvn-verify-check.yml Co-authored-by: Gera Shegalov <[email protected]> * change key format Signed-off-by: YanxuanLiu <[email protected]> * change time format Signed-off-by: YanxuanLiu <[email protected]> * add date Signed-off-by: YanxuanLiu <[email protected]> * fix bug Signed-off-by: YanxuanLiu <[email protected]> * fix time Signed-off-by: YanxuanLiu <[email protected]> * chang key name Signed-off-by: YanxuanLiu <[email protected]> * fix bug Signed-off-by: YanxuanLiu <[email protected]> * change time def Signed-off-by: YanxuanLiu <[email protected]> * change time Signed-off-by: YanxuanLiu <[email protected]> * add step to get current date Signed-off-by: YanxuanLiu <[email protected]> * recover excess deletion. Signed-off-by: YanxuanLiu <[email protected]> * change ref to target branch Signed-off-by: YanxuanLiu <[email protected]> * generate key in cache job Signed-off-by: YanxuanLiu <[email protected]> * upadte key Signed-off-by: YanxuanLiu <[email protected]> * echo key for testing Signed-off-by: YanxuanLiu <[email protected]> * fix bug of time Signed-off-by: YanxuanLiu <[email protected]> * test date Signed-off-by: YanxuanLiu <[email protected]> * remove java version matrix Signed-off-by: YanxuanLiu <[email protected]> * add runner os Signed-off-by: YanxuanLiu <[email protected]> * Update .github/workflows/mvn-verify-check.yml Co-authored-by: Gera Shegalov <[email protected]> * Update .github/workflows/mvn-verify-check.yml Co-authored-by: Gera Shegalov <[email protected]> * add scala 213, and adjusted sequence of steps Signed-off-by: YanxuanLiu <[email protected]> * change iteration var name Signed-off-by: YanxuanLiu <[email protected]> * Update .github/workflows/mvn-verify-check.yml Co-authored-by: Gera Shegalov <[email protected]> --------- Signed-off-by: YanxuanLiu <[email protected]> Co-authored-by: Gera Shegalov <[email protected]>
NVIDIA · Dec 1, 2023 · 8adc3cb · 8adc3cb
1 parent fcad227
commit 8adc3cb
Showing 1 changed file with 70 additions and 4 deletions.
diff --git a/.github/workflows/mvn-verify-check.yml b/.github/workflows/mvn-verify-check.yml
@@ -30,7 +30,49 @@ env:
     -Dmaven.scaladoc.skip
 
 jobs:
+  cache-dependencies:
+    runs-on: ubuntu-latest
+    outputs:
+      dailyCacheKey: ${{ steps.generateCacheKey.outputs.dailyCacheKey }}
+    steps:
+      - uses: actions/checkout@v3 # refs/pull/:prNumber/merge
+      - uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: 8
+      - name: Generate daily cache key
+        id: generateCacheKey
+        run: |
+          set -x
+          cacheKey="${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-${{ github.event.pull_request.base.ref }}-$(date +'%Y-%m-%d')"
+          echo "dailyCacheKey=$cacheKey" | tee $GITHUB_ENV $GITHUB_OUTPUT
+      - name: Cache local Maven repository
+        id: cache
+        uses: actions/cache@v3
+        with:
+          path: ~/.m2
+          key: ${{ env.dailyCacheKey }}
+      - name: populate-daily-cache
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          set -x
+          max_retry=3; delay=30; i=1
+          while true; do
+            for pom in pom.xml scala2.13/pom.xml; do 
+              mvn -f $pom help:evaluate -pl dist -Dexpression=included_buildvers \
+              -DforceStdout -PnoSnapshots -q | tr -d ',' | \
+              xargs -n 1 bash -c 'mvn -f $1 initialize -pl sql-plugin-api -am -Dbuildver=$2' _ $pom;
+            done && break || {
+            if [[ $i -le $max_retry ]]; then
+              echo "mvn command failed. Retry $i/$max_retry."; ((i++)); sleep $delay; ((delay=delay*2))
+            else
+              echo "mvn command failed. Exit 1"; exit 1
+            fi
+          }
+          done 
+
   get-shim-versions-from-dist:
+    needs: cache-dependencies
     runs-on: ubuntu-latest
     outputs:
       defaultSparkVersion: ${{ steps.allShimVersionsStep.outputs.defaultSparkVersion }}
@@ -97,7 +139,7 @@ jobs:
           echo "scala213Versions=$svJsonStr" >> $GITHUB_OUTPUT
 
   package-tests:
-    needs: get-shim-versions-from-dist
+    needs: [get-shim-versions-from-dist, cache-dependencies]
     continue-on-error: ${{ matrix.isSnapshot }}
     strategy:
       matrix: ${{ fromJSON(needs.get-shim-versions-from-dist.outputs.sparkTailVersions) }}
@@ -112,6 +154,12 @@ jobs:
           distribution: adopt
           java-version: 8
 
+      - name: Cache local Maven repository
+        uses: actions/cache@v3
+        with:
+          path: ~/.m2
+          key: ${{ needs.cache-dependencies.outputs.dailyCacheKey }}
+
       - name: check runtime before tests
         run: |
           env | grep JAVA
@@ -142,7 +190,7 @@ jobs:
           done
 
   package-tests-scala213:
-    needs: get-shim-versions-from-dist
+    needs: [get-shim-versions-from-dist, cache-dependencies]
     continue-on-error: ${{ matrix.isSnapshot }}
     strategy:
       matrix: ${{ fromJSON(needs.get-shim-versions-from-dist.outputs.scala213Versions) }}
@@ -157,6 +205,12 @@ jobs:
           distribution: adopt
           java-version: 8
 
+      - name: Cache local Maven repository
+        uses: actions/cache@v3
+        with:
+          path: ~/.m2
+          key: ${{ needs.cache-dependencies.outputs.dailyCacheKey }}
+
       - name: check runtime before tests
         run: |
           env | grep JAVA
@@ -198,7 +252,7 @@ jobs:
 
 
   verify-all-modules:
-    needs: get-shim-versions-from-dist
+    needs: [get-shim-versions-from-dist, cache-dependencies]
     runs-on: ubuntu-latest
     strategy:
       matrix: ${{ fromJSON(needs.get-shim-versions-from-dist.outputs.sparkJDKVersions) }}
@@ -211,6 +265,12 @@ jobs:
           distribution: adopt
           java-version: ${{ matrix.java-version }}
 
+      - name: Cache local Maven repository
+        uses: actions/cache@v3
+        with:
+          path: ~/.m2
+          key: ${{ needs.cache-dependencies.outputs.dailyCacheKey }}
+
       - name: check runtime before tests
         run: |
           env | grep JAVA
@@ -240,7 +300,7 @@ jobs:
           done
 
   install-modules:
-    needs: get-shim-versions-from-dist
+    needs: [get-shim-versions-from-dist, cache-dependencies]
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -254,6 +314,12 @@ jobs:
           distribution: adopt
           java-version: 11
 
+      - name: Cache local Maven repository
+        uses: actions/cache@v3
+        with:
+          path: ~/.m2
+          key: ${{ needs.cache-dependencies.outputs.dailyCacheKey }}
+
       - name: Setup Maven Wrapper
         run: mvn wrapper:wrapper -Dmaven=${{ matrix.maven-version }}