chore: add short lived cache key

GaloyMoney · Oct 23, 2024 · 62845a9 · 62845a9
1 parent 2b662e5
commit 62845a9
Show file tree

Hide file tree

Showing 2 changed files with 146 additions and 66 deletions.
diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml
@@ -26,16 +26,11 @@ jobs:
 
       - name: Buck2 build
         run: |
-          nix develop -c buck2 build //core/api \
-            //core/api-ws-server \
-            //apps/dashboard \
-            //apps/consent \
-            //apps/pay \
-            //apps/admin-panel \
-            //apps/map \
-            //apps/voucher \
-            //core/api-keys \
-            //core/notifications
+          nix develop -c buck2 build //:node_modules \
+            //core/api //core/api-ws-server //core/api-trigger //core/api-exporter \
+            //apps/dashboard //apps/consent //apps/pay //apps/admin-panel //apps/map //apps/voucher \
+            //core/api-keys //core/notifications \
+            //bats/helpers/callback:run //bats/helpers/subscriber:run //bats/helpers/totp:generate
 
       - name: Run bats tests
         run: |

diff --git a/.github/workflows/e2e-test2.yml b/.github/workflows/e2e-test2.yml
@@ -6,30 +6,35 @@ on:
 
 jobs:
   build:
-    name: parallel buck2 builds
+    name: clustered buck2 builds
     runs-on: ubuntu-latest
-    timeout-minutes: 10
+    timeout-minutes: 15
     strategy:
       matrix:
-        target:
-          - //core/api
-          - //core/api-ws-server
-          - //core/api-trigger
-          - //core/api-exporter
-          - //apps/dashboard
-          - //apps/consent
-          - //apps/pay
-          - //apps/admin-panel
-          - //apps/map
-          - //apps/voucher
-          - //core/api-keys
-          - //core/notifications
-          - //bats/helpers/callback:run
-          - //bats/helpers/subscriber:run
-          - //bats/helpers/totp:generate
+        cluster:
+          - name: core-apis
+            targets: "//:node_modules //core/api //core/api-ws-server //core/api-trigger //core/api-exporter"
+          - name: core-apps
+            targets: "//apps/dashboard //apps/consent //apps/admin-panel"
+          - name: apps
+            targets: "//apps/pay //apps/map //apps/voucher"
+          - name: core-services
+            targets: "//core/api-keys //core/notifications"
+          - name: bats-helpers
+            targets: "//bats/helpers/callback:run //bats/helpers/subscriber:run //bats/helpers/totp:generate"
       fail-fast: false
 
     steps:
+      - id: vars
+        run: echo "short_sha=${GITHUB_SHA:0:7}" >> $GITHUB_ENV
+
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          root-reserve-mb: 15360
+          temp-reserve-mb: 12288
+          remove-dotnet: "true"
+
       - uses: actions/checkout@v4
 
       - name: Install Nix
@@ -38,22 +43,31 @@ jobs:
       - name: Run the Magic Nix Cache
         uses: DeterminateSystems/magic-nix-cache-action@v8
 
-      - name: Build
+      - name: Create cache directories
+        run: |
+          mkdir -p ~/.buck2 ~/.cache/buck2 buck-out/v2
+
+      - name: Build cluster
+        run: |
+          nix develop -c buck2 build ${{ matrix.cluster.targets }}
+
+      - name: Prepare cache directory
+        run: |
+          # Create a cluster-specific directory
+          mkdir -p cluster-cache/${{ matrix.cluster.name }}
+          # Move buck-out contents to cluster-specific directory
+          cp -r buck-out/* cluster-cache/${{ matrix.cluster.name }}/
+
+      - name: Debug cache content
         run: |
-          nix develop -c buck2 build ${{ matrix.target }}
+          echo "Cluster ${{ matrix.cluster.name }} cache size:"
+          du -sh cluster-cache/${{ matrix.cluster.name }}
 
       - uses: actions/cache/save@v4
         with:
-          path: |
-            ~/.buck2
-            buck-out
-            .buck-cache
-            prebuilt
-            installed
-            target
-            .buckd
-            buck-out/cache
-          key: buck2-${{ runner.os }}-${{ matrix.target }}
+          path: cluster-cache/${{ matrix.cluster.name }}
+          key: b2-${{ matrix.cluster.name }}-${{ env.short_sha }}
+          enableCrossOsArchive: true
 
   tests:
     name: execute via bats
@@ -62,6 +76,16 @@ jobs:
     needs: build
 
     steps:
+      - id: vars
+        run: echo "short_sha=${GITHUB_SHA:0:7}" >> $GITHUB_ENV
+
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          root-reserve-mb: 15360
+          temp-reserve-mb: 12288
+          remove-dotnet: "true"
+
       - uses: actions/checkout@v4
 
       - name: Install Nix
@@ -70,36 +94,97 @@ jobs:
       - name: Run the Magic Nix Cache
         uses: DeterminateSystems/magic-nix-cache-action@v8
 
-      - uses: actions/cache/restore@v4
+      - name: Create cache directories
+        run: |
+          mkdir -p ~/.buck2 ~/.cache/buck2 buck-out/v2 cluster-cache
+
+      # Restore each cluster cache separately
+      - name: Restore core-apis cache
+        uses: actions/cache/restore@v4
+        with:
+          path: cluster-cache/core-apis
+          key: b2-core-apis-${{ env.short_sha }}
+          fail-on-cache-miss: true
+
+      - name: Restore core-apps cache
+        uses: actions/cache/restore@v4
+        with:
+          path: cluster-cache/core-apps
+          key: b2-core-apps-${{ env.short_sha }}
+          fail-on-cache-miss: true
+
+      - name: Restore apps cache
+        uses: actions/cache/restore@v4
+        with:
+          path: cluster-cache/apps
+          key: b2-apps-${{ env.short_sha }}
+          fail-on-cache-miss: true
+
+      - name: Restore core-services cache
+        uses: actions/cache/restore@v4
+        with:
+          path: cluster-cache/core-services
+          key: b2-core-services-${{ env.short_sha }}
+          fail-on-cache-miss: true
+
+      - name: Restore bats-helpers cache
+        uses: actions/cache/restore@v4
         with:
-          path: |
-            ~/.buck2
-            buck-out
-            .buck-cache
-            prebuilt
-            installed
-            target
-            .buckd
-            buck-out/cache
-          key: buck2-${{ runner.os }}-
+          path: cluster-cache/bats-helpers
+          key: b2-bats-helpers-${{ env.short_sha }}
+          fail-on-cache-miss: true
+
+      - name: Install rsync
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y rsync
+
+      - name: Merge cluster caches
+        run: |
+          # Remove existing buck-out if it exists
+          rm -rf buck-out
+
+          # Move the first cluster's contents directly to buck-out
+          if [ -d "cluster-cache/core-apis" ]; then
+            echo "Moving core-apis cache to buck-out"
+            mv cluster-cache/core-apis buck-out
+          fi
+
+          # For remaining clusters, use rsync to merge their contents
+          for cluster in core-apps apps core-services bats-helpers; do
+            if [ -d "cluster-cache/$cluster" ]; then
+              echo "Merging cache from $cluster"
+              rsync -a "cluster-cache/$cluster/" buck-out/
+            fi
+          done
+
+          # Ensure correct permissions
+          chmod -R u+w buck-out
+
+      - name: Debug restored caches
+        run: |
+          echo "Contents before cleanup:"
+          ls -la cluster-cache/ || true
+          echo "Buck-out size after merge:"
+          du -sh buck-out
+          echo "Buck-out structure:"
+          find buck-out -type d -maxdepth 6
+          echo "Permissions:"
+          ls -la buck-out/
+
+      - name: Cleanup cluster cache
+        run: |
+          rm -rf cluster-cache
 
       - name: Run bats tests
         run: |
-          nix develop -c buck2 build //core/api \
-            //core/api-ws-server \
-            //core/api-trigger \
-            //core/api-exporter \
-            //apps/dashboard \
-            //apps/consent \
-            //apps/pay \
-            //apps/admin-panel \
-            //apps/map \
-            //apps/voucher \
-            //core/api-keys \
-            //core/notifications \
-            //bats/helpers/callback:run \
-            //bats/helpers/subscriber:run \
-            //bats/helpers/totp:generate
+          echo "Buck build status before tests:"
+          nix develop -c buck2 build --show-output \
+            //core/api //core/api-ws-server //core/api-trigger //core/api-exporter \
+            //apps/dashboard //apps/consent //apps/pay //apps/admin-panel //apps/map //apps/voucher \
+            //core/api-keys //core/notifications \
+            //bats/helpers/callback:run //bats/helpers/subscriber:run //bats/helpers/totp:generate
+
           nix develop -c bats --setup-suite-file bats/ci_setup_suite.bash -t bats/core/**
 
       - name: Rename Tilt log