From 4b8f03cd6552cf2ef3cc6998a3e1b4e13f76d287 Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Mon, 20 Jan 2025 15:24:31 +0800
Subject: [PATCH 01/34] feat: added gauge metrics for batch, blob, aggregation
 size

---
 .../GlobalAggregationCalculator.kt            | 40 ++++++++++++++++++-
 .../blob/BlobCompressionProofCoordinator.kt   | 13 ++++++
 .../conflation/ConflationServiceImpl.kt       |  9 ++++-
 3 files changed, 59 insertions(+), 3 deletions(-)

diff --git a/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/aggregation/GlobalAggregationCalculator.kt b/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/aggregation/GlobalAggregationCalculator.kt
index 4bd180bc5..2a1e97823 100644
--- a/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/aggregation/GlobalAggregationCalculator.kt
+++ b/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/aggregation/GlobalAggregationCalculator.kt
@@ -8,6 +8,7 @@ import net.consensys.zkevm.domain.BlobsToAggregate
 import org.apache.logging.log4j.LogManager
 import org.apache.logging.log4j.Logger
 import java.util.PriorityQueue
+import java.util.concurrent.atomic.AtomicInteger
 
 class GlobalAggregationCalculator(
   private var lastBlockNumber: ULong,
@@ -23,6 +24,9 @@ class GlobalAggregationCalculator(
     b1.startBlockNumber.compareTo(b2.startBlockNumber)
   }
 
+  private val aggregationSizeInBlocks = AtomicInteger(0)
+  private val aggregationSizeInBatches = AtomicInteger(0)
+  private val aggregationSizeInBlobs = AtomicInteger(0)
   private val blobsCounter: Counter = metricsFacade.createCounter(
     category = LineaMetricsCategory.AGGREGATION,
     name = "calculator.blobs.accepted",
@@ -61,6 +65,30 @@ class GlobalAggregationCalculator(
         pendingBlobs.size
       }
     )
+    metricsFacade.createGauge(
+      category = LineaMetricsCategory.AGGREGATION,
+      name = "blocks.size",
+      description = "Number of blocks in each aggregation",
+      measurementSupplier = {
+        aggregationSizeInBlocks.get()
+      }
+    )
+    metricsFacade.createGauge(
+      category = LineaMetricsCategory.AGGREGATION,
+      name = "batches.size",
+      description = "Number of batches in each aggregation",
+      measurementSupplier = {
+        aggregationSizeInBatches.get()
+      }
+    )
+    metricsFacade.createGauge(
+      category = LineaMetricsCategory.AGGREGATION,
+      name = "blobs.size",
+      description = "Number of blobs in each aggregation",
+      measurementSupplier = {
+        aggregationSizeInBlobs.get()
+      }
+    )
   }
 
   private fun processBlob(blobCounters: BlobCounters) {
@@ -144,18 +172,26 @@ class GlobalAggregationCalculator(
       endBlockNumber = blobsInUpdatedAggregation.last().endBlockNumber
     )
 
+    val blocksCount = updatedAggregation.blocksRange.count()
+    val batchesCount = blobsInUpdatedAggregation.sumOf { it.numberOfBatches }.toInt()
+    val blobsCount = blobsInUpdatedAggregation.size
+
     log.info(
       "aggregation: trigger={} aggregation={} updatedAggregation={} " +
         "blobsCount={} batchesCount={} blobs={} aggregationSizeMultiple={}",
       aggregationTrigger.aggregationTriggerType.name,
       aggregation.intervalString(),
       updatedAggregation.intervalString(),
-      blobsInUpdatedAggregation.size,
-      blobsInUpdatedAggregation.sumOf { it.numberOfBatches },
+      blobsCount,
+      batchesCount,
       blobsInUpdatedAggregation.map { it.intervalString() },
       aggregationSizeMultipleOf
     )
 
+    aggregationSizeInBlocks.set(blocksCount)
+    aggregationSizeInBatches.set(batchesCount)
+    aggregationSizeInBlobs.set(blobsCount)
+
     // Reset the trigger calculators now that we have a valid aggregation to handle
     resetTriggerCalculators()
 
diff --git a/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/blob/BlobCompressionProofCoordinator.kt b/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/blob/BlobCompressionProofCoordinator.kt
index ed1e33973..758f1b253 100644
--- a/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/blob/BlobCompressionProofCoordinator.kt
+++ b/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/blob/BlobCompressionProofCoordinator.kt
@@ -21,6 +21,7 @@ import org.apache.logging.log4j.Logger
 import tech.pegasys.teku.infrastructure.async.SafeFuture
 import java.util.concurrent.CompletableFuture
 import java.util.concurrent.LinkedBlockingDeque
+import java.util.concurrent.atomic.AtomicInteger
 import kotlin.time.Duration
 
 class BlobCompressionProofCoordinator(
@@ -62,6 +63,18 @@ class BlobCompressionProofCoordinator(
       description = "Size of blob compression proving queue",
       measurementSupplier = { blobsToHandle.size }
     )
+    metricsFacade.createGauge(
+      category = LineaMetricsCategory.BLOB,
+      name = "blocks.size",
+      description = "Number of blocks in each blob",
+      measurementSupplier = { blobSizeInBlocks.get() }
+    )
+    metricsFacade.createGauge(
+      category = LineaMetricsCategory.BLOB,
+      name = "batches.size",
+      description = "Number of batches in each blob",
+      measurementSupplier = { blobSizeInBatches.get() }
+    )
   }
 
   data class Config(
diff --git a/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/ConflationServiceImpl.kt b/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/ConflationServiceImpl.kt
index ed70033df..d231b44c1 100644
--- a/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/ConflationServiceImpl.kt
+++ b/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/ConflationServiceImpl.kt
@@ -11,6 +11,7 @@ import org.apache.logging.log4j.Logger
 import tech.pegasys.teku.infrastructure.async.SafeFuture
 import java.util.concurrent.PriorityBlockingQueue
 import java.util.concurrent.TimeUnit
+import java.util.concurrent.atomic.AtomicInteger
 
 class ConflationServiceImpl(
   private val calculator: TracesConflationCalculator,
@@ -31,7 +32,7 @@ class ConflationServiceImpl(
   }
 
   internal val blocksToConflate = PriorityBlockingQueue<PayloadAndBlockCounters>()
-
+  private val batchSizeInBlocks = AtomicInteger(0)
   private val blocksCounter = metricsFacade.createCounter(
     category = LineaMetricsCategory.CONFLATION,
     name = "blocks.imported",
@@ -56,6 +57,12 @@ class ConflationServiceImpl(
       description = "Number of blocks in conflation queue",
       measurementSupplier = { blocksToConflate.size }
     )
+    metricsFacade.createGauge(
+      category = LineaMetricsCategory.CONFLATION,
+      name = "blocks.size",
+      description = "Number of blocks in each conflated batch",
+      measurementSupplier = { batchSizeInBlocks.get() }
+    )
     calculator.onConflatedBatch(this::handleConflation)
   }
 

From ab80c056009fb2c787f047a57af4d249bd35f001 Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Tue, 21 Jan 2025 15:19:12 +0800
Subject: [PATCH 02/34] feat: upgrade actions/cache save and restore to 4.2.0

---
 .github/workflows/coordinator-testing.yml          |  3 ---
 .github/workflows/staterecovery-testing.yml        |  3 +--
 .../transaction-exclusion-api-testing.yml          | 14 ++++++++++++++
 3 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/coordinator-testing.yml b/.github/workflows/coordinator-testing.yml
index 0df306ec5..c231f3dab 100644
--- a/.github/workflows/coordinator-testing.yml
+++ b/.github/workflows/coordinator-testing.yml
@@ -17,9 +17,6 @@ concurrency:
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
 
 jobs:
-  cache-docker-images:
-    uses: ./.github/workflows/cache-docker-images.yml
-    secrets: inherit
   run-tests:
     env:
       COMMIT_TAG: ${{ inputs.commit_tag }}
diff --git a/.github/workflows/staterecovery-testing.yml b/.github/workflows/staterecovery-testing.yml
index 43100ccc5..641ef1346 100644
--- a/.github/workflows/staterecovery-testing.yml
+++ b/.github/workflows/staterecovery-testing.yml
@@ -23,10 +23,9 @@ jobs:
   run-tests:
     env:
       COMMIT_TAG: ${{ inputs.commit_tag }}
-      GITHUB_TOKEN: ${{ secrets._GITHUB_TOKEN_RELEASE_ACCESS }}
       DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
-      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+      # CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
     # ~2.5 mins saved vs large
     runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-xl
     name: Staterecovery tests
diff --git a/.github/workflows/transaction-exclusion-api-testing.yml b/.github/workflows/transaction-exclusion-api-testing.yml
index f76f57ca4..e479343d4 100644
--- a/.github/workflows/transaction-exclusion-api-testing.yml
+++ b/.github/workflows/transaction-exclusion-api-testing.yml
@@ -2,6 +2,11 @@ name: transaction-exclusion-api-testing
 
 on:
   workflow_call:
+    secrets:
+      DOCKERHUB_USERNAME:
+        required: false
+      DOCKERHUB_TOKEN:
+        required: false
   workflow_dispatch:
     inputs:
       coverage:
@@ -16,6 +21,9 @@ concurrency:
 
 jobs:
   run-tests:
+    env:
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
     # ~1.5 mins saved vs small
     runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-med
     name: Transaction exclusion api tests
@@ -36,6 +44,12 @@ jobs:
         if: ${{ !inputs.coverage }}
         run: |
           ./gradlew transaction-exclusion-api:app:buildNeeded
+      - name: Login to Docker Hub
+        if: ${{ env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != '' }}
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Run integration tests
         run: |
           ./gradlew transaction-exclusion-api:app:integrationTestAllNeeded

From 8f1037524c5e6bbc5b0448d8e235e35cec54cd85 Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Tue, 21 Jan 2025 15:42:07 +0800
Subject: [PATCH 03/34] feat: add secret inherit for transaction-exclusion-api

---
 .github/workflows/testing.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index f973c8ee4..5ef176c13 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -50,6 +50,7 @@ jobs:
   transaction-exclusion-api:
     uses: ./.github/workflows/transaction-exclusion-api-testing.yml
     if: ${{ inputs.transaction_exclusion_api_changed == 'true' }}
+    secrets: inherit
 
   staterecovery:
     uses: ./.github/workflows/staterecovery-testing.yml

From 17884478e12015b8f8fe1866adcd6442fff9230b Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Tue, 21 Jan 2025 16:37:00 +0800
Subject: [PATCH 04/34] feat: revert to include cache-docker-images job in
 coordinator-testing.yml

---
 .github/workflows/coordinator-testing.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/coordinator-testing.yml b/.github/workflows/coordinator-testing.yml
index c231f3dab..0df306ec5 100644
--- a/.github/workflows/coordinator-testing.yml
+++ b/.github/workflows/coordinator-testing.yml
@@ -17,6 +17,9 @@ concurrency:
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
 
 jobs:
+  cache-docker-images:
+    uses: ./.github/workflows/cache-docker-images.yml
+    secrets: inherit
   run-tests:
     env:
       COMMIT_TAG: ${{ inputs.commit_tag }}

From e68bf18cf796988c2a196c3d8f26454f61ae922f Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Tue, 21 Jan 2025 18:55:26 +0800
Subject: [PATCH 05/34] feat: move cache-docker-images out from
 coordinator-testing and staterecovery-test

---
 .github/workflows/coordinator-testing.yml   | 3 ---
 .github/workflows/staterecovery-testing.yml | 3 ---
 .github/workflows/testing.yml               | 4 ++++
 3 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/coordinator-testing.yml b/.github/workflows/coordinator-testing.yml
index 0df306ec5..c231f3dab 100644
--- a/.github/workflows/coordinator-testing.yml
+++ b/.github/workflows/coordinator-testing.yml
@@ -17,9 +17,6 @@ concurrency:
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
 
 jobs:
-  cache-docker-images:
-    uses: ./.github/workflows/cache-docker-images.yml
-    secrets: inherit
   run-tests:
     env:
       COMMIT_TAG: ${{ inputs.commit_tag }}
diff --git a/.github/workflows/staterecovery-testing.yml b/.github/workflows/staterecovery-testing.yml
index 641ef1346..0cca72747 100644
--- a/.github/workflows/staterecovery-testing.yml
+++ b/.github/workflows/staterecovery-testing.yml
@@ -17,9 +17,6 @@ concurrency:
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
 
 jobs:
-  cache-docker-images:
-    uses: ./.github/workflows/cache-docker-images.yml
-    secrets: inherit
   run-tests:
     env:
       COMMIT_TAG: ${{ inputs.commit_tag }}
diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index 5ef176c13..b1bac977e 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -26,6 +26,10 @@ on:
         type: string
 
 jobs:
+  cache-docker-images:
+    uses: ./.github/workflows/cache-docker-images.yml
+    secrets: inherit
+  
   coordinator:
     uses: ./.github/workflows/coordinator-testing.yml
     if: ${{ inputs.coordinator_changed == 'true' }}

From 19e9c734a299e7c1d4c55b222929010e393da1b1 Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Tue, 21 Jan 2025 19:08:17 +0800
Subject: [PATCH 06/34] feat: text changes for testing

---
 docker/compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/compose.yml b/docker/compose.yml
index 04c4b55a2..78f19bb68 100644
--- a/docker/compose.yml
+++ b/docker/compose.yml
@@ -16,7 +16,7 @@ networks:
       config:
         - subnet: 10.10.10.0/24
 
-# To debug inside the network and volumes
+# To debug inside the network and volumes:
 # docker run --rm -it --network=docker_linea -v=linea-local-dev:/data -v=linea-logs:/logs weibeld/ubuntu-networking bash
 
 services:

From 0bdc2ab9e431bdabd9c3df0a72741ea896a20f35 Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Tue, 21 Jan 2025 20:36:10 +0800
Subject: [PATCH 07/34] feat: set restore-keys as cached-images

---
 .github/workflows/reuse-run-e2e-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/reuse-run-e2e-tests.yml b/.github/workflows/reuse-run-e2e-tests.yml
index 66fa2141f..1a5b7acb7 100644
--- a/.github/workflows/reuse-run-e2e-tests.yml
+++ b/.github/workflows/reuse-run-e2e-tests.yml
@@ -104,7 +104,7 @@ jobs:
           path: ~/docker-images
           key: cached-images
           restore-keys: |
-            cached-
+            cached-images
       - name: Pull all images with retry
         uses: nick-fields/retry@v3
         with:

From 802dc40e5d20262883fb75cf69a9fc6fe1f5f03b Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Tue, 21 Jan 2025 21:28:04 +0800
Subject: [PATCH 08/34] feat: add step to create directory for cached docker
 images

---
 .github/workflows/coordinator-testing.yml   | 3 +++
 .github/workflows/reuse-run-e2e-tests.yml   | 3 +++
 .github/workflows/staterecovery-testing.yml | 3 +++
 3 files changed, 9 insertions(+)

diff --git a/.github/workflows/coordinator-testing.yml b/.github/workflows/coordinator-testing.yml
index c231f3dab..2d809331a 100644
--- a/.github/workflows/coordinator-testing.yml
+++ b/.github/workflows/coordinator-testing.yml
@@ -37,6 +37,9 @@ jobs:
         # Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies.
         # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md
         uses: gradle/actions/setup-gradle@cc4fc85e6b35bafd578d5ffbc76a5518407e1af0 #v4.2.1
+      - name: Create directory for cached docker images
+        run: |
+          mkdir -p ~/docker-images
       - name: Restore cached images
         id: restore-cached-images
         uses: actions/cache/restore@v4.2.0
diff --git a/.github/workflows/reuse-run-e2e-tests.yml b/.github/workflows/reuse-run-e2e-tests.yml
index 1a5b7acb7..52706a011 100644
--- a/.github/workflows/reuse-run-e2e-tests.yml
+++ b/.github/workflows/reuse-run-e2e-tests.yml
@@ -97,6 +97,9 @@ jobs:
         run: |
           mkdir -p tmp/local/traces/v2/conflated
           chmod -R a+w tmp/local/traces/v2/conflated
+      - name: Create directory for cached docker images
+        run: |
+          mkdir -p ~/docker-images
       - name: Restore cached images
         id: restore-cached-images
         uses: actions/cache/restore@v4.2.0
diff --git a/.github/workflows/staterecovery-testing.yml b/.github/workflows/staterecovery-testing.yml
index 0cca72747..79adc484e 100644
--- a/.github/workflows/staterecovery-testing.yml
+++ b/.github/workflows/staterecovery-testing.yml
@@ -37,6 +37,9 @@ jobs:
         # Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies.
         # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md
         uses: gradle/actions/setup-gradle@cc4fc85e6b35bafd578d5ffbc76a5518407e1af0 #v4.2.1
+      - name: Create directory for cached docker images
+        run: |
+          mkdir -p ~/docker-images
       - name: Restore cached images
         id: restore-cached-images
         uses: actions/cache/restore@v4.2.0

From 380fc22840d6df00a025d5dbcaee662757b6d26c Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Tue, 21 Jan 2025 22:23:46 +0800
Subject: [PATCH 09/34] feat: change cache dir from ~/docker-images to
 ./docker-images

---
 .github/workflows/cache-docker-images.yml   | 6 +++---
 .github/workflows/coordinator-testing.yml   | 4 ++--
 .github/workflows/reuse-run-e2e-tests.yml   | 4 ++--
 .github/workflows/staterecovery-testing.yml | 4 ++--
 4 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/cache-docker-images.yml b/.github/workflows/cache-docker-images.yml
index 584fd92f6..13e5ed1a9 100644
--- a/.github/workflows/cache-docker-images.yml
+++ b/.github/workflows/cache-docker-images.yml
@@ -61,16 +61,16 @@ jobs:
           docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 pull
       - name: Save Docker images
         run: |
-          mkdir -p ~/docker-images
+          mkdir -p ./docker-images
           images=$(docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 config | grep "image:" | awk '{print $2}')
           for image in $images; do
             imageFileName=$(echo $image | sed -e 's|.*/||' -e 's|:|-|' -e 's/\./_/g')
             echo $image - ${imageFileName}
-            docker save $image > ~/docker-images/${imageFileName}.tar
+            docker save $image > ./docker-images/${imageFileName}.tar
           done
       - name: Cache common docker images
         continue-on-error: true
         uses: actions/cache/save@v4.2.0
         with:
-          path: ~/docker-images
+          path: ./docker-images
           key: cached-images
diff --git a/.github/workflows/coordinator-testing.yml b/.github/workflows/coordinator-testing.yml
index 2d809331a..f10c377f0 100644
--- a/.github/workflows/coordinator-testing.yml
+++ b/.github/workflows/coordinator-testing.yml
@@ -39,12 +39,12 @@ jobs:
         uses: gradle/actions/setup-gradle@cc4fc85e6b35bafd578d5ffbc76a5518407e1af0 #v4.2.1
       - name: Create directory for cached docker images
         run: |
-          mkdir -p ~/docker-images
+          mkdir -p ./docker-images
       - name: Restore cached images
         id: restore-cached-images
         uses: actions/cache/restore@v4.2.0
         with:
-          path: ~/docker-images
+          path: ./docker-images
           key: cached-images
           restore-keys: |
             cached-images
diff --git a/.github/workflows/reuse-run-e2e-tests.yml b/.github/workflows/reuse-run-e2e-tests.yml
index 52706a011..fc1471104 100644
--- a/.github/workflows/reuse-run-e2e-tests.yml
+++ b/.github/workflows/reuse-run-e2e-tests.yml
@@ -99,12 +99,12 @@ jobs:
           chmod -R a+w tmp/local/traces/v2/conflated
       - name: Create directory for cached docker images
         run: |
-          mkdir -p ~/docker-images
+          mkdir -p ./docker-images
       - name: Restore cached images
         id: restore-cached-images
         uses: actions/cache/restore@v4.2.0
         with:
-          path: ~/docker-images
+          path: ./docker-images
           key: cached-images
           restore-keys: |
             cached-images
diff --git a/.github/workflows/staterecovery-testing.yml b/.github/workflows/staterecovery-testing.yml
index 79adc484e..07ddd7716 100644
--- a/.github/workflows/staterecovery-testing.yml
+++ b/.github/workflows/staterecovery-testing.yml
@@ -39,12 +39,12 @@ jobs:
         uses: gradle/actions/setup-gradle@cc4fc85e6b35bafd578d5ffbc76a5518407e1af0 #v4.2.1
       - name: Create directory for cached docker images
         run: |
-          mkdir -p ~/docker-images
+          mkdir -p ./docker-images
       - name: Restore cached images
         id: restore-cached-images
         uses: actions/cache/restore@v4.2.0
         with:
-          path: ~/docker-images
+          path: ./docker-images
           key: cached-images
           restore-keys: |
             cached-images

From 87dddf2d26eecdf6c1bb6b00bc77456b2d7c957c Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Tue, 21 Jan 2025 23:29:14 +0800
Subject: [PATCH 10/34] feat: change actions cache version and keys

---
 .github/workflows/cache-docker-images.yml   | 8 ++++----
 .github/workflows/coordinator-testing.yml   | 8 ++++----
 .github/workflows/reuse-run-e2e-tests.yml   | 8 ++++----
 .github/workflows/staterecovery-testing.yml | 8 ++++----
 4 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/cache-docker-images.yml b/.github/workflows/cache-docker-images.yml
index 13e5ed1a9..3b95b692b 100644
--- a/.github/workflows/cache-docker-images.yml
+++ b/.github/workflows/cache-docker-images.yml
@@ -61,7 +61,7 @@ jobs:
           docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 pull
       - name: Save Docker images
         run: |
-          mkdir -p ./docker-images
+          mkdir -p ./cached-docker-images
           images=$(docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 config | grep "image:" | awk '{print $2}')
           for image in $images; do
             imageFileName=$(echo $image | sed -e 's|.*/||' -e 's|:|-|' -e 's/\./_/g')
@@ -70,7 +70,7 @@ jobs:
           done
       - name: Cache common docker images
         continue-on-error: true
-        uses: actions/cache/save@v4.2.0
+        uses: actions/cache/save@v4
         with:
-          path: ./docker-images
-          key: cached-images
+          path: ./cached-docker-images
+          key: cached-docker-images
diff --git a/.github/workflows/coordinator-testing.yml b/.github/workflows/coordinator-testing.yml
index f10c377f0..142b3f92d 100644
--- a/.github/workflows/coordinator-testing.yml
+++ b/.github/workflows/coordinator-testing.yml
@@ -42,12 +42,12 @@ jobs:
           mkdir -p ./docker-images
       - name: Restore cached images
         id: restore-cached-images
-        uses: actions/cache/restore@v4.2.0
+        uses: actions/cache/restore@v4
         with:
-          path: ./docker-images
-          key: cached-images
+          path: ./cached-docker-images
+          key: cached-docker-images
           restore-keys: |
-            cached-images
+            cached-docker-images
       # Install pnpm to compile smart contracts
       - name: Setup nodejs environment
         uses: ./.github/actions/setup-nodejs
diff --git a/.github/workflows/reuse-run-e2e-tests.yml b/.github/workflows/reuse-run-e2e-tests.yml
index fc1471104..e53676a7f 100644
--- a/.github/workflows/reuse-run-e2e-tests.yml
+++ b/.github/workflows/reuse-run-e2e-tests.yml
@@ -102,12 +102,12 @@ jobs:
           mkdir -p ./docker-images
       - name: Restore cached images
         id: restore-cached-images
-        uses: actions/cache/restore@v4.2.0
+        uses: actions/cache/restore@v4
         with:
-          path: ./docker-images
-          key: cached-images
+          path: ./cached-docker-images
+          key: cached-docker-images
           restore-keys: |
-            cached-images
+            cached-docker-images
       - name: Pull all images with retry
         uses: nick-fields/retry@v3
         with:
diff --git a/.github/workflows/staterecovery-testing.yml b/.github/workflows/staterecovery-testing.yml
index 07ddd7716..c0bc466ea 100644
--- a/.github/workflows/staterecovery-testing.yml
+++ b/.github/workflows/staterecovery-testing.yml
@@ -42,12 +42,12 @@ jobs:
           mkdir -p ./docker-images
       - name: Restore cached images
         id: restore-cached-images
-        uses: actions/cache/restore@v4.2.0
+        uses: actions/cache/restore@v4
         with:
-          path: ./docker-images
-          key: cached-images
+          path: ./cached-docker-images
+          key: cached-docker-images
           restore-keys: |
-            cached-images
+            cached-docker-images
       - name: Staterecovery - Build and Unit tests
         run: |
           ./gradlew state-recovery:besu-plugin:buildNeeded

From b2d81c77ba25e36677c5acf1702f8e4d6d5a5f6d Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Tue, 21 Jan 2025 23:39:13 +0800
Subject: [PATCH 11/34] fix: cache-docker-images folder name

---
 .github/workflows/cache-docker-images.yml   | 2 +-
 .github/workflows/coordinator-testing.yml   | 2 +-
 .github/workflows/reuse-run-e2e-tests.yml   | 2 +-
 .github/workflows/staterecovery-testing.yml | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/cache-docker-images.yml b/.github/workflows/cache-docker-images.yml
index 3b95b692b..18e107dbd 100644
--- a/.github/workflows/cache-docker-images.yml
+++ b/.github/workflows/cache-docker-images.yml
@@ -66,7 +66,7 @@ jobs:
           for image in $images; do
             imageFileName=$(echo $image | sed -e 's|.*/||' -e 's|:|-|' -e 's/\./_/g')
             echo $image - ${imageFileName}
-            docker save $image > ./docker-images/${imageFileName}.tar
+            docker save $image > ./cached-docker-images/${imageFileName}.tar
           done
       - name: Cache common docker images
         continue-on-error: true
diff --git a/.github/workflows/coordinator-testing.yml b/.github/workflows/coordinator-testing.yml
index 142b3f92d..3e8c5eb70 100644
--- a/.github/workflows/coordinator-testing.yml
+++ b/.github/workflows/coordinator-testing.yml
@@ -39,7 +39,7 @@ jobs:
         uses: gradle/actions/setup-gradle@cc4fc85e6b35bafd578d5ffbc76a5518407e1af0 #v4.2.1
       - name: Create directory for cached docker images
         run: |
-          mkdir -p ./docker-images
+          mkdir -p ./cached-docker-images
       - name: Restore cached images
         id: restore-cached-images
         uses: actions/cache/restore@v4
diff --git a/.github/workflows/reuse-run-e2e-tests.yml b/.github/workflows/reuse-run-e2e-tests.yml
index e53676a7f..2aff149d6 100644
--- a/.github/workflows/reuse-run-e2e-tests.yml
+++ b/.github/workflows/reuse-run-e2e-tests.yml
@@ -99,7 +99,7 @@ jobs:
           chmod -R a+w tmp/local/traces/v2/conflated
       - name: Create directory for cached docker images
         run: |
-          mkdir -p ./docker-images
+          mkdir -p ./cached-docker-images
       - name: Restore cached images
         id: restore-cached-images
         uses: actions/cache/restore@v4
diff --git a/.github/workflows/staterecovery-testing.yml b/.github/workflows/staterecovery-testing.yml
index c0bc466ea..c88f28e60 100644
--- a/.github/workflows/staterecovery-testing.yml
+++ b/.github/workflows/staterecovery-testing.yml
@@ -39,7 +39,7 @@ jobs:
         uses: gradle/actions/setup-gradle@cc4fc85e6b35bafd578d5ffbc76a5518407e1af0 #v4.2.1
       - name: Create directory for cached docker images
         run: |
-          mkdir -p ./docker-images
+          mkdir -p ./cached-docker-images
       - name: Restore cached images
         id: restore-cached-images
         uses: actions/cache/restore@v4

From 25d5f640ce6f39a15647ab1d42e819a8c9418c2f Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Tue, 21 Jan 2025 23:58:44 +0800
Subject: [PATCH 12/34] feat: temp disable cache-docker-images

---
 .github/workflows/testing.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index b1bac977e..3f5fac9d7 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -26,9 +26,9 @@ on:
         type: string
 
 jobs:
-  cache-docker-images:
-    uses: ./.github/workflows/cache-docker-images.yml
-    secrets: inherit
+  # cache-docker-images:
+  #   uses: ./.github/workflows/cache-docker-images.yml
+  #   secrets: inherit
   
   coordinator:
     uses: ./.github/workflows/coordinator-testing.yml

From f232c4a380efb0239db0d115513fc326e94a78c6 Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Wed, 22 Jan 2025 02:23:34 +0800
Subject: [PATCH 13/34] feat: makeover with cache-docker-images and
 restore-cached-images

---
 .github/workflows/cache-docker-images.yml   | 68 +++++++++++++++------
 .github/workflows/coordinator-testing.yml   | 10 +--
 .github/workflows/restore-docker-images.yml | 34 +++++++++++
 .github/workflows/reuse-run-e2e-tests.yml   | 10 +--
 .github/workflows/staterecovery-testing.yml | 10 +--
 .github/workflows/testing.yml               |  6 +-
 6 files changed, 91 insertions(+), 47 deletions(-)
 create mode 100644 .github/workflows/restore-docker-images.yml

diff --git a/.github/workflows/cache-docker-images.yml b/.github/workflows/cache-docker-images.yml
index 18e107dbd..eea53df9c 100644
--- a/.github/workflows/cache-docker-images.yml
+++ b/.github/workflows/cache-docker-images.yml
@@ -50,27 +50,61 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+      - name: Compute docker images hash
+        id: compute-docker-images-hash
+        run: |
+          docker_images=$(docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 config 2>/dev/null | grep "image:" | awk '{print $2}')
+          docker_images_unique=$(echo $docker_images | sort -u | xargs)
+          echo DOCKER_IMAGES=$(echo -n $docker_images_unique) >> $GITHUB_OUTPUT
+          echo DOCKER_IMAGES_HASH=$(echo -n $docker_images_unique | sha256sum | awk '{print $1}') >> $GITHUB_OUTPUT
+      - name: Show docker images hash
+        id: show-docker-images-hash
+        run: |
+          echo "DOCKER_IMAGES: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES }}"
+          echo "DOCKER_IMAGES_HASH: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_HASH }}"
+      - name: Create cached docker images folder
+        run: |
+          mkdir -p ~/docker-images-cache
+      - name: Cache common docker images
+        id: cache-common-docker-images
+        continue-on-error: true
+        uses: actions/cache@v4
+        with:
+          path: ~/docker-images-cache
+          key: docker-images-cache-${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_HASH }}
       - name: Login to Docker Hub
+        if: steps.cache-common-docker-images.outputs.cache-hit != 'true'
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Pull docker images from compose
-        continue-on-error: true
+      - name: Pull and save docker images
+        if: steps.cache-common-docker-images.outputs.cache-hit != 'true'
         run: |
           docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 pull
-      - name: Save Docker images
-        run: |
-          mkdir -p ./cached-docker-images
-          images=$(docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 config | grep "image:" | awk '{print $2}')
-          for image in $images; do
-            imageFileName=$(echo $image | sed -e 's|.*/||' -e 's|:|-|' -e 's/\./_/g')
-            echo $image - ${imageFileName}
-            docker save $image > ./cached-docker-images/${imageFileName}.tar
-          done
-      - name: Cache common docker images
-        continue-on-error: true
-        uses: actions/cache/save@v4
-        with:
-          path: ./cached-docker-images
-          key: cached-docker-images
+          docker save ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES }} -o  ~/docker-images-cache/docker-images.tar
+          
+      # - name: Login to Docker Hub
+      #   uses: docker/login-action@v3
+      #   with:
+      #     username: ${{ secrets.DOCKERHUB_USERNAME }}
+      #     password: ${{ secrets.DOCKERHUB_TOKEN }}
+      # - name: Pull docker images from compose
+      #   continue-on-error: true
+      #   run: |
+      #     docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 pull
+      # - name: Save Docker images
+      #   run: |
+      #     mkdir -p ./cached-docker-images
+      #     images=$(docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 config | grep "image:" | awk '{print $2}')
+      #     for image in $images; do
+      #       imageFileName=$(echo $image | sed -e 's|.*/||' -e 's|:|-|' -e 's/\./_/g')
+      #       echo $image - ${imageFileName}
+      #       docker save $image > ./cached-docker-images/${imageFileName}.tar
+      #     done
+      # - name: Cache common docker images
+      #   continue-on-error: true
+      #   uses: actions/cache/save@v4
+      #   with:
+      #     path: ./cached-docker-images
+      #     key: cached-docker-images
diff --git a/.github/workflows/coordinator-testing.yml b/.github/workflows/coordinator-testing.yml
index 3e8c5eb70..0e5e96a7b 100644
--- a/.github/workflows/coordinator-testing.yml
+++ b/.github/workflows/coordinator-testing.yml
@@ -37,17 +37,9 @@ jobs:
         # Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies.
         # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md
         uses: gradle/actions/setup-gradle@cc4fc85e6b35bafd578d5ffbc76a5518407e1af0 #v4.2.1
-      - name: Create directory for cached docker images
-        run: |
-          mkdir -p ./cached-docker-images
       - name: Restore cached images
         id: restore-cached-images
-        uses: actions/cache/restore@v4
-        with:
-          path: ./cached-docker-images
-          key: cached-docker-images
-          restore-keys: |
-            cached-docker-images
+        uses: ./.github/workflows/restore-docker-images.yml
       # Install pnpm to compile smart contracts
       - name: Setup nodejs environment
         uses: ./.github/actions/setup-nodejs
diff --git a/.github/workflows/restore-docker-images.yml b/.github/workflows/restore-docker-images.yml
new file mode 100644
index 000000000..d98be0bea
--- /dev/null
+++ b/.github/workflows/restore-docker-images.yml
@@ -0,0 +1,34 @@
+name: Restore Docker Images CI
+on:
+  workflow_call:
+
+jobs:
+  restore-cached-images:
+    runs-on: [self-hosted, ubuntu-20.04, X64, small]
+    steps:
+      - name: Compute docker images hash
+        id: compute-docker-images-hash
+        run: |
+          docker_images=$(docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 config 2>/dev/null | grep "image:" | awk '{print $2}')
+          docker_images_unique=$(echo $docker_images | sort -u | xargs)
+          echo DOCKER_IMAGES=$(echo -n $docker_images_unique) >> $GITHUB_OUTPUT
+          echo DOCKER_IMAGES_HASH=$(echo -n $docker_images_unique | sha256sum | awk '{print $1}') >> $GITHUB_OUTPUT
+      - name: Show docker images hash
+        id: show-docker-images-hash
+        run: |
+          echo "DOCKER_IMAGES: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES }}"
+          echo "DOCKER_IMAGES_HASH: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_HASH }}"
+      - name: Create cached docker images folder
+        run: |
+          mkdir -p ~/docker-images-cache
+      - name: Cache common docker images
+        id: cache-common-docker-images
+        continue-on-error: true
+        uses: actions/cache@v4
+        with:
+          path: ~/docker-images-cache
+          key: docker-images-cache-${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_HASH }}
+      - name: Load docker images
+        if: steps.cache-common-docker-images.outputs.cache-hit == 'true'
+        run: |
+          docker load -i ~/docker-images-cache/docker-images.tar
diff --git a/.github/workflows/reuse-run-e2e-tests.yml b/.github/workflows/reuse-run-e2e-tests.yml
index 2aff149d6..703cbaa80 100644
--- a/.github/workflows/reuse-run-e2e-tests.yml
+++ b/.github/workflows/reuse-run-e2e-tests.yml
@@ -97,17 +97,9 @@ jobs:
         run: |
           mkdir -p tmp/local/traces/v2/conflated
           chmod -R a+w tmp/local/traces/v2/conflated
-      - name: Create directory for cached docker images
-        run: |
-          mkdir -p ./cached-docker-images
       - name: Restore cached images
         id: restore-cached-images
-        uses: actions/cache/restore@v4
-        with:
-          path: ./cached-docker-images
-          key: cached-docker-images
-          restore-keys: |
-            cached-docker-images
+        uses: ./.github/workflows/restore-docker-images.yml
       - name: Pull all images with retry
         uses: nick-fields/retry@v3
         with:
diff --git a/.github/workflows/staterecovery-testing.yml b/.github/workflows/staterecovery-testing.yml
index c88f28e60..6e0fd3ed9 100644
--- a/.github/workflows/staterecovery-testing.yml
+++ b/.github/workflows/staterecovery-testing.yml
@@ -37,17 +37,9 @@ jobs:
         # Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies.
         # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md
         uses: gradle/actions/setup-gradle@cc4fc85e6b35bafd578d5ffbc76a5518407e1af0 #v4.2.1
-      - name: Create directory for cached docker images
-        run: |
-          mkdir -p ./cached-docker-images
       - name: Restore cached images
         id: restore-cached-images
-        uses: actions/cache/restore@v4
-        with:
-          path: ./cached-docker-images
-          key: cached-docker-images
-          restore-keys: |
-            cached-docker-images
+        uses: ./.github/workflows/restore-docker-images.yml
       - name: Staterecovery - Build and Unit tests
         run: |
           ./gradlew state-recovery:besu-plugin:buildNeeded
diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index 3f5fac9d7..b1bac977e 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -26,9 +26,9 @@ on:
         type: string
 
 jobs:
-  # cache-docker-images:
-  #   uses: ./.github/workflows/cache-docker-images.yml
-  #   secrets: inherit
+  cache-docker-images:
+    uses: ./.github/workflows/cache-docker-images.yml
+    secrets: inherit
   
   coordinator:
     uses: ./.github/workflows/coordinator-testing.yml

From 45a27ad3789e6e59acf3da61d0c7202ffcd1c4dc Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Wed, 22 Jan 2025 03:29:19 +0800
Subject: [PATCH 14/34] feat: put restore-cached-images in actions folder

---
 .../actions/restore-docker-images/action.yml  | 38 +++++++++++++++++++
 .github/workflows/cache-docker-images.yml     |  6 ++-
 .github/workflows/coordinator-testing.yml     |  2 +-
 .github/workflows/restore-docker-images.yml   | 34 -----------------
 .github/workflows/reuse-run-e2e-tests.yml     |  2 +-
 .github/workflows/staterecovery-testing.yml   |  2 +-
 6 files changed, 45 insertions(+), 39 deletions(-)
 create mode 100644 .github/actions/restore-docker-images/action.yml
 delete mode 100644 .github/workflows/restore-docker-images.yml

diff --git a/.github/actions/restore-docker-images/action.yml b/.github/actions/restore-docker-images/action.yml
new file mode 100644
index 000000000..6ea3991b8
--- /dev/null
+++ b/.github/actions/restore-docker-images/action.yml
@@ -0,0 +1,38 @@
+name: 'Restore Cached Docker Images'
+description: 'Restore Cached Docker Images'
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Compute docker images hash
+      id: compute-docker-images-hash
+      shell: bash
+      run: |
+        docker_images=$(docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 config 2>/dev/null | grep "image:" | awk '{print $2}')
+        docker_images_unique=$(echo $docker_images | sort -u | xargs)
+        echo DOCKER_IMAGES=$(echo $docker_images) >> $GITHUB_OUTPUT
+        echo DOCKER_IMAGES_UNIQUE=$(echo -n $docker_images_unique) >> $GITHUB_OUTPUT
+        echo DOCKER_IMAGES_HASH=$(echo -n $docker_images_unique | sha256sum | awk '{print $1}') >> $GITHUB_OUTPUT
+    - name: Show docker images hash
+      shell: bash
+      id: show-docker-images-hash
+      run: |
+        echo "DOCKER_IMAGES: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES }}"
+        echo "DOCKER_IMAGES_UNIQUE: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_UNIQUE }}"
+        echo "DOCKER_IMAGES_HASH: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_HASH }}"
+    - name: Create cached docker images folder
+      shell: bash
+      run: |
+        mkdir -p ~/docker-images-cache
+    - name: Cache common docker images
+      id: cache-common-docker-images
+      continue-on-error: true
+      uses: actions/cache@v4
+      with:
+        path: ~/docker-images-cache
+        key: docker-images-cache-${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_HASH }}
+    - name: Load docker images
+      shell: bash
+      if: steps.cache-common-docker-images.outputs.cache-hit == 'true'
+      run: |
+        docker load -i ~/docker-images-cache/docker-images.tar
diff --git a/.github/workflows/cache-docker-images.yml b/.github/workflows/cache-docker-images.yml
index eea53df9c..09840e252 100644
--- a/.github/workflows/cache-docker-images.yml
+++ b/.github/workflows/cache-docker-images.yml
@@ -55,12 +55,14 @@ jobs:
         run: |
           docker_images=$(docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 config 2>/dev/null | grep "image:" | awk '{print $2}')
           docker_images_unique=$(echo $docker_images | sort -u | xargs)
-          echo DOCKER_IMAGES=$(echo -n $docker_images_unique) >> $GITHUB_OUTPUT
+          echo DOCKER_IMAGES=$(echo $docker_images) >> $GITHUB_OUTPUT
+          echo DOCKER_IMAGES_UNIQUE=$(echo -n $docker_images_unique) >> $GITHUB_OUTPUT
           echo DOCKER_IMAGES_HASH=$(echo -n $docker_images_unique | sha256sum | awk '{print $1}') >> $GITHUB_OUTPUT
       - name: Show docker images hash
         id: show-docker-images-hash
         run: |
           echo "DOCKER_IMAGES: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES }}"
+          echo "DOCKER_IMAGES_UNIQUE: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_UNIQUE }}"
           echo "DOCKER_IMAGES_HASH: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_HASH }}"
       - name: Create cached docker images folder
         run: |
@@ -83,7 +85,7 @@ jobs:
         run: |
           docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 pull
           docker save ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES }} -o  ~/docker-images-cache/docker-images.tar
-          
+
       # - name: Login to Docker Hub
       #   uses: docker/login-action@v3
       #   with:
diff --git a/.github/workflows/coordinator-testing.yml b/.github/workflows/coordinator-testing.yml
index 0e5e96a7b..69e83d9e6 100644
--- a/.github/workflows/coordinator-testing.yml
+++ b/.github/workflows/coordinator-testing.yml
@@ -39,7 +39,7 @@ jobs:
         uses: gradle/actions/setup-gradle@cc4fc85e6b35bafd578d5ffbc76a5518407e1af0 #v4.2.1
       - name: Restore cached images
         id: restore-cached-images
-        uses: ./.github/workflows/restore-docker-images.yml
+        uses: ./.github/actions/restore-docker-images
       # Install pnpm to compile smart contracts
       - name: Setup nodejs environment
         uses: ./.github/actions/setup-nodejs
diff --git a/.github/workflows/restore-docker-images.yml b/.github/workflows/restore-docker-images.yml
deleted file mode 100644
index d98be0bea..000000000
--- a/.github/workflows/restore-docker-images.yml
+++ /dev/null
@@ -1,34 +0,0 @@
-name: Restore Docker Images CI
-on:
-  workflow_call:
-
-jobs:
-  restore-cached-images:
-    runs-on: [self-hosted, ubuntu-20.04, X64, small]
-    steps:
-      - name: Compute docker images hash
-        id: compute-docker-images-hash
-        run: |
-          docker_images=$(docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 config 2>/dev/null | grep "image:" | awk '{print $2}')
-          docker_images_unique=$(echo $docker_images | sort -u | xargs)
-          echo DOCKER_IMAGES=$(echo -n $docker_images_unique) >> $GITHUB_OUTPUT
-          echo DOCKER_IMAGES_HASH=$(echo -n $docker_images_unique | sha256sum | awk '{print $1}') >> $GITHUB_OUTPUT
-      - name: Show docker images hash
-        id: show-docker-images-hash
-        run: |
-          echo "DOCKER_IMAGES: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES }}"
-          echo "DOCKER_IMAGES_HASH: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_HASH }}"
-      - name: Create cached docker images folder
-        run: |
-          mkdir -p ~/docker-images-cache
-      - name: Cache common docker images
-        id: cache-common-docker-images
-        continue-on-error: true
-        uses: actions/cache@v4
-        with:
-          path: ~/docker-images-cache
-          key: docker-images-cache-${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_HASH }}
-      - name: Load docker images
-        if: steps.cache-common-docker-images.outputs.cache-hit == 'true'
-        run: |
-          docker load -i ~/docker-images-cache/docker-images.tar
diff --git a/.github/workflows/reuse-run-e2e-tests.yml b/.github/workflows/reuse-run-e2e-tests.yml
index 703cbaa80..2109d046d 100644
--- a/.github/workflows/reuse-run-e2e-tests.yml
+++ b/.github/workflows/reuse-run-e2e-tests.yml
@@ -99,7 +99,7 @@ jobs:
           chmod -R a+w tmp/local/traces/v2/conflated
       - name: Restore cached images
         id: restore-cached-images
-        uses: ./.github/workflows/restore-docker-images.yml
+        uses: ./.github/actions/restore-docker-images
       - name: Pull all images with retry
         uses: nick-fields/retry@v3
         with:
diff --git a/.github/workflows/staterecovery-testing.yml b/.github/workflows/staterecovery-testing.yml
index 6e0fd3ed9..15259955c 100644
--- a/.github/workflows/staterecovery-testing.yml
+++ b/.github/workflows/staterecovery-testing.yml
@@ -39,7 +39,7 @@ jobs:
         uses: gradle/actions/setup-gradle@cc4fc85e6b35bafd578d5ffbc76a5518407e1af0 #v4.2.1
       - name: Restore cached images
         id: restore-cached-images
-        uses: ./.github/workflows/restore-docker-images.yml
+        uses: ./.github/actions/restore-docker-images
       - name: Staterecovery - Build and Unit tests
         run: |
           ./gradlew state-recovery:besu-plugin:buildNeeded

From 1476117a8944559c5d80802dd5e47fee9c6c2a52 Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Wed, 22 Jan 2025 12:09:55 +0800
Subject: [PATCH 15/34] feat: change runner for save cache step and revise hash

---
 .github/actions/restore-docker-images/action.yml | 2 +-
 .github/workflows/cache-docker-images.yml        | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/actions/restore-docker-images/action.yml b/.github/actions/restore-docker-images/action.yml
index 6ea3991b8..354ca085d 100644
--- a/.github/actions/restore-docker-images/action.yml
+++ b/.github/actions/restore-docker-images/action.yml
@@ -9,7 +9,7 @@ runs:
       shell: bash
       run: |
         docker_images=$(docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 config 2>/dev/null | grep "image:" | awk '{print $2}')
-        docker_images_unique=$(echo $docker_images | sort -u | xargs)
+        docker_images_unique=$(echo $docker_images | xargs -n1 | sort -u | xargs)
         echo DOCKER_IMAGES=$(echo $docker_images) >> $GITHUB_OUTPUT
         echo DOCKER_IMAGES_UNIQUE=$(echo -n $docker_images_unique) >> $GITHUB_OUTPUT
         echo DOCKER_IMAGES_HASH=$(echo -n $docker_images_unique | sha256sum | awk '{print $1}') >> $GITHUB_OUTPUT
diff --git a/.github/workflows/cache-docker-images.yml b/.github/workflows/cache-docker-images.yml
index 09840e252..792e66bc4 100644
--- a/.github/workflows/cache-docker-images.yml
+++ b/.github/workflows/cache-docker-images.yml
@@ -54,7 +54,7 @@ jobs:
         id: compute-docker-images-hash
         run: |
           docker_images=$(docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 config 2>/dev/null | grep "image:" | awk '{print $2}')
-          docker_images_unique=$(echo $docker_images | sort -u | xargs)
+          docker_images_unique=$(echo $docker_images | xargs -n1 | sort -u | xargs)
           echo DOCKER_IMAGES=$(echo $docker_images) >> $GITHUB_OUTPUT
           echo DOCKER_IMAGES_UNIQUE=$(echo -n $docker_images_unique) >> $GITHUB_OUTPUT
           echo DOCKER_IMAGES_HASH=$(echo -n $docker_images_unique | sha256sum | awk '{print $1}') >> $GITHUB_OUTPUT
@@ -69,11 +69,12 @@ jobs:
           mkdir -p ~/docker-images-cache
       - name: Cache common docker images
         id: cache-common-docker-images
-        continue-on-error: true
+        # continue-on-error: true
         uses: actions/cache@v4
         with:
           path: ~/docker-images-cache
           key: docker-images-cache-${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_HASH }}
+          lookup-only: true
       - name: Login to Docker Hub
         if: steps.cache-common-docker-images.outputs.cache-hit != 'true'
         uses: docker/login-action@v3

From 10536a79300114fa33ee0a757badc1d7f9690bcb Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Wed, 22 Jan 2025 14:15:00 +0800
Subject: [PATCH 16/34] feat: testing with diff cache path

---
 .../actions/restore-docker-images/action.yml    |  6 +++---
 .github/workflows/cache-docker-images.yml       | 17 +++++++++++++----
 2 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/.github/actions/restore-docker-images/action.yml b/.github/actions/restore-docker-images/action.yml
index 354ca085d..f7fd27e8c 100644
--- a/.github/actions/restore-docker-images/action.yml
+++ b/.github/actions/restore-docker-images/action.yml
@@ -23,16 +23,16 @@ runs:
     - name: Create cached docker images folder
       shell: bash
       run: |
-        mkdir -p ~/docker-images-cache
+        mkdir -p ~/docker-images-cached
     - name: Cache common docker images
       id: cache-common-docker-images
       continue-on-error: true
       uses: actions/cache@v4
       with:
-        path: ~/docker-images-cache
+        path: ~/docker-images-cached
         key: docker-images-cache-${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_HASH }}
     - name: Load docker images
       shell: bash
       if: steps.cache-common-docker-images.outputs.cache-hit == 'true'
       run: |
-        docker load -i ~/docker-images-cache/docker-images.tar
+        docker load -i ~/docker-images-cached/docker-images.tar
diff --git a/.github/workflows/cache-docker-images.yml b/.github/workflows/cache-docker-images.yml
index 792e66bc4..42ee5da45 100644
--- a/.github/workflows/cache-docker-images.yml
+++ b/.github/workflows/cache-docker-images.yml
@@ -7,6 +7,12 @@ on:
       DOCKERHUB_TOKEN:
         required: false
 
+permissions:
+  contents: write
+  issues: write
+  pull-requests: write
+  actions: write
+
 jobs:
   check-dockerhub-secrets-present:
     runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-small
@@ -66,13 +72,13 @@ jobs:
           echo "DOCKER_IMAGES_HASH: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_HASH }}"
       - name: Create cached docker images folder
         run: |
-          mkdir -p ~/docker-images-cache
+          mkdir -p ~/docker-images-cached
       - name: Cache common docker images
         id: cache-common-docker-images
         # continue-on-error: true
         uses: actions/cache@v4
         with:
-          path: ~/docker-images-cache
+          path: ~/docker-images-cached
           key: docker-images-cache-${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_HASH }}
           lookup-only: true
       - name: Login to Docker Hub
@@ -81,11 +87,14 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Pull and save docker images
+      - name: Pull docker images
         if: steps.cache-common-docker-images.outputs.cache-hit != 'true'
         run: |
           docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 pull
-          docker save ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES }} -o  ~/docker-images-cache/docker-images.tar
+      - name: Save docker images
+        if: steps.cache-common-docker-images.outputs.cache-hit != 'true'
+        run: |
+          docker save ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_UNIQUE }} -o  ~/docker-images-cached/docker-images.tar
 
       # - name: Login to Docker Hub
       #   uses: docker/login-action@v3

From 8d07d11159435a3324115f9db6b890b1c3447a71 Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Wed, 22 Jan 2025 14:25:46 +0800
Subject: [PATCH 17/34] feat: testing with diff hash

---
 .github/actions/restore-docker-images/action.yml | 2 +-
 .github/workflows/cache-docker-images.yml        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/actions/restore-docker-images/action.yml b/.github/actions/restore-docker-images/action.yml
index f7fd27e8c..46dae2488 100644
--- a/.github/actions/restore-docker-images/action.yml
+++ b/.github/actions/restore-docker-images/action.yml
@@ -12,7 +12,7 @@ runs:
         docker_images_unique=$(echo $docker_images | xargs -n1 | sort -u | xargs)
         echo DOCKER_IMAGES=$(echo $docker_images) >> $GITHUB_OUTPUT
         echo DOCKER_IMAGES_UNIQUE=$(echo -n $docker_images_unique) >> $GITHUB_OUTPUT
-        echo DOCKER_IMAGES_HASH=$(echo -n $docker_images_unique | sha256sum | awk '{print $1}') >> $GITHUB_OUTPUT
+        echo DOCKER_IMAGES_HASH=$(echo $docker_images_unique | sha256sum | awk '{print $1}') >> $GITHUB_OUTPUT
     - name: Show docker images hash
       shell: bash
       id: show-docker-images-hash
diff --git a/.github/workflows/cache-docker-images.yml b/.github/workflows/cache-docker-images.yml
index 42ee5da45..ad9c7dc44 100644
--- a/.github/workflows/cache-docker-images.yml
+++ b/.github/workflows/cache-docker-images.yml
@@ -63,7 +63,7 @@ jobs:
           docker_images_unique=$(echo $docker_images | xargs -n1 | sort -u | xargs)
           echo DOCKER_IMAGES=$(echo $docker_images) >> $GITHUB_OUTPUT
           echo DOCKER_IMAGES_UNIQUE=$(echo -n $docker_images_unique) >> $GITHUB_OUTPUT
-          echo DOCKER_IMAGES_HASH=$(echo -n $docker_images_unique | sha256sum | awk '{print $1}') >> $GITHUB_OUTPUT
+          echo DOCKER_IMAGES_HASH=$(echo $docker_images_unique | sha256sum | awk '{print $1}') >> $GITHUB_OUTPUT
       - name: Show docker images hash
         id: show-docker-images-hash
         run: |

From 730d601e4cbdc0fe6d0cb3d6ba730720ccaf1f94 Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Wed, 22 Jan 2025 14:42:02 +0800
Subject: [PATCH 18/34] feat: testing with original runner

---
 .github/workflows/cache-docker-images.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/cache-docker-images.yml b/.github/workflows/cache-docker-images.yml
index ad9c7dc44..a3d0e3dd1 100644
--- a/.github/workflows/cache-docker-images.yml
+++ b/.github/workflows/cache-docker-images.yml
@@ -95,6 +95,7 @@ jobs:
         if: steps.cache-common-docker-images.outputs.cache-hit != 'true'
         run: |
           docker save ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_UNIQUE }} -o  ~/docker-images-cached/docker-images.tar
+          echo docker-images.tar=$(ls -lh ~/docker-images-cached/docker-images.tar)
 
       # - name: Login to Docker Hub
       #   uses: docker/login-action@v3

From 33bd01fe9fc071063592a85103121396d20be9b2 Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Wed, 22 Jan 2025 15:32:46 +0800
Subject: [PATCH 19/34] feat: added composite action for Compute Docker Images
 Hash

---
 .../compute-docker-images-hash/action.yml     | 30 +++++++++++++
 .../actions/restore-docker-images/action.yml  | 17 +-------
 .github/workflows/cache-docker-images.yml     | 43 ++-----------------
 .github/workflows/coordinator-testing.yml     |  1 +
 .github/workflows/reuse-run-e2e-tests.yml     |  1 +
 .github/workflows/staterecovery-testing.yml   |  1 +
 6 files changed, 38 insertions(+), 55 deletions(-)
 create mode 100644 .github/actions/compute-docker-images-hash/action.yml

diff --git a/.github/actions/compute-docker-images-hash/action.yml b/.github/actions/compute-docker-images-hash/action.yml
new file mode 100644
index 000000000..bdca88885
--- /dev/null
+++ b/.github/actions/compute-docker-images-hash/action.yml
@@ -0,0 +1,30 @@
+name: 'Compute Docker Images Hash'
+description: 'Compute Docker Images Hash for Cache Key'
+
+outputs:
+  docker_images_hash:
+    description: 'Docker Images Hash'
+    value: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_HASH }}
+  docker_images_unique:
+    description: 'Unique List of Docker Images'
+    value: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_UNIQUE }}
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Compute docker images hash
+      id: compute-docker-images-hash
+      shell: bash
+      run: |
+        docker_images=$(docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 config 2>/dev/null | grep "image:" | awk '{print $2}')
+        docker_images_unique=$(echo $docker_images | xargs -n1 | sort -u | xargs)
+        echo DOCKER_IMAGES=$(echo $docker_images) >> $GITHUB_OUTPUT
+        echo DOCKER_IMAGES_UNIQUE=$(echo -n $docker_images_unique) >> $GITHUB_OUTPUT
+        echo DOCKER_IMAGES_HASH=$(echo -n $docker_images_unique | sha256sum | awk '{print $1}') >> $GITHUB_OUTPUT
+    - name: Show docker images hash
+      shell: bash
+      id: show-docker-images-hash
+      run: |
+        echo "DOCKER_IMAGES: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES }}"
+        echo "DOCKER_IMAGES_UNIQUE: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_UNIQUE }}"
+        echo "DOCKER_IMAGES_HASH: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_HASH }}"
diff --git a/.github/actions/restore-docker-images/action.yml b/.github/actions/restore-docker-images/action.yml
index 46dae2488..17f156bc1 100644
--- a/.github/actions/restore-docker-images/action.yml
+++ b/.github/actions/restore-docker-images/action.yml
@@ -6,20 +6,7 @@ runs:
   steps:
     - name: Compute docker images hash
       id: compute-docker-images-hash
-      shell: bash
-      run: |
-        docker_images=$(docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 config 2>/dev/null | grep "image:" | awk '{print $2}')
-        docker_images_unique=$(echo $docker_images | xargs -n1 | sort -u | xargs)
-        echo DOCKER_IMAGES=$(echo $docker_images) >> $GITHUB_OUTPUT
-        echo DOCKER_IMAGES_UNIQUE=$(echo -n $docker_images_unique) >> $GITHUB_OUTPUT
-        echo DOCKER_IMAGES_HASH=$(echo $docker_images_unique | sha256sum | awk '{print $1}') >> $GITHUB_OUTPUT
-    - name: Show docker images hash
-      shell: bash
-      id: show-docker-images-hash
-      run: |
-        echo "DOCKER_IMAGES: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES }}"
-        echo "DOCKER_IMAGES_UNIQUE: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_UNIQUE }}"
-        echo "DOCKER_IMAGES_HASH: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_HASH }}"
+      uses: ./.github/actions/compute-docker-images-hash
     - name: Create cached docker images folder
       shell: bash
       run: |
@@ -30,7 +17,7 @@ runs:
       uses: actions/cache@v4
       with:
         path: ~/docker-images-cached
-        key: docker-images-cache-${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_HASH }}
+        key: docker-images-cache-${{ steps.compute-docker-images-hash.outputs.docker_images_hash }}
     - name: Load docker images
       shell: bash
       if: steps.cache-common-docker-images.outputs.cache-hit == 'true'
diff --git a/.github/workflows/cache-docker-images.yml b/.github/workflows/cache-docker-images.yml
index a3d0e3dd1..ffa02b238 100644
--- a/.github/workflows/cache-docker-images.yml
+++ b/.github/workflows/cache-docker-images.yml
@@ -58,28 +58,16 @@ jobs:
         uses: actions/checkout@v4
       - name: Compute docker images hash
         id: compute-docker-images-hash
-        run: |
-          docker_images=$(docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 config 2>/dev/null | grep "image:" | awk '{print $2}')
-          docker_images_unique=$(echo $docker_images | xargs -n1 | sort -u | xargs)
-          echo DOCKER_IMAGES=$(echo $docker_images) >> $GITHUB_OUTPUT
-          echo DOCKER_IMAGES_UNIQUE=$(echo -n $docker_images_unique) >> $GITHUB_OUTPUT
-          echo DOCKER_IMAGES_HASH=$(echo $docker_images_unique | sha256sum | awk '{print $1}') >> $GITHUB_OUTPUT
-      - name: Show docker images hash
-        id: show-docker-images-hash
-        run: |
-          echo "DOCKER_IMAGES: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES }}"
-          echo "DOCKER_IMAGES_UNIQUE: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_UNIQUE }}"
-          echo "DOCKER_IMAGES_HASH: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_HASH }}"
+        uses: ./.github/actions/compute-docker-images-hash
       - name: Create cached docker images folder
         run: |
           mkdir -p ~/docker-images-cached
       - name: Cache common docker images
         id: cache-common-docker-images
-        # continue-on-error: true
         uses: actions/cache@v4
         with:
           path: ~/docker-images-cached
-          key: docker-images-cache-${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_HASH }}
+          key: docker-images-cache-${{ steps.compute-docker-images-hash.outputs.docker_images_hash }}
           lookup-only: true
       - name: Login to Docker Hub
         if: steps.cache-common-docker-images.outputs.cache-hit != 'true'
@@ -94,30 +82,5 @@ jobs:
       - name: Save docker images
         if: steps.cache-common-docker-images.outputs.cache-hit != 'true'
         run: |
-          docker save ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_UNIQUE }} -o  ~/docker-images-cached/docker-images.tar
+          docker save ${{ steps.compute-docker-images-hash.outputs.docker_images_unique }} -o  ~/docker-images-cached/docker-images.tar
           echo docker-images.tar=$(ls -lh ~/docker-images-cached/docker-images.tar)
-
-      # - name: Login to Docker Hub
-      #   uses: docker/login-action@v3
-      #   with:
-      #     username: ${{ secrets.DOCKERHUB_USERNAME }}
-      #     password: ${{ secrets.DOCKERHUB_TOKEN }}
-      # - name: Pull docker images from compose
-      #   continue-on-error: true
-      #   run: |
-      #     docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 pull
-      # - name: Save Docker images
-      #   run: |
-      #     mkdir -p ./cached-docker-images
-      #     images=$(docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 config | grep "image:" | awk '{print $2}')
-      #     for image in $images; do
-      #       imageFileName=$(echo $image | sed -e 's|.*/||' -e 's|:|-|' -e 's/\./_/g')
-      #       echo $image - ${imageFileName}
-      #       docker save $image > ./cached-docker-images/${imageFileName}.tar
-      #     done
-      # - name: Cache common docker images
-      #   continue-on-error: true
-      #   uses: actions/cache/save@v4
-      #   with:
-      #     path: ./cached-docker-images
-      #     key: cached-docker-images
diff --git a/.github/workflows/coordinator-testing.yml b/.github/workflows/coordinator-testing.yml
index 69e83d9e6..22ab36c18 100644
--- a/.github/workflows/coordinator-testing.yml
+++ b/.github/workflows/coordinator-testing.yml
@@ -40,6 +40,7 @@ jobs:
       - name: Restore cached images
         id: restore-cached-images
         uses: ./.github/actions/restore-docker-images
+        continue-on-error: true
       # Install pnpm to compile smart contracts
       - name: Setup nodejs environment
         uses: ./.github/actions/setup-nodejs
diff --git a/.github/workflows/reuse-run-e2e-tests.yml b/.github/workflows/reuse-run-e2e-tests.yml
index 2109d046d..4b00106ed 100644
--- a/.github/workflows/reuse-run-e2e-tests.yml
+++ b/.github/workflows/reuse-run-e2e-tests.yml
@@ -100,6 +100,7 @@ jobs:
       - name: Restore cached images
         id: restore-cached-images
         uses: ./.github/actions/restore-docker-images
+        continue-on-error: true
       - name: Pull all images with retry
         uses: nick-fields/retry@v3
         with:
diff --git a/.github/workflows/staterecovery-testing.yml b/.github/workflows/staterecovery-testing.yml
index 15259955c..f22164066 100644
--- a/.github/workflows/staterecovery-testing.yml
+++ b/.github/workflows/staterecovery-testing.yml
@@ -40,6 +40,7 @@ jobs:
       - name: Restore cached images
         id: restore-cached-images
         uses: ./.github/actions/restore-docker-images
+        continue-on-error: true
       - name: Staterecovery - Build and Unit tests
         run: |
           ./gradlew state-recovery:besu-plugin:buildNeeded

From 7579c42aed94f92fcaf99c077b590388951d0bf2 Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Wed, 22 Jan 2025 15:56:48 +0800
Subject: [PATCH 20/34] feat: remove action permissions and use cache restore

---
 .github/actions/restore-docker-images/action.yml | 2 +-
 .github/workflows/cache-docker-images.yml        | 6 ------
 2 files changed, 1 insertion(+), 7 deletions(-)

diff --git a/.github/actions/restore-docker-images/action.yml b/.github/actions/restore-docker-images/action.yml
index 17f156bc1..7691327da 100644
--- a/.github/actions/restore-docker-images/action.yml
+++ b/.github/actions/restore-docker-images/action.yml
@@ -14,7 +14,7 @@ runs:
     - name: Cache common docker images
       id: cache-common-docker-images
       continue-on-error: true
-      uses: actions/cache@v4
+      uses: actions/cache/restore@v4
       with:
         path: ~/docker-images-cached
         key: docker-images-cache-${{ steps.compute-docker-images-hash.outputs.docker_images_hash }}
diff --git a/.github/workflows/cache-docker-images.yml b/.github/workflows/cache-docker-images.yml
index ffa02b238..3d5fb73f2 100644
--- a/.github/workflows/cache-docker-images.yml
+++ b/.github/workflows/cache-docker-images.yml
@@ -7,12 +7,6 @@ on:
       DOCKERHUB_TOKEN:
         required: false
 
-permissions:
-  contents: write
-  issues: write
-  pull-requests: write
-  actions: write
-
 jobs:
   check-dockerhub-secrets-present:
     runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-small

From 7cbf62588344072431052b176d7d745bef2396a1 Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Wed, 22 Jan 2025 16:52:49 +0800
Subject: [PATCH 21/34] Revert "feat: added gauge metrics for batch, blob,
 aggregation size"

This reverts commit 4b1bfc4b27f5c5ee80ed1554625600d10e771f49.
---
 .../GlobalAggregationCalculator.kt            | 41 ++-----------------
 .../blob/BlobCompressionProofCoordinator.kt   | 13 ------
 .../conflation/ConflationServiceImpl.kt       |  9 +---
 3 files changed, 4 insertions(+), 59 deletions(-)

diff --git a/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/aggregation/GlobalAggregationCalculator.kt b/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/aggregation/GlobalAggregationCalculator.kt
index 2a1e97823..1ecf2e7f2 100644
--- a/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/aggregation/GlobalAggregationCalculator.kt
+++ b/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/aggregation/GlobalAggregationCalculator.kt
@@ -7,8 +7,8 @@ import net.consensys.zkevm.domain.BlobCounters
 import net.consensys.zkevm.domain.BlobsToAggregate
 import org.apache.logging.log4j.LogManager
 import org.apache.logging.log4j.Logger
+import java.lang.IllegalStateException
 import java.util.PriorityQueue
-import java.util.concurrent.atomic.AtomicInteger
 
 class GlobalAggregationCalculator(
   private var lastBlockNumber: ULong,
@@ -24,9 +24,6 @@ class GlobalAggregationCalculator(
     b1.startBlockNumber.compareTo(b2.startBlockNumber)
   }
 
-  private val aggregationSizeInBlocks = AtomicInteger(0)
-  private val aggregationSizeInBatches = AtomicInteger(0)
-  private val aggregationSizeInBlobs = AtomicInteger(0)
   private val blobsCounter: Counter = metricsFacade.createCounter(
     category = LineaMetricsCategory.AGGREGATION,
     name = "calculator.blobs.accepted",
@@ -65,30 +62,6 @@ class GlobalAggregationCalculator(
         pendingBlobs.size
       }
     )
-    metricsFacade.createGauge(
-      category = LineaMetricsCategory.AGGREGATION,
-      name = "blocks.size",
-      description = "Number of blocks in each aggregation",
-      measurementSupplier = {
-        aggregationSizeInBlocks.get()
-      }
-    )
-    metricsFacade.createGauge(
-      category = LineaMetricsCategory.AGGREGATION,
-      name = "batches.size",
-      description = "Number of batches in each aggregation",
-      measurementSupplier = {
-        aggregationSizeInBatches.get()
-      }
-    )
-    metricsFacade.createGauge(
-      category = LineaMetricsCategory.AGGREGATION,
-      name = "blobs.size",
-      description = "Number of blobs in each aggregation",
-      measurementSupplier = {
-        aggregationSizeInBlobs.get()
-      }
-    )
   }
 
   private fun processBlob(blobCounters: BlobCounters) {
@@ -172,26 +145,18 @@ class GlobalAggregationCalculator(
       endBlockNumber = blobsInUpdatedAggregation.last().endBlockNumber
     )
 
-    val blocksCount = updatedAggregation.blocksRange.count()
-    val batchesCount = blobsInUpdatedAggregation.sumOf { it.numberOfBatches }.toInt()
-    val blobsCount = blobsInUpdatedAggregation.size
-
     log.info(
       "aggregation: trigger={} aggregation={} updatedAggregation={} " +
         "blobsCount={} batchesCount={} blobs={} aggregationSizeMultiple={}",
       aggregationTrigger.aggregationTriggerType.name,
       aggregation.intervalString(),
       updatedAggregation.intervalString(),
-      blobsCount,
-      batchesCount,
+      blobsInUpdatedAggregation.size,
+      blobsInUpdatedAggregation.sumOf { it.numberOfBatches },
       blobsInUpdatedAggregation.map { it.intervalString() },
       aggregationSizeMultipleOf
     )
 
-    aggregationSizeInBlocks.set(blocksCount)
-    aggregationSizeInBatches.set(batchesCount)
-    aggregationSizeInBlobs.set(blobsCount)
-
     // Reset the trigger calculators now that we have a valid aggregation to handle
     resetTriggerCalculators()
 
diff --git a/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/blob/BlobCompressionProofCoordinator.kt b/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/blob/BlobCompressionProofCoordinator.kt
index 758f1b253..ed1e33973 100644
--- a/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/blob/BlobCompressionProofCoordinator.kt
+++ b/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/blob/BlobCompressionProofCoordinator.kt
@@ -21,7 +21,6 @@ import org.apache.logging.log4j.Logger
 import tech.pegasys.teku.infrastructure.async.SafeFuture
 import java.util.concurrent.CompletableFuture
 import java.util.concurrent.LinkedBlockingDeque
-import java.util.concurrent.atomic.AtomicInteger
 import kotlin.time.Duration
 
 class BlobCompressionProofCoordinator(
@@ -63,18 +62,6 @@ class BlobCompressionProofCoordinator(
       description = "Size of blob compression proving queue",
       measurementSupplier = { blobsToHandle.size }
     )
-    metricsFacade.createGauge(
-      category = LineaMetricsCategory.BLOB,
-      name = "blocks.size",
-      description = "Number of blocks in each blob",
-      measurementSupplier = { blobSizeInBlocks.get() }
-    )
-    metricsFacade.createGauge(
-      category = LineaMetricsCategory.BLOB,
-      name = "batches.size",
-      description = "Number of batches in each blob",
-      measurementSupplier = { blobSizeInBatches.get() }
-    )
   }
 
   data class Config(
diff --git a/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/ConflationServiceImpl.kt b/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/ConflationServiceImpl.kt
index d231b44c1..ed70033df 100644
--- a/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/ConflationServiceImpl.kt
+++ b/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/ConflationServiceImpl.kt
@@ -11,7 +11,6 @@ import org.apache.logging.log4j.Logger
 import tech.pegasys.teku.infrastructure.async.SafeFuture
 import java.util.concurrent.PriorityBlockingQueue
 import java.util.concurrent.TimeUnit
-import java.util.concurrent.atomic.AtomicInteger
 
 class ConflationServiceImpl(
   private val calculator: TracesConflationCalculator,
@@ -32,7 +31,7 @@ class ConflationServiceImpl(
   }
 
   internal val blocksToConflate = PriorityBlockingQueue<PayloadAndBlockCounters>()
-  private val batchSizeInBlocks = AtomicInteger(0)
+
   private val blocksCounter = metricsFacade.createCounter(
     category = LineaMetricsCategory.CONFLATION,
     name = "blocks.imported",
@@ -57,12 +56,6 @@ class ConflationServiceImpl(
       description = "Number of blocks in conflation queue",
       measurementSupplier = { blocksToConflate.size }
     )
-    metricsFacade.createGauge(
-      category = LineaMetricsCategory.CONFLATION,
-      name = "blocks.size",
-      description = "Number of blocks in each conflated batch",
-      measurementSupplier = { batchSizeInBlocks.get() }
-    )
     calculator.onConflatedBatch(this::handleConflation)
   }
 

From af9aa608fa94b6e254714553773124095efbdd41 Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Wed, 22 Jan 2025 17:26:52 +0800
Subject: [PATCH 22/34] feat: removed internal built images from cache

---
 .github/actions/compute-docker-images-hash/action.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/actions/compute-docker-images-hash/action.yml b/.github/actions/compute-docker-images-hash/action.yml
index bdca88885..44093b8a0 100644
--- a/.github/actions/compute-docker-images-hash/action.yml
+++ b/.github/actions/compute-docker-images-hash/action.yml
@@ -17,8 +17,10 @@ runs:
       shell: bash
       run: |
         docker_images=$(docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 config 2>/dev/null | grep "image:" | awk '{print $2}')
-        docker_images_unique=$(echo $docker_images | xargs -n1 | sort -u | xargs)
+        docker_images_trimmed=$(echo $docker_images | grep -Ev "linea-postman|linea-coordinator|linea-transaction-exclusion-api|linea-traces-api-facade|linea-prover")
+        docker_images_unique=$(echo $docker_images_trimmed | xargs -n1 | sort -u | xargs)
         echo DOCKER_IMAGES=$(echo $docker_images) >> $GITHUB_OUTPUT
+        echo DOCKER_IMAGES_TRIMMED=$(echo $docker_images_trimmed) >> $GITHUB_OUTPUT
         echo DOCKER_IMAGES_UNIQUE=$(echo -n $docker_images_unique) >> $GITHUB_OUTPUT
         echo DOCKER_IMAGES_HASH=$(echo -n $docker_images_unique | sha256sum | awk '{print $1}') >> $GITHUB_OUTPUT
     - name: Show docker images hash
@@ -26,5 +28,6 @@ runs:
       id: show-docker-images-hash
       run: |
         echo "DOCKER_IMAGES: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES }}"
+        echo "DOCKER_IMAGES_TRIMMED: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_TRIMMED }}"
         echo "DOCKER_IMAGES_UNIQUE: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_UNIQUE }}"
         echo "DOCKER_IMAGES_HASH: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_HASH }}"

From b7fa2c2180692bbee6d4e310abe4e180a5d4690b Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Wed, 22 Jan 2025 17:40:45 +0800
Subject: [PATCH 23/34] feat: fix docker_images_unique command line

---
 .github/actions/compute-docker-images-hash/action.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/actions/compute-docker-images-hash/action.yml b/.github/actions/compute-docker-images-hash/action.yml
index 44093b8a0..e43c0f54c 100644
--- a/.github/actions/compute-docker-images-hash/action.yml
+++ b/.github/actions/compute-docker-images-hash/action.yml
@@ -17,8 +17,8 @@ runs:
       shell: bash
       run: |
         docker_images=$(docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 config 2>/dev/null | grep "image:" | awk '{print $2}')
-        docker_images_trimmed=$(echo $docker_images | grep -Ev "linea-postman|linea-coordinator|linea-transaction-exclusion-api|linea-traces-api-facade|linea-prover")
-        docker_images_unique=$(echo $docker_images_trimmed | xargs -n1 | sort -u | xargs)
+        docker_images_trimmed=$(echo $docker_images | xargs -n1 | grep -Ev "linea-postman|linea-coordinator|linea-transaction-exclusion-api|linea-traces-api-facade|linea-prover")
+        docker_images_unique=$(echo $docker_images_trimmed | sort -u | xargs)
         echo DOCKER_IMAGES=$(echo $docker_images) >> $GITHUB_OUTPUT
         echo DOCKER_IMAGES_TRIMMED=$(echo $docker_images_trimmed) >> $GITHUB_OUTPUT
         echo DOCKER_IMAGES_UNIQUE=$(echo -n $docker_images_unique) >> $GITHUB_OUTPUT

From 7433a0029bf8d7df8f39369b03c90594c3b345cf Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Wed, 22 Jan 2025 17:46:54 +0800
Subject: [PATCH 24/34] feat: fix docker_images_unique command line

---
 .github/actions/compute-docker-images-hash/action.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/actions/compute-docker-images-hash/action.yml b/.github/actions/compute-docker-images-hash/action.yml
index e43c0f54c..e60c8665b 100644
--- a/.github/actions/compute-docker-images-hash/action.yml
+++ b/.github/actions/compute-docker-images-hash/action.yml
@@ -18,7 +18,7 @@ runs:
       run: |
         docker_images=$(docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 config 2>/dev/null | grep "image:" | awk '{print $2}')
         docker_images_trimmed=$(echo $docker_images | xargs -n1 | grep -Ev "linea-postman|linea-coordinator|linea-transaction-exclusion-api|linea-traces-api-facade|linea-prover")
-        docker_images_unique=$(echo $docker_images_trimmed | sort -u | xargs)
+        docker_images_unique=$(echo $docker_images_trimmed | | xargs -n1 | sort -u | xargs)
         echo DOCKER_IMAGES=$(echo $docker_images) >> $GITHUB_OUTPUT
         echo DOCKER_IMAGES_TRIMMED=$(echo $docker_images_trimmed) >> $GITHUB_OUTPUT
         echo DOCKER_IMAGES_UNIQUE=$(echo -n $docker_images_unique) >> $GITHUB_OUTPUT

From 1eba9d530ebeaeb49e23afb4628f74578e784bfb Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Wed, 22 Jan 2025 17:49:47 +0800
Subject: [PATCH 25/34] feat: fix docker_images_unique command line

---
 .github/actions/compute-docker-images-hash/action.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/actions/compute-docker-images-hash/action.yml b/.github/actions/compute-docker-images-hash/action.yml
index e60c8665b..3c14d7b7b 100644
--- a/.github/actions/compute-docker-images-hash/action.yml
+++ b/.github/actions/compute-docker-images-hash/action.yml
@@ -18,7 +18,7 @@ runs:
       run: |
         docker_images=$(docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 config 2>/dev/null | grep "image:" | awk '{print $2}')
         docker_images_trimmed=$(echo $docker_images | xargs -n1 | grep -Ev "linea-postman|linea-coordinator|linea-transaction-exclusion-api|linea-traces-api-facade|linea-prover")
-        docker_images_unique=$(echo $docker_images_trimmed | | xargs -n1 | sort -u | xargs)
+        docker_images_unique=$(echo $docker_images_trimmed | xargs -n1 | sort -u | xargs)
         echo DOCKER_IMAGES=$(echo $docker_images) >> $GITHUB_OUTPUT
         echo DOCKER_IMAGES_TRIMMED=$(echo $docker_images_trimmed) >> $GITHUB_OUTPUT
         echo DOCKER_IMAGES_UNIQUE=$(echo -n $docker_images_unique) >> $GITHUB_OUTPUT

From ab3e2c023215254f1041b9a282a4cdef492ed186 Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Wed, 22 Jan 2025 18:18:37 +0800
Subject: [PATCH 26/34] feat: add name for pull-and-cache-images

---
 .github/workflows/cache-docker-images.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/cache-docker-images.yml b/.github/workflows/cache-docker-images.yml
index 3d5fb73f2..46c6c8068 100644
--- a/.github/workflows/cache-docker-images.yml
+++ b/.github/workflows/cache-docker-images.yml
@@ -47,6 +47,7 @@ jobs:
     needs: [ check-dockerhub-secrets-present, changes ]
     if: ${{ always() && needs.check-dockerhub-secrets-present.outputs.secrets_present == 'true' && needs.changes.outputs.cache_images == 'true' }}
     runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-med
+    name: Pull and cache images
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -73,7 +74,7 @@ jobs:
         if: steps.cache-common-docker-images.outputs.cache-hit != 'true'
         run: |
           docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 pull
-      - name: Save docker images
+      - name: Cache docker images
         if: steps.cache-common-docker-images.outputs.cache-hit != 'true'
         run: |
           docker save ${{ steps.compute-docker-images-hash.outputs.docker_images_unique }} -o  ~/docker-images-cached/docker-images.tar

From b118ff9fa82d8e6cc69f1d82eadc4ea539a4e2fc Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Wed, 22 Jan 2025 21:50:17 +0800
Subject: [PATCH 27/34] feat: update cache docker image list with those from
 traces-v1

---
 .../compute-docker-images-hash/action.yml     | 20 ++++++++++---------
 .github/workflows/cache-docker-images.yml     |  2 +-
 2 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/.github/actions/compute-docker-images-hash/action.yml b/.github/actions/compute-docker-images-hash/action.yml
index 3c14d7b7b..7f195e68f 100644
--- a/.github/actions/compute-docker-images-hash/action.yml
+++ b/.github/actions/compute-docker-images-hash/action.yml
@@ -5,9 +5,9 @@ outputs:
   docker_images_hash:
     description: 'Docker Images Hash'
     value: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_HASH }}
-  docker_images_unique:
-    description: 'Unique List of Docker Images'
-    value: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_UNIQUE }}
+  docker_images_trimmed:
+    description: 'Trimmed List of Docker Images'
+    value: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_TRIMMED }}
 
 runs:
   using: 'composite'
@@ -16,18 +16,20 @@ runs:
       id: compute-docker-images-hash
       shell: bash
       run: |
-        docker_images=$(docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 config 2>/dev/null | grep "image:" | awk '{print $2}')
-        docker_images_trimmed=$(echo $docker_images | xargs -n1 | grep -Ev "linea-postman|linea-coordinator|linea-transaction-exclusion-api|linea-traces-api-facade|linea-prover")
-        docker_images_unique=$(echo $docker_images_trimmed | xargs -n1 | sort -u | xargs)
+        docker_images_traces_v1=$(docker compose -f docker/compose.yml -f docker/compose-local-dev.overrides.yml --profile l1 --profile l2 config 2>/dev/null | grep "image:" | awk '{print $2}')
+        docker_images_traces_v2=$(docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 config 2>/dev/null | grep "image:" | awk '{print $2}')
+        docker_images=$(echo "$docker_images_traces_v1 $docker_images_traces_v2")
+        docker_images_unique=$(echo $docker_images | xargs -n1 | sort -u)
+        docker_images_trimmed=$(echo $docker_images_unique | xargs -n1 | grep -Ev "linea-postman|linea-coordinator|linea-transaction-exclusion-api|linea-traces-api-facade|linea-prover" | xargs)
         echo DOCKER_IMAGES=$(echo $docker_images) >> $GITHUB_OUTPUT
+        echo DOCKER_IMAGES_UNIQUE=$(echo $docker_images_unique) >> $GITHUB_OUTPUT
         echo DOCKER_IMAGES_TRIMMED=$(echo $docker_images_trimmed) >> $GITHUB_OUTPUT
-        echo DOCKER_IMAGES_UNIQUE=$(echo -n $docker_images_unique) >> $GITHUB_OUTPUT
-        echo DOCKER_IMAGES_HASH=$(echo -n $docker_images_unique | sha256sum | awk '{print $1}') >> $GITHUB_OUTPUT
+        echo DOCKER_IMAGES_HASH=$(echo -n $docker_images_trimmed | sha256sum | awk '{print $1}') >> $GITHUB_OUTPUT
     - name: Show docker images hash
       shell: bash
       id: show-docker-images-hash
       run: |
         echo "DOCKER_IMAGES: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES }}"
-        echo "DOCKER_IMAGES_TRIMMED: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_TRIMMED }}"
         echo "DOCKER_IMAGES_UNIQUE: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_UNIQUE }}"
+        echo "DOCKER_IMAGES_TRIMMED: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_TRIMMED }}"
         echo "DOCKER_IMAGES_HASH: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_HASH }}"
diff --git a/.github/workflows/cache-docker-images.yml b/.github/workflows/cache-docker-images.yml
index 46c6c8068..bc9d44089 100644
--- a/.github/workflows/cache-docker-images.yml
+++ b/.github/workflows/cache-docker-images.yml
@@ -77,5 +77,5 @@ jobs:
       - name: Cache docker images
         if: steps.cache-common-docker-images.outputs.cache-hit != 'true'
         run: |
-          docker save ${{ steps.compute-docker-images-hash.outputs.docker_images_unique }} -o  ~/docker-images-cached/docker-images.tar
+          docker save ${{ steps.compute-docker-images-hash.outputs.docker_images_trimmed }} -o  ~/docker-images-cached/docker-images.tar
           echo docker-images.tar=$(ls -lh ~/docker-images-cached/docker-images.tar)

From 4aaddbc410d2a4171733cf3671579bd949a83545 Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Thu, 23 Jan 2025 02:55:00 +0800
Subject: [PATCH 28/34] feat: move cache-docker-images to main workflow

---
 .github/workflows/main.yml    | 8 +++++++-
 .github/workflows/testing.yml | 4 ----
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 677229433..a0ece052d 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -178,8 +178,14 @@ jobs:
       transaction_exclusion_api_image_tagged: ${{ needs.check-and-tag-images.outputs.image_tagged_transaction_exclusion_api }}
     secrets: inherit
 
+  cache-docker-images:
+    needs: [ filter-commit-changes, check-and-tag-images ]
+    if: ${{ always() && needs.filter-commit-changes.outputs.has-changes-requiring-build == 'true' }}
+    uses: ./.github/workflows/cache-docker-images.yml
+    secrets: inherit
+
   testing:
-    needs: [ store-image-name-and-tags, filter-commit-changes, check-and-tag-images ]
+    needs: [ store-image-name-and-tags, filter-commit-changes, check-and-tag-images, cache-docker-images ]
     if: ${{ always() && needs.filter-commit-changes.outputs.has-changes-requiring-build == 'true' }}
     uses: ./.github/workflows/testing.yml
     with:
diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index b1bac977e..5ef176c13 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -26,10 +26,6 @@ on:
         type: string
 
 jobs:
-  cache-docker-images:
-    uses: ./.github/workflows/cache-docker-images.yml
-    secrets: inherit
-  
   coordinator:
     uses: ./.github/workflows/coordinator-testing.yml
     if: ${{ inputs.coordinator_changed == 'true' }}

From ef72a534432e1307fee11e12e89c6b25bcbc4a3e Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Thu, 23 Jan 2025 14:30:35 +0800
Subject: [PATCH 29/34] feat: move cache-docker-images back to testing workflow

---
 .github/workflows/main.yml    | 8 +-------
 .github/workflows/testing.yml | 6 ++++++
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index a0ece052d..677229433 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -178,14 +178,8 @@ jobs:
       transaction_exclusion_api_image_tagged: ${{ needs.check-and-tag-images.outputs.image_tagged_transaction_exclusion_api }}
     secrets: inherit
 
-  cache-docker-images:
-    needs: [ filter-commit-changes, check-and-tag-images ]
-    if: ${{ always() && needs.filter-commit-changes.outputs.has-changes-requiring-build == 'true' }}
-    uses: ./.github/workflows/cache-docker-images.yml
-    secrets: inherit
-
   testing:
-    needs: [ store-image-name-and-tags, filter-commit-changes, check-and-tag-images, cache-docker-images ]
+    needs: [ store-image-name-and-tags, filter-commit-changes, check-and-tag-images ]
     if: ${{ always() && needs.filter-commit-changes.outputs.has-changes-requiring-build == 'true' }}
     uses: ./.github/workflows/testing.yml
     with:
diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index 5ef176c13..8198e90c7 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -26,7 +26,12 @@ on:
         type: string
 
 jobs:
+  cache-docker-images:
+    uses: ./.github/workflows/cache-docker-images.yml
+    secrets: inherit
+
   coordinator:
+    needs: [ cache-docker-images ]
     uses: ./.github/workflows/coordinator-testing.yml
     if: ${{ inputs.coordinator_changed == 'true' }}
     with:
@@ -53,6 +58,7 @@ jobs:
     secrets: inherit
 
   staterecovery:
+    needs: [ cache-docker-images ]
     uses: ./.github/workflows/staterecovery-testing.yml
     if: ${{ inputs.staterecovery_changed == 'true' }}
     with:

From dd641dad10fe770685ab20b6cd0c274b63304eed Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Thu, 23 Jan 2025 16:16:55 +0800
Subject: [PATCH 30/34] feat: remove leftover outputs in
 reuse-images-tags-and-push-workflow

---
 .github/workflows/reuse-check-images-tags-and-push.yml | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/.github/workflows/reuse-check-images-tags-and-push.yml b/.github/workflows/reuse-check-images-tags-and-push.yml
index 4d78b8448..9b2d82480 100644
--- a/.github/workflows/reuse-check-images-tags-and-push.yml
+++ b/.github/workflows/reuse-check-images-tags-and-push.yml
@@ -53,15 +53,10 @@ jobs:
     name: Check image tags exist
     outputs:
       last_commit_tag_exists_coordinator: ${{ steps.check_image_tags_exist_coordinator.outputs.last_commit_tag_exists }}
-      common_ancestor_commit_tag_exists_coordinator: ${{ steps.check_image_tags_exist_coordinator.outputs.common_ancestor_commit_tag_exists }}
       last_commit_tag_exists_postman: ${{ steps.check_image_tags_exist_postman.outputs.last_commit_tag_exists }}
-      common_ancestor_commit_tag_exists_postman: ${{ steps.check_image_tags_exist_postman.outputs.common_ancestor_commit_tag_exists }}
       last_commit_tag_exists_prover: ${{ steps.check_image_tags_exist_prover.outputs.last_commit_tag_exists }}
-      common_ancestor_commit_tag_exists_prover: ${{ steps.check_image_tags_exist_prover.outputs.common_ancestor_commit_tag_exists }}
       last_commit_tag_exists_traces_api_facade: ${{ steps.check_image_tags_exist_traces_api_facade.outputs.last_commit_tag_exists }}
-      common_ancestor_commit_tag_exists_traces_api_facade: ${{ steps.check_image_tags_exist_traces_api_facade.outputs.common_ancestor_commit_tag_exists }}
       last_commit_tag_exists_transaction_exclusion_api: ${{ steps.check_image_tags_exist_transaction_exclusion_api.outputs.last_commit_tag_exists }}
-      common_ancestor_commit_tag_exists_transaction_exclusion_api: ${{ steps.check_image_tags_exist_transaction_exclusion_api.outputs.common_ancestor_commit_tag_exists }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4

From 69928e5a95e774289698708fa34ab3ccdce67874 Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Thu, 23 Jan 2025 16:44:16 +0800
Subject: [PATCH 31/34] feat: bump to exact version for actions/cache

---
 .github/actions/restore-docker-images/action.yml | 2 +-
 .github/workflows/cache-docker-images.yml        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/actions/restore-docker-images/action.yml b/.github/actions/restore-docker-images/action.yml
index 7691327da..5139c23b3 100644
--- a/.github/actions/restore-docker-images/action.yml
+++ b/.github/actions/restore-docker-images/action.yml
@@ -14,7 +14,7 @@ runs:
     - name: Cache common docker images
       id: cache-common-docker-images
       continue-on-error: true
-      uses: actions/cache/restore@v4
+      uses: actions/cache/restore@v4.2.0
       with:
         path: ~/docker-images-cached
         key: docker-images-cache-${{ steps.compute-docker-images-hash.outputs.docker_images_hash }}
diff --git a/.github/workflows/cache-docker-images.yml b/.github/workflows/cache-docker-images.yml
index bc9d44089..5614aab38 100644
--- a/.github/workflows/cache-docker-images.yml
+++ b/.github/workflows/cache-docker-images.yml
@@ -59,7 +59,7 @@ jobs:
           mkdir -p ~/docker-images-cached
       - name: Cache common docker images
         id: cache-common-docker-images
-        uses: actions/cache@v4
+        uses: actions/cache@v4.2.0
         with:
           path: ~/docker-images-cached
           key: docker-images-cache-${{ steps.compute-docker-images-hash.outputs.docker_images_hash }}

From 96a7a26495b252898684b3b75fd9e6a13ad125a7 Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Thu, 23 Jan 2025 16:56:33 +0800
Subject: [PATCH 32/34] feat: temp disable cache for docker images

---
 .github/workflows/coordinator-testing.yml   |  9 +++++----
 .github/workflows/reuse-run-e2e-tests.yml   |  9 +++++----
 .github/workflows/staterecovery-testing.yml |  9 +++++----
 .github/workflows/testing.yml               | 11 ++++++-----
 4 files changed, 21 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/coordinator-testing.yml b/.github/workflows/coordinator-testing.yml
index 22ab36c18..c4a0f6247 100644
--- a/.github/workflows/coordinator-testing.yml
+++ b/.github/workflows/coordinator-testing.yml
@@ -37,10 +37,11 @@ jobs:
         # Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies.
         # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md
         uses: gradle/actions/setup-gradle@cc4fc85e6b35bafd578d5ffbc76a5518407e1af0 #v4.2.1
-      - name: Restore cached images
-        id: restore-cached-images
-        uses: ./.github/actions/restore-docker-images
-        continue-on-error: true
+      # Disable cache for pulling docker images
+      # - name: Restore cached images
+      #   id: restore-cached-images
+      #   uses: ./.github/actions/restore-docker-images
+      #   continue-on-error: true
       # Install pnpm to compile smart contracts
       - name: Setup nodejs environment
         uses: ./.github/actions/setup-nodejs
diff --git a/.github/workflows/reuse-run-e2e-tests.yml b/.github/workflows/reuse-run-e2e-tests.yml
index 4b00106ed..0f822ab70 100644
--- a/.github/workflows/reuse-run-e2e-tests.yml
+++ b/.github/workflows/reuse-run-e2e-tests.yml
@@ -97,10 +97,11 @@ jobs:
         run: |
           mkdir -p tmp/local/traces/v2/conflated
           chmod -R a+w tmp/local/traces/v2/conflated
-      - name: Restore cached images
-        id: restore-cached-images
-        uses: ./.github/actions/restore-docker-images
-        continue-on-error: true
+      # Disable cache for pulling docker images
+      # - name: Restore cached images
+      #   id: restore-cached-images
+      #   uses: ./.github/actions/restore-docker-images
+      #   continue-on-error: true
       - name: Pull all images with retry
         uses: nick-fields/retry@v3
         with:
diff --git a/.github/workflows/staterecovery-testing.yml b/.github/workflows/staterecovery-testing.yml
index f22164066..0018e5de0 100644
--- a/.github/workflows/staterecovery-testing.yml
+++ b/.github/workflows/staterecovery-testing.yml
@@ -37,10 +37,11 @@ jobs:
         # Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies.
         # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md
         uses: gradle/actions/setup-gradle@cc4fc85e6b35bafd578d5ffbc76a5518407e1af0 #v4.2.1
-      - name: Restore cached images
-        id: restore-cached-images
-        uses: ./.github/actions/restore-docker-images
-        continue-on-error: true
+      # Disable cache for pulling docker images
+      # - name: Restore cached images
+      #   id: restore-cached-images
+      #   uses: ./.github/actions/restore-docker-images
+      #   continue-on-error: true
       - name: Staterecovery - Build and Unit tests
         run: |
           ./gradlew state-recovery:besu-plugin:buildNeeded
diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index 8198e90c7..b963d9ee9 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -26,12 +26,13 @@ on:
         type: string
 
 jobs:
-  cache-docker-images:
-    uses: ./.github/workflows/cache-docker-images.yml
-    secrets: inherit
+  # Disable cache for pulling docker images
+  # cache-docker-images:
+  #   uses: ./.github/workflows/cache-docker-images.yml
+  #   secrets: inherit
 
   coordinator:
-    needs: [ cache-docker-images ]
+    # needs: [ cache-docker-images ]
     uses: ./.github/workflows/coordinator-testing.yml
     if: ${{ inputs.coordinator_changed == 'true' }}
     with:
@@ -58,7 +59,7 @@ jobs:
     secrets: inherit
 
   staterecovery:
-    needs: [ cache-docker-images ]
+    # needs: [ cache-docker-images ]
     uses: ./.github/workflows/staterecovery-testing.yml
     if: ${{ inputs.staterecovery_changed == 'true' }}
     with:

From 2b10d353365d86be818c622020c69d86130bdef6 Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Sat, 25 Jan 2025 01:57:20 +0800
Subject: [PATCH 33/34] feat: add more detailed comments

---
 .github/workflows/coordinator-testing.yml   | 3 ++-
 .github/workflows/reuse-run-e2e-tests.yml   | 3 ++-
 .github/workflows/staterecovery-testing.yml | 3 ++-
 .github/workflows/testing.yml               | 3 ++-
 4 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/coordinator-testing.yml b/.github/workflows/coordinator-testing.yml
index c4a0f6247..064a90c38 100644
--- a/.github/workflows/coordinator-testing.yml
+++ b/.github/workflows/coordinator-testing.yml
@@ -37,7 +37,8 @@ jobs:
         # Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies.
         # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md
         uses: gradle/actions/setup-gradle@cc4fc85e6b35bafd578d5ffbc76a5518407e1af0 #v4.2.1
-      # Disable cache for pulling docker images
+      # Disable cache for pulling docker images as empirically found that this 
+      # (retrieving cache and loading docker images) actually increased test time-to-completion
       # - name: Restore cached images
       #   id: restore-cached-images
       #   uses: ./.github/actions/restore-docker-images
diff --git a/.github/workflows/reuse-run-e2e-tests.yml b/.github/workflows/reuse-run-e2e-tests.yml
index 0f822ab70..3aa99b74e 100644
--- a/.github/workflows/reuse-run-e2e-tests.yml
+++ b/.github/workflows/reuse-run-e2e-tests.yml
@@ -97,7 +97,8 @@ jobs:
         run: |
           mkdir -p tmp/local/traces/v2/conflated
           chmod -R a+w tmp/local/traces/v2/conflated
-      # Disable cache for pulling docker images
+      # Disable cache for pulling docker images as empirically found that this 
+      # (retrieving cache and loading docker images) actually increased test time-to-completion
       # - name: Restore cached images
       #   id: restore-cached-images
       #   uses: ./.github/actions/restore-docker-images
diff --git a/.github/workflows/staterecovery-testing.yml b/.github/workflows/staterecovery-testing.yml
index 0018e5de0..b45c39594 100644
--- a/.github/workflows/staterecovery-testing.yml
+++ b/.github/workflows/staterecovery-testing.yml
@@ -37,7 +37,8 @@ jobs:
         # Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies.
         # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md
         uses: gradle/actions/setup-gradle@cc4fc85e6b35bafd578d5ffbc76a5518407e1af0 #v4.2.1
-      # Disable cache for pulling docker images
+      # Disable cache for pulling docker images as empirically found that this 
+      # (retrieving cache and loading docker images) actually increased test time-to-completion
       # - name: Restore cached images
       #   id: restore-cached-images
       #   uses: ./.github/actions/restore-docker-images
diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index b963d9ee9..104efa353 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -26,7 +26,8 @@ on:
         type: string
 
 jobs:
-  # Disable cache for pulling docker images
+  # Disable cache for pulling docker images as empirically found that this 
+  # (retrieving cache and loading docker images) actually increased test time-to-completion
   # cache-docker-images:
   #   uses: ./.github/workflows/cache-docker-images.yml
   #   secrets: inherit

From 284d38f953a98952e67974f68891ad88ab003013 Mon Sep 17 00:00:00 2001
From: jonesho <jones.ho@consensys.net>
Date: Sat, 25 Jan 2025 02:02:09 +0800
Subject: [PATCH 34/34] feat:remove unnecessary exception import

---
 .../coordination/aggregation/GlobalAggregationCalculator.kt      | 1 -
 1 file changed, 1 deletion(-)

diff --git a/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/aggregation/GlobalAggregationCalculator.kt b/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/aggregation/GlobalAggregationCalculator.kt
index 1ecf2e7f2..4bd180bc5 100644
--- a/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/aggregation/GlobalAggregationCalculator.kt
+++ b/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/aggregation/GlobalAggregationCalculator.kt
@@ -7,7 +7,6 @@ import net.consensys.zkevm.domain.BlobCounters
 import net.consensys.zkevm.domain.BlobsToAggregate
 import org.apache.logging.log4j.LogManager
 import org.apache.logging.log4j.Logger
-import java.lang.IllegalStateException
 import java.util.PriorityQueue
 
 class GlobalAggregationCalculator(