Merge branch 'develop' into feat-rollup-relayer-gas-oracle-graceful-r…

…estart
scroll-tech · Jan 24, 2024 · e2c41aa · e2c41aa
2 parents b472e00 + d3ec65f
commit e2c41aa
Show file tree

Hide file tree

Showing 26 changed files with 2,667 additions and 759 deletions.
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -5,6 +5,9 @@ on:
     tags:
       - v**
 
+env:
+  AWS_REGION: us-west-2
+
 jobs:
   event_watcher:
     runs-on: ubuntu-latest
@@ -29,6 +32,29 @@ jobs:
             scrolltech/event-watcher:latest
           # cache-from: type=gha,scope=${{ github.workflow }}
           # cache-to: type=gha,scope=${{ github.workflow }}
+
+      # build and push to aws ecr
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@0e613a0980cbf65ed5b322eb7a1e075d28913a83
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@62f4f872db3836360b72999f4b87f1ff13310f3a
+      - name: Build, tag, and push image to Amazon ECR
+        id: build-image
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          ECR_REPOSITORY: event-watcher
+          IMAGE_TAG: ${{github.ref_name}}
+        run: |
+          # Build a docker container and push it to ECR
+          # docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG .
+          docker tag scrolltech/$ECR_REPOSITORY:$IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+
   gas_oracle:
     runs-on: ubuntu-latest
     steps:
@@ -52,6 +78,30 @@ jobs:
             scrolltech/gas-oracle:latest
           # cache-from: type=gha,scope=${{ github.workflow }}
           # cache-to: type=gha,scope=${{ github.workflow }}
+
+      # build and push to aws ecr
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@0e613a0980cbf65ed5b322eb7a1e075d28913a83
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@62f4f872db3836360b72999f4b87f1ff13310f3a
+
+      - name: Build, tag, and push image to Amazon ECR
+        id: build-image
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          ECR_REPOSITORY: gas-oracle
+          IMAGE_TAG: ${{github.ref_name}}
+        run: |
+          # Build a docker container and push it to ECR
+          # docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG .
+          docker tag scrolltech/$ECR_REPOSITORY:$IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+
   rollup_relayer:
     runs-on: ubuntu-latest
     steps:
@@ -75,52 +125,124 @@ jobs:
             scrolltech/rollup-relayer:latest
           # cache-from: type=gha,scope=${{ github.workflow }}
           # cache-to: type=gha,scope=${{ github.workflow }}
+
+      # build and push to aws ecr
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@0e613a0980cbf65ed5b322eb7a1e075d28913a83
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@62f4f872db3836360b72999f4b87f1ff13310f3a
+
+      - name: Build, tag, and push image to Amazon ECR
+        id: build-image
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          ECR_REPOSITORY: rollup-relayer
+          IMAGE_TAG: ${{github.ref_name}}
+        run: |
+          # Build a docker container and push it to ECR
+          # docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG .
+          docker tag scrolltech/$ECR_REPOSITORY:$IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+
   bridgehistoryapi-fetcher:
     runs-on: ubuntu-latest
     steps:
-    - name: Checkout code
-      uses: actions/checkout@v2
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2
-    - name: Login to Docker Hub
-      uses: docker/login-action@v2
-      with:
-        username: ${{ secrets.DOCKERHUB_USERNAME }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
-    - name: Build and push bridgehistoryapi-fetcher docker
-      uses: docker/build-push-action@v2
-      with:
-        context: .
-        file: ./build/dockerfiles/bridgehistoryapi-fetcher.Dockerfile
-        push: true
-        tags: |
-          scrolltech/bridgehistoryapi-fetcher:${{github.ref_name}}
-          scrolltech/bridgehistoryapi-fetcher:latest
-        # cache-from: type=gha,scope=${{ github.workflow }}
-        # cache-to: type=gha,scope=${{ github.workflow }}
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push bridgehistoryapi-fetcher docker
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./build/dockerfiles/bridgehistoryapi-fetcher.Dockerfile
+          push: true
+          tags: |
+            scrolltech/bridgehistoryapi-fetcher:${{github.ref_name}}
+            scrolltech/bridgehistoryapi-fetcher:latest
+          # cache-from: type=gha,scope=${{ github.workflow }}
+          # cache-to: type=gha,scope=${{ github.workflow }}
+
+      # build and push to aws ecr
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@0e613a0980cbf65ed5b322eb7a1e075d28913a83
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@62f4f872db3836360b72999f4b87f1ff13310f3a
+
+      - name: Build, tag, and push image to Amazon ECR
+        id: build-image
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          ECR_REPOSITORY: bridgehistoryapi-fetcher
+          IMAGE_TAG: ${{github.ref_name}}
+        run: |
+          # Build a docker container and push it to ECR
+          # docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG .
+          docker tag scrolltech/$ECR_REPOSITORY:$IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+
   bridgehistoryapi-api:
     runs-on: ubuntu-latest
     steps:
-    - name: Checkout code
-      uses: actions/checkout@v2
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2
-    - name: Login to Docker Hub
-      uses: docker/login-action@v2
-      with:
-        username: ${{ secrets.DOCKERHUB_USERNAME }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
-    - name: Build and push bridgehistoryapi-api docker
-      uses: docker/build-push-action@v2
-      with:
-        context: .
-        file: ./build/dockerfiles/bridgehistoryapi-api.Dockerfile
-        push: true
-        tags: |
-          scrolltech/bridgehistoryapi-api:${{github.ref_name}}
-          scrolltech/bridgehistoryapi-api:latest
-        # cache-from: type=gha,scope=${{ github.workflow }}
-        # cache-to: type=gha,scope=${{ github.workflow }}
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push bridgehistoryapi-api docker
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./build/dockerfiles/bridgehistoryapi-api.Dockerfile
+          push: true
+          tags: |
+            scrolltech/bridgehistoryapi-api:${{github.ref_name}}
+            scrolltech/bridgehistoryapi-api:latest
+          # cache-from: type=gha,scope=${{ github.workflow }}
+          # cache-to: type=gha,scope=${{ github.workflow }}
+
+      # build and push to aws ecr
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@0e613a0980cbf65ed5b322eb7a1e075d28913a83
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@62f4f872db3836360b72999f4b87f1ff13310f3a
+
+      - name: Build, tag, and push image to Amazon ECR
+        id: build-image
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          ECR_REPOSITORY: bridgehistoryapi-api
+          IMAGE_TAG: ${{github.ref_name}}
+        run: |
+          # Build a docker container and push it to ECR
+          # docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG .
+          docker tag scrolltech/$ECR_REPOSITORY:$IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+
   coordinator-api:
     runs-on: ubuntu-latest
     steps:
@@ -144,6 +266,30 @@ jobs:
             scrolltech/coordinator-api:latest
           # cache-from: type=gha,scope=${{ github.workflow }}
           # cache-to: type=gha,scope=${{ github.workflow }}
+
+      # build and push to aws ecr
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@0e613a0980cbf65ed5b322eb7a1e075d28913a83
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@62f4f872db3836360b72999f4b87f1ff13310f3a
+
+      - name: Build, tag, and push image to Amazon ECR
+        id: build-image
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          ECR_REPOSITORY: coordinator-api
+          IMAGE_TAG: ${{github.ref_name}}
+        run: |
+          # Build a docker container and push it to ECR
+          # docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG .
+          docker tag scrolltech/$ECR_REPOSITORY:$IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+
   coordinator-cron:
     runs-on: ubuntu-latest
     steps:
@@ -167,3 +313,26 @@ jobs:
             scrolltech/coordinator-cron:latest
           # cache-from: type=gha,scope=${{ github.workflow }}
           # cache-to: type=gha,scope=${{ github.workflow }}
+
+      # build and push to aws ecr
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@0e613a0980cbf65ed5b322eb7a1e075d28913a83
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@62f4f872db3836360b72999f4b87f1ff13310f3a
+
+      - name: Build, tag, and push image to Amazon ECR
+        id: build-image
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          ECR_REPOSITORY: coordinator-cron
+          IMAGE_TAG: ${{github.ref_name}}
+        run: |
+          # Build a docker container and push it to ECR
+          # docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG .
+          docker tag scrolltech/$ECR_REPOSITORY:$IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
diff --git a/bridge-history-api/conf/config.json b/bridge-history-api/conf/config.json
@@ -3,8 +3,8 @@
 		"confirmation": 0,
 		"endpoint": "https://rpc.ankr.com/eth",
 		"startHeight": 18306000,
-		"blockTime": 10,
-		"fetchLimit": 30,
+		"blockTime": 12,
+		"fetchLimit": 16,
 		"MessengerAddr": "0x6774Bcbd5ceCeF1336b5300fb5186a12DDD8b367",
 		"ETHGatewayAddr": "0x7F2b8C31F88B6006c382775eea88297Ec1e3E905",
 		"WETHGatewayAddr": "0x7AC440cAe8EB6328de4fA621163a792c1EA9D4fE",
@@ -23,7 +23,7 @@
 		"confirmation": 0,
 		"endpoint": "https://rpc.scroll.io",
 		"blockTime": 3,
-		"fetchLimit": 100,
+		"fetchLimit": 64,
 		"MessengerAddr": "0x781e90f1c8Fc4611c9b7497C3B47F99Ef6969CbC",
 		"ETHGatewayAddr": "0x6EA73e05AdC79974B931123675ea8F78FfdacDF0",
 		"WETHGatewayAddr": "0x7003E7B7186f0E6601203b99F7B8DECBfA391cf9",

diff --git a/bridge-history-api/go.mod b/bridge-history-api/go.mod
@@ -6,7 +6,6 @@ require (
 	github.com/gin-contrib/cors v1.5.0
 	github.com/gin-gonic/gin v1.9.1
 	github.com/go-redis/redis/v8 v8.11.5
-	github.com/google/uuid v1.4.0
 	github.com/pressly/goose/v3 v3.16.0
 	github.com/prometheus/client_golang v1.14.0
 	github.com/scroll-tech/go-ethereum v1.10.14-0.20231130005111-38a3a9c9198c
@@ -40,6 +39,7 @@ require (
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/uuid v1.4.0 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/holiman/uint256 v1.2.4 // indirect
 	github.com/huin/goupnp v1.3.0 // indirect

diff --git a/bridge-history-api/internal/config/config.go b/bridge-history-api/internal/config/config.go
@@ -8,11 +8,11 @@ import (
 	"scroll-tech/common/database"
 )
 
-// LayerConfig is the configuration of Layer1/Layer2
-type LayerConfig struct {
+// FetcherConfig is the configuration of Layer1 or Layer2 fetcher.
+type FetcherConfig struct {
 	Confirmation             uint64 `json:"confirmation"`
 	Endpoint                 string `json:"endpoint"`
-	StartHeight              uint64 `json:"startHeight"` // Can only be configured to contract deployment height, otherwise in the current implementation, the message proof could not be successfully updated.
+	StartHeight              uint64 `json:"startHeight"` // Can only be configured to contract deployment height, message proof should be updated from the very beginning.
 	BlockTime                int64  `json:"blockTime"`
 	FetchLimit               uint64 `json:"fetchLimit"`
 	MessengerAddr            string `json:"MessengerAddr"`
@@ -43,8 +43,8 @@ type RedisConfig struct {
 
 // Config is the configuration of the bridge history backend
 type Config struct {
-	L1    *LayerConfig     `json:"L1"`
-	L2    *LayerConfig     `json:"L2"`
+	L1    *FetcherConfig   `json:"L1"`
+	L2    *FetcherConfig   `json:"L2"`
 	DB    *database.Config `json:"db"`
 	Redis *RedisConfig     `json:"redis"`
 }

diff --git a/bridge-history-api/internal/controller/fetcher/l1_fetcher.go b/bridge-history-api/internal/controller/fetcher/l1_fetcher.go
@@ -20,7 +20,7 @@ import (
 // L1MessageFetcher fetches cross message events from L1 and saves them to database.
 type L1MessageFetcher struct {
 	ctx    context.Context
-	cfg    *config.LayerConfig
+	cfg    *config.FetcherConfig
 	client *ethclient.Client
 
 	l1SyncHeight        uint64
@@ -35,7 +35,7 @@ type L1MessageFetcher struct {
 }
 
 // NewL1MessageFetcher creates a new L1MessageFetcher instance.
-func NewL1MessageFetcher(ctx context.Context, cfg *config.LayerConfig, db *gorm.DB, client *ethclient.Client) *L1MessageFetcher {
+func NewL1MessageFetcher(ctx context.Context, cfg *config.FetcherConfig, db *gorm.DB, client *ethclient.Client) *L1MessageFetcher {
 	c := &L1MessageFetcher{
 		ctx:              ctx,
 		cfg:              cfg,
@@ -108,7 +108,6 @@ func (c *L1MessageFetcher) Start() {
 }
 
 func (c *L1MessageFetcher) fetchAndSaveEvents(confirmation uint64) {
-	c.l1MessageFetcherRunningTotal.Inc()
 	startHeight := c.l1SyncHeight + 1
 	endHeight, rpcErr := utils.GetBlockNumber(c.ctx, c.client, confirmation)
 	if rpcErr != nil {
@@ -134,6 +133,7 @@ func (c *L1MessageFetcher) fetchAndSaveEvents(confirmation uint64) {
 			c.l1MessageFetcherReorgTotal.Inc()
 			log.Warn("L1 reorg happened, exit and re-enter fetchAndSaveEvents", "re-sync height", resyncHeight)
 			c.updateL1SyncHeight(resyncHeight, lastBlockHash)
+			c.l1MessageFetcherRunningTotal.Inc()
 			return
 		}
 
@@ -143,6 +143,7 @@ func (c *L1MessageFetcher) fetchAndSaveEvents(confirmation uint64) {
 		}
 
 		c.updateL1SyncHeight(to, lastBlockHash)
+		c.l1MessageFetcherRunningTotal.Inc()
 	}
 }