Merge branch 'ccip-develop' into offchain-usdc

# Conflicts:
#	core/services/ocr2/plugins/ccip/execution_plugin.go

.github/actions/setup-postgres/wait-for-healthy-postgres.sh

@@ -2,10 +2,24 @@
 RETRIES=10
 
 until [ $RETRIES -eq 0 ]; do
-  if docker compose ps postgres --status running --format json | jq >/dev/null -e 'if (.[0].Health == "healthy") then true else false end'; then
+  DOCKER_OUTPUT=$(docker compose ps postgres --status running --format json)
+  JSON_TYPE=$(echo "$DOCKER_OUTPUT" | jq -r 'type')
+
+  if [ "$JSON_TYPE" == "array" ]; then
+    HEALTH_STATUS=$(echo "$DOCKER_OUTPUT" | jq -r '.[0].Health')
+  elif [ "$JSON_TYPE" == "object" ]; then
+    HEALTH_STATUS=$(echo "$DOCKER_OUTPUT" | jq -r '.Health')
+  else
+    HEALTH_STATUS="Unknown JSON type: $JSON_TYPE"
+  fi
+
+  echo "postgres health status: $HEALTH_STATUS"
+  if [ "$HEALTH_STATUS" == "healthy" ]; then
     exit 0
   fi
+
   echo "Waiting for postgres server, $((RETRIES--)) remaining attempts..."
   sleep 2
 done
+
 exit 1

.github/workflows/automation-benchmark-tests.yml

@@ -97,7 +97,7 @@ jobs:
             done
           done <<< "$EVM_HTTP_URLS"
       - name: Checkout the repo
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
         with:
           ref: ${{ env.REF_NAME }}
       - name: Build Test Image

.github/workflows/automation-ondemand-tests.yml

@@ -43,7 +43,7 @@ jobs:
           this-job-name: Build Chainlink Image ${{ matrix.image.name }}
         continue-on-error: true
       - name: Checkout the repo
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
         with:
           ref: ${{ github.head_ref || github.ref_name }}
       - name: Check if image exists
@@ -88,7 +88,7 @@ jobs:
           this-job-name: Build Test Image
         continue-on-error: true
       - name: Checkout the repo
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
         with:
           ref: ${{ github.head_ref || github.ref_name }}
       - name: Build Test Image
@@ -139,7 +139,7 @@ jobs:
     name: Automation On Demand ${{ matrix.tests.name }} Test
     steps:
       - name: Checkout the repo
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
         with:
           ref: ${{ github.head_ref || github.ref_name }}
       - name: Determine build to use

.github/workflows/bash-cicd-scripts.yml

@@ -11,7 +11,7 @@ jobs:
       bash-cicd-scripts-src: ${{ steps.bash-cicd-scripts.outputs.src }}
     steps:
       - name: Checkout the repo
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
       - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1
         id: bash-cicd-scripts
         with:
@@ -25,7 +25,7 @@ jobs:
     needs: [changes]
     steps:
       - name: Checkout the repo
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
       - name: Run ShellCheck
         if: needs.changes.outputs.bash-cicd-scripts-src == 'true'
         uses: ludeeus/action-shellcheck@00cae500b08a931fb5698e11e79bfbd38e612a38 # v2.0.0
@@ -40,7 +40,7 @@ jobs:
         shell: bash
     steps:
       - name: Checkout the repo
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
       - name: Install shellspec
         if: needs.changes.outputs.bash-cicd-scripts-src == 'true'
         env:

.github/workflows/build.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout repository
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
 
       - name: Build chainlink image
         uses: ./.github/actions/build-sign-publish-chainlink

.github/workflows/ci-core.yml

@@ -27,7 +27,7 @@ jobs:
       src: ${{ steps.golangci-changes.outputs.src }}
     steps:
       - name: Checkout the repo
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
       - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1
         id: golangci-changes
         with:
@@ -50,7 +50,7 @@ jobs:
       on_trigger_lint: ${{ steps.golangci-lint.outputs.on_trigger }}
     steps:
       - name: Checkout the repo
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
       - name: Check if event should trigger lint
         id: golangci-lint
         env:
@@ -65,7 +65,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: [init]
     steps:
-      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
         with:
           fetch-depth: 0
       - uses: actions/setup-go@v4
@@ -80,7 +80,7 @@ jobs:
         if: needs.init.outputs.on_trigger_lint == 'true'
         uses: golangci/golangci-lint-action@3a919529898de77ec3da873e3063ca4b10e7f5cc # v3.7.0
         with:
-          version: v1.53.3
+          version: v1.54.2
           only-new-issues: ${{ github.event.schedule == '' }} # show only new issues, unless it's a scheduled run
           args: --out-format checkstyle:golangci-lint-report.xml
       - name: Print lint report artifact
@@ -109,7 +109,7 @@ jobs:
       splits: ${{ steps.split.outputs.splits }}
     steps:
       - name: Checkout the repo
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
       - name: Setup Go
         uses: ./.github/actions/setup-go
         with:
@@ -144,7 +144,7 @@ jobs:
       CL_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/chainlink_test?sslmode=disable
     steps:
       - name: Checkout the repo
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
       - name: Setup node
         uses: actions/setup-node@v3
       - name: Setup NodeJS
@@ -232,7 +232,7 @@ jobs:
           SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
         with:
           channel-id: '#topic-data-races'
-          slack-message: "Race tests failed: ${{ job.html_url }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url || format('https://github.com/smartcontractkit/chainlink/commit/{0}', github.sha) }}"
+          slack-message: "Race tests failed: ${{ job.html_url }}\n${{ format('https://github.com/smartcontractkit/chainlink/actions/runs/{0}', github.run_id) }}"
       - name: Collect Metrics
         if: always()
         id: collect-gha-metrics
@@ -252,7 +252,7 @@ jobs:
       CL_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/chainlink_test?sslmode=disable
     steps:
       - name: Checkout the repo
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
       - name: Setup node
         uses: actions/setup-node@v3
       - name: Setup NodeJS
@@ -289,7 +289,7 @@ jobs:
             -gh_sha=$GITHUB_SHA \
             -gh_event_path=$GITHUB_EVENT_PATH \
             -command=./tools/bin/go_core_tests \
-            `ls -R ./artifacts/go_core_tests*/output-short.txt`
+            `ls -R ./artifacts/go_core_tests*/output.txt`
       - name: Store logs artifacts
         if: always()
         uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0
@@ -305,7 +305,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout the repo
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
         with:
           fetch-depth: 0 # fetches all history for all tags and branches to provide more metadata for sonar reports
       - name: Download all workflow run artifacts
@@ -317,7 +317,7 @@ jobs:
           echo "sonarqube_tests_report_paths=$(find go_core_tests_*_logs -name output.txt | paste -sd "," -)" >> $GITHUB_OUTPUT
           echo "sonarqube_coverage_report_paths=$(find go_core_tests_*_logs -name coverage.txt | paste -sd "," -)" >> $GITHUB_OUTPUT
       - name: SonarQube Scan
-        uses: sonarsource/sonarqube-scan-action@4b0bfc149f5e285930eeb5e917327e66660c6e92 # v2.0.0
+        uses: sonarsource/sonarqube-scan-action@69c1a75940dec6249b86dace6b630d3a2ae9d2a7 # v2.0.1
         with:
           args: >
             -Dsonar.go.tests.reportPaths=${{ steps.sonarqube_report_paths.outputs.sonarqube_tests_report_paths }}
@@ -343,7 +343,7 @@ jobs:
       run:
         shell: bash
     steps:
-      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
         with:
           fetch-depth: 0
       - name: Setup Go

.github/workflows/codeql-analysis.yml

@@ -25,7 +25,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
 
       - name: Set up Go
         if: ${{ matrix.language == 'go' }}

.github/workflows/dependency-check.yml

@@ -11,7 +11,7 @@ jobs:
       changes: ${{ steps.changes.outputs.src }}
     steps:
       - name: Checkout the repo
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
       - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1
         id: changes
         with:
@@ -25,7 +25,7 @@ jobs:
     needs: [changes]
     steps:
       - name: Check out code
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
 
       - name: Set up Go
         if: needs.changes.outputs.src == 'true'

.github/workflows/generic-test-runner.yml

@@ -50,7 +50,7 @@ jobs:
     runs-on: ubuntu20.04-16cores-64GB
     steps:
       - name: Checkout the repo
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
       - name: Check if image exists
         id: check-image
         uses: smartcontractkit/chainlink-github-actions/docker/image-exists@00c6214deb10a3f374c6d3430c32c5202015d463 # v2.2.12
@@ -85,7 +85,7 @@ jobs:
       TEST_SUITE: ${{ github.event.inputs.directory }}
     steps:
       - name: Checkout the repo
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
       - name: Build Test Image
         uses: ./.github/actions/build-test-image
         with:
@@ -124,7 +124,7 @@ jobs:
       - name: Debug Input
         run: echo ${{ github.event.inputs.testInputs }}
       - name: Checkout the repo
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
       - name: Run Tests
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@00c6214deb10a3f374c6d3430c32c5202015d463 # v2.2.12
         with:

.github/workflows/goreleaser-build-publish-develop.yml

@@ -18,7 +18,7 @@ jobs:
       contents: read
     steps:
       - name: Checkout repository
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
       - name: Configure aws credentials
         uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0
         with:
@@ -55,7 +55,7 @@ jobs:
       contents: read
     steps:
       - name: Checkout repository
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
       - name: Configure aws credentials
         uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0
         with:

.github/workflows/integration-staging-tests.yml

@@ -0,0 +1,63 @@
+name: E2E Functions staging tests
+
+on:
+# TODO: enable when env will be stable
+#  schedule:
+#    - cron: "0 0 * * *"
+  workflow_dispatch:
+    inputs:
+      network:
+        description: Blockchain network (testnet)
+        type: choice
+        default: "MUMBAI"
+        options:
+          - "MUMBAI"
+      test_type:
+        description: Test type
+        type: choice
+        default: "mumbai_functions_soak_test_real"
+        options:
+          - "mumbai_functions_soak_test_http"
+          - "mumbai_functions_stress_test_http"
+          - "mumbai_functions_soak_test_only_secrets"
+          - "mumbai_functions_stress_test_only_secrets"
+          - "mumbai_functions_soak_test_real"
+          - "mumbai_functions_stress_test_real"
+# TODO: disabled, need GATI access
+#          - "gateway_secrets_set_soak_test"
+#          - "gateway_secrets_list_soak_test"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  e2e-soak-test:
+    environment: sdlc
+    runs-on: ubuntu20.04-8cores-32GB
+    permissions:
+      contents: read
+      id-token: write
+    env:
+      LOKI_URL: ${{ secrets.LOKI_URL }}
+      LOKI_TOKEN: ${{ secrets.LOKI_TOKEN }}
+
+      SELECTED_NETWORKS: ${{ inputs.network }}
+      SELECTED_TEST: ${{ inputs.test_type }}
+      MUMBAI_URLS: ${{ secrets.FUNCTIONS_STAGING_MUMBAI_URLS }}
+      MUMBAI_KEYS: ${{ secrets.FUNCTIONS_STAGING_MUMBAI_KEYS }}
+
+      WASP_LOG_LEVEL: info
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Run E2E soak tests
+        run: |
+          cd integration-tests/load/functions
+          if [[ $SELECTED_TEST == mumbai_functions* ]]; then
+            go test -v -timeout 6h -run TestFunctionsLoad/$SELECTED_TEST
+          elif [[ $SELECTED_TEST == gateway* ]]; then
+            go test -v -timeout 6h -run TestGatewayLoad/$SELECTED_TEST
+          fi

.github/workflows/integration-tests-publish.yml

@@ -27,7 +27,7 @@ jobs:
           this-job-name: Publish Integration Test Image
         continue-on-error: true
       - name: Checkout the repo
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: Build Image