Merge branch 'main' into stable

PeerDB-io · Nov 25, 2024 · 5814889 · 5814889
2 parents a851ee0 + 4379613
commit 5814889
Show file tree

Hide file tree

Showing 121 changed files with 3,839 additions and 1,690 deletions.
diff --git a/.github/actions/genprotos/action.yml b/.github/actions/genprotos/action.yml
@@ -3,10 +3,10 @@ description: 'Install buf with local plugins, generate protos and cache'
 runs:
   using: "composite"
   steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
     - name: check cache
       id: cache
-      uses: ubicloud/cache@v4
+      uses: ubicloud/cache@0a97811d53629b143a56b3c2b1f729fd11719ef7 # v4
       with:
         path: |
           ./flow/generated/protos
@@ -15,7 +15,7 @@ runs:
         key: ${{ runner.os }}-build-genprotos-${{ hashFiles('buf.gen.yaml', './protos/peers.proto', './protos/flow.proto', './protos/route.proto') }}
 
     - if: steps.cache.outputs.cache-hit != 'true'
-      uses: bufbuild/buf-action@v1
+      uses: bufbuild/buf-action@3fb70352251376e958c4c2c92c3818de82a71c2b # v1
       with:
         setup_only: true
         github_token: ${{ github.token }}

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -5,18 +5,19 @@ on:
     branches: [main, release/*]
   pull_request:
     branches: [main, release/*]
-    paths: [nexus/**, protos/**]
+    paths: [nexus/**, protos/**, .github/workflows/ci.yml]
 
 jobs:
   build:
     strategy:
       matrix:
         runner: [ubicloud-standard-2-ubuntu-2204-arm]
+        postgres-version: [13, 14, 15, 16, 17]
     runs-on: ${{ matrix.runner }}
     timeout-minutes: 30
     services:
       catalog_peer:
-        image: debezium/postgres:14-alpine
+        image: postgres:${{ matrix.postgres-version }}-alpine
         ports:
           - 7132:5432
         env:
@@ -29,7 +30,7 @@ jobs:
           --health-timeout 5s
           --health-retries 5
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
 
       - name: generate or hydrate protos
         uses: ./.github/actions/genprotos
@@ -41,21 +42,21 @@ jobs:
 
       - name: setup gcp service account
         id: gcp-service-account
-        uses: jsdaniell/[email protected]
+        uses: jsdaniell/create-json@b8e77fa01397ca39cc4a6198cc29a3be5481afef # v1.2.3
         with:
           name: "bq_service_account.json"
           json: ${{ secrets.GCP_GH_CI_PKEY }}
           dir: "nexus/server/tests/assets/"
 
       - name: setup snowflake credentials
         id: sf-credentials
-        uses: jsdaniell/[email protected]
+        uses: jsdaniell/create-json@b8e77fa01397ca39cc4a6198cc29a3be5481afef # v1.2.3
         with:
           name: "snowflake_creds.json"
           json: ${{ secrets.SNOWFLAKE_GH_CI_PKEY }}
           dir: "nexus/server/tests/assets/"
 
-      - uses: ubicloud/rust-cache@v2
+      - uses: ubicloud/rust-cache@69587b2b3f26e8938580c44a643d265ed12f3119 # v2
         with:
           workspaces: nexus
 

diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml
@@ -10,9 +10,9 @@ jobs:
     timeout-minutes: 60
     steps:
       - name: checkout sources
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
 
-      - uses: ubicloud/setup-go@v5
+      - uses: ubicloud/setup-go@35680fe0723d4a9309d4b1ac1c67e0d46eac5f24 # v5
         with:
           go-version: '1.23.0'
           cache-dependency-path: e2e_cleanup/go.sum
@@ -24,28 +24,28 @@ jobs:
 
       - name: setup gcp service account
         id: gcp-service-account
-        uses: jsdaniell/[email protected]
+        uses: jsdaniell/create-json@b8e77fa01397ca39cc4a6198cc29a3be5481afef # v1.2.3
         with:
           name: "bq_service_account.json"
           json: ${{ secrets.GCP_GH_CI_PKEY }}
 
       - name: setup snowflake credentials
         id: sf-credentials
-        uses: jsdaniell/[email protected]
+        uses: jsdaniell/create-json@b8e77fa01397ca39cc4a6198cc29a3be5481afef # v1.2.3
         with:
           name: "snowflake_creds.json"
           json: ${{ secrets.SNOWFLAKE_GH_CI_PKEY }}
 
       - name: setup S3 credentials
         id: s3-credentials
-        uses: jsdaniell/[email protected]
+        uses: jsdaniell/create-json@b8e77fa01397ca39cc4a6198cc29a3be5481afef # v1.2.3
         with:
           name: "s3_creds.json"
           json: ${{ secrets.S3_CREDS }}
 
       - name: setup GCS credentials
         id: gcs-credentials
-        uses: jsdaniell/[email protected]
+        uses: jsdaniell/create-json@b8e77fa01397ca39cc4a6198cc29a3be5481afef # v1.2.3
         with:
           name: "gcs_creds.json"
           json: ${{ secrets.GCS_CREDS }}
@@ -54,20 +54,5 @@ jobs:
         run: go run main.go
         working-directory: ./e2e_cleanup
         env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          AWS_REGION: ${{ secrets.AWS_REGION }}
           TEST_BQ_CREDS: ${{ github.workspace }}/bq_service_account.json
           TEST_SF_CREDS: ${{ github.workspace }}/snowflake_creds.json
-          TEST_S3_CREDS: ${{ github.workspace }}/s3_creds.json
-          TEST_GCS_CREDS: ${{ github.workspace }}/gcs_creds.json
-          AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
-          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
-          AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
-          AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
-          ENABLE_SQLSERVER_TESTS: true
-          SQLSERVER_HOST: ${{ secrets.SQLSERVER_HOST }}
-          SQLSERVER_PORT: ${{ secrets.SQLSERVER_PORT }}
-          SQLSERVER_USER: ${{ secrets.SQLSERVER_USER }}
-          SQLSERVER_PASSWORD: ${{ secrets.SQLSERVER_PASSWORD }}
-          SQLSERVER_DB: ${{ secrets.SQLSERVER_DB }}
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -34,7 +34,7 @@ jobs:
           build-mode: none
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
 
     - name: generate or hydrate protos
       uses: ./.github/actions/genprotos
@@ -47,12 +47,12 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v3
+      uses: github/codeql-action/init@ea9e4e37992a54ee68a9622e985e60c8e8f12d9f # v3
       with:
         languages: ${{ matrix.language }}
         build-mode: ${{ matrix.build-mode }}
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v3
+      uses: github/codeql-action/analyze@ea9e4e37992a54ee68a9622e985e60c8e8f12d9f # v3
       with:
         category: "/language:${{matrix.language}}"
diff --git a/.github/workflows/customer-docker.yml b/.github/workflows/customer-docker.yml
@@ -18,15 +18,15 @@ jobs:
       contents: read
       packages: write
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
 
       - name: generate or hydrate protos
         uses: ./.github/actions/genprotos
 
-      - uses: depot/setup-action@v1
+      - uses: depot/setup-action@b0b1ea4f69e92ebf5dea3f8713a1b0c37b2126a5 # v1
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3
         with:
           registry: ghcr.io
           username: ${{github.actor}}
@@ -42,7 +42,7 @@ jobs:
           echo "branch=$(echo $GITHUB_REF | sed -e 's/.*customer-//')" >> $GITHUB_OUTPUT
 
       - name: Build (optionally publish) PeerDB Images
-        uses: depot/bake-action@v1
+        uses: depot/bake-action@143e50b965398f1f5dc8463be7dde6f62b9e9c21 # v1
         with:
           token: ${{ secrets.DEPOT_TOKEN }}
           files: ./docker-bake.hcl

diff --git a/.github/workflows/dev-docker.yml b/.github/workflows/dev-docker.yml
@@ -17,15 +17,15 @@ jobs:
       contents: read
       packages: write
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
 
       - name: generate or hydrate protos
         uses: ./.github/actions/genprotos
 
-      - uses: depot/setup-action@v1
+      - uses: depot/setup-action@b0b1ea4f69e92ebf5dea3f8713a1b0c37b2126a5 # v1
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3
         with:
           registry: ghcr.io
           username: ${{github.actor}}
@@ -36,7 +36,7 @@ jobs:
         run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
 
       - name: Build (optionally publish) PeerDB Images
-        uses: depot/bake-action@v1
+        uses: depot/bake-action@143e50b965398f1f5dc8463be7dde6f62b9e9c21 # v1
         with:
           token: ${{ secrets.DEPOT_TOKEN }}
           files: ./docker-bake.hcl

diff --git a/.github/workflows/flow-api-client.yml b/.github/workflows/flow-api-client.yml
@@ -9,7 +9,7 @@ jobs:
     permissions:
       contents: write
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
 
     - name: generate or hydrate protos
       uses: ./.github/actions/genprotos

diff --git a/.github/workflows/flow.yml b/.github/workflows/flow.yml
@@ -11,11 +11,12 @@ jobs:
     strategy:
       matrix:
         runner: [ubicloud-standard-16-ubuntu-2204-arm]
+        postgres-version: [15, 16, 17]
     runs-on: ${{ matrix.runner }}
     timeout-minutes: 30
     services:
       catalog:
-        image: imresamu/postgis:15-3.4-alpine
+        image: imresamu/postgis:${{ matrix.postgres-version }}-3.5-alpine
         ports:
           - 5432:5432
         env:
@@ -24,21 +25,31 @@ jobs:
           POSTGRES_DB: postgres
           POSTGRES_INITDB_ARGS: --locale=C.UTF-8
       elasticsearch:
-        image: elasticsearch:8.13.0
+        image: elasticsearch:8.16.0@sha256:a411f7c17549209c5839b69f929de00bd91f1e2dcf08b65d5f41b122eae17f5e
         ports:
           - 9200:9200
         env:
           discovery.type: single-node
           xpack.security.enabled: false
           xpack.security.enrollment.enabled: false
+      minio:
+        image: bitnami/minio:2024.11.7@sha256:9f2d9c45006a2ada1bc485e1393291ce7d54ae1a46260dd491381a4eb8b2fd47
+        ports:
+          - 9999:9999
+        env:
+          MINIO_ROOT_USER: minio
+          MINIO_ROOT_PASSWORD: miniosecret
+          MINIO_API_PORT_NUMBER: 9999
+          AWS_EC2_METADATA_DISABLED: true
+          MINIO_DEFAULT_BUCKETS: peerdb
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
 
       - name: generate or hydrate protos
         uses: ./.github/actions/genprotos
 
-      - uses: ubicloud/setup-go@v5
+      - uses: ubicloud/setup-go@35680fe0723d4a9309d4b1ac1c67e0d46eac5f24 # v5
         with:
           go-version: '1.23.0'
           cache-dependency-path: flow/go.sum
@@ -53,35 +64,35 @@ jobs:
 
       - name: setup gcp service account
         id: gcp-service-account
-        uses: jsdaniell/[email protected]
+        uses: jsdaniell/create-json@b8e77fa01397ca39cc4a6198cc29a3be5481afef # v1.2.3
         with:
           name: "bq_service_account.json"
           json: ${{ secrets.GCP_GH_CI_PKEY }}
 
       - name: setup snowflake credentials
         id: sf-credentials
-        uses: jsdaniell/[email protected]
+        uses: jsdaniell/create-json@b8e77fa01397ca39cc4a6198cc29a3be5481afef # v1.2.3
         with:
           name: "snowflake_creds.json"
           json: ${{ secrets.SNOWFLAKE_GH_CI_PKEY }}
 
       - name: setup S3 credentials
         id: s3-credentials
-        uses: jsdaniell/[email protected]
+        uses: jsdaniell/create-json@b8e77fa01397ca39cc4a6198cc29a3be5481afef # v1.2.3
         with:
           name: "s3_creds.json"
           json: ${{ secrets.S3_CREDS }}
 
       - name: setup GCS credentials
         id: gcs-credentials
-        uses: jsdaniell/[email protected]
+        uses: jsdaniell/create-json@b8e77fa01397ca39cc4a6198cc29a3be5481afef # v1.2.3
         with:
           name: "gcs_creds.json"
           json: ${{ secrets.GCS_CREDS }}
 
       - name: setup Eventhubs credentials
         id: eventhubs-credentials
-        uses: jsdaniell/[email protected]
+        uses: jsdaniell/create-json@b8e77fa01397ca39cc4a6198cc29a3be5481afef # v1.2.3
         with:
           name: "eh_creds.json"
           json: ${{ secrets.EH_CREDS }}
@@ -99,15 +110,27 @@ jobs:
           PGPASSWORD: postgres
 
       - name: start redpanda
-        uses: redpanda-data/[email protected]
+        uses: redpanda-data/github-action@c68af8edc420b987e871615ca40b3a5dd70eb5b1 # v0.1.4
         with:
           version: "latest"
 
-      - name: start clickhouse
-        uses: getsentry/action-clickhouse-in-ci@v1
+      - uses: ubicloud/cache@0a97811d53629b143a56b3c2b1f729fd11719ef7 # v4
+        id: cache-clickhouse
+        with:
+          path: ./clickhouse
+          key: ${{ runner.os }}-clickhouse
+
+      - name: Install ClickHouse
+        if: steps.cache-clickhouse.outputs.cache-hit != 'true'
+        run: |
+          curl https://clickhouse.com | sh
+
+      - name: Run ClickHouse
+        run: |
+          ./clickhouse server &
 
       - name: Install Temporal CLI
-        uses: temporalio/setup-temporal@v0
+        uses: temporalio/setup-temporal@1059a504f87e7fa2f385e3fa40d1aa7e62f1c6ca # v0
 
       - name: run tests
         run: |
@@ -119,9 +142,20 @@ jobs:
           go test -p 32 ./... -timeout 900s
         working-directory: ./flow
         env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          AWS_REGION: ${{ secrets.AWS_REGION }}
+          AWS_ENDPOINT_URL_S3: http://localhost:9999
+          AWS_ACCESS_KEY_ID: minio
+          AWS_SECRET_ACCESS_KEY: miniosecret
+          AWS_REGION: us-east-1
+          PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ACCESS_KEY_ID: minio
+          PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_SECRET_ACCESS_KEY: miniosecret
+          PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_REGION: us-east-1
+          PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ENDPOINT_URL_S3: http://localhost:9999
+          PEERDB_CLICKHOUSE_AWS_S3_BUCKET_NAME: peerdb
+          PEERDB_SNOWFLAKE_AWS_CREDENTIALS_AWS_ACCESS_KEY_ID: minio
+          PEERDB_SNOWFLAKE_AWS_CREDENTIALS_AWS_SECRET_ACCESS_KEY: miniosecret
+          PEERDB_SNOWFLAKE_AWS_CREDENTIALS_AWS_REGION: us-east-1
+          PEERDB_SNOWFLAKE_AWS_CREDENTIALS_AWS_ENDPOINT_URL_S3: http://localhost:9999
+          PEERDB_SNOWFLAKE_AWS_S3_BUCKET_NAME: peerdb
           TEST_BQ_CREDS: ${{ github.workspace }}/bq_service_account.json
           TEST_SF_CREDS: ${{ github.workspace }}/snowflake_creds.json
           TEST_S3_CREDS: ${{ github.workspace }}/s3_creds.json