Merge branch 'main' into stable

PeerDB-io · Apr 27, 2024 · 8640b57 · 8640b57
2 parents ac156ae + 8de414b
commit 8640b57
Show file tree

Hide file tree

Showing 141 changed files with 4,715 additions and 2,484 deletions.
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -0,0 +1,61 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "main", "customer-*", "stable" ]
+  pull_request:
+    branches: [ "main", "customer-*", "stable" ]
+  schedule:
+    - cron: '15 17 * * 2'
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubicloud-standard-16' }}
+    timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
+    permissions:
+      # required for all workflows
+      security-events: write
+
+      # required to fetch internal or private CodeQL packs
+      packages: read
+
+      # only required for workflows in private repositories
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - language: go
+          build-mode: autobuild
+        - language: javascript-typescript
+          build-mode: none
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - uses: bufbuild/[email protected]
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+
+    - run: buf generate protos
+
+    - name: install lib-geos
+      run: |
+        sudo apt-get update
+        sudo apt-get install libgeos-dev
+
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        build-mode: ${{ matrix.build-mode }}
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"
diff --git a/.github/workflows/flow.yml b/.github/workflows/flow.yml
@@ -22,6 +22,14 @@ jobs:
           PGUSER: postgres
           POSTGRES_PASSWORD: postgres
           POSTGRES_DB: postgres
+      elasticsearch:
+        image: elasticsearch:8.13.0
+        ports:
+          - 9200:9200
+        env:
+          discovery.type: single-node
+          xpack.security.enabled: false
+          xpack.security.enrollment.enabled: false
 
     steps:
       - name: checkout sources
@@ -74,6 +82,13 @@ jobs:
           name: "gcs_creds.json"
           json: ${{ secrets.GCS_CREDS }}
 
+      - name: setup Eventhubs credentials
+        id: eventhubs-credentials
+        uses: jsdaniell/[email protected]
+        with:
+          name: "eh_creds.json"
+          json: ${{ secrets.EH_CREDS }}
+
       - name: create hstore extension, increase logical replication limits, and setup catalog database
         run: >
           docker exec "${{ job.services.catalog.id }}" psql -U postgres -c "CREATE EXTENSION hstore;"
@@ -111,6 +126,7 @@ jobs:
           TEST_SF_CREDS: ${{ github.workspace }}/snowflake_creds.json
           TEST_S3_CREDS: ${{ github.workspace }}/s3_creds.json
           TEST_GCS_CREDS: ${{ github.workspace }}/gcs_creds.json
+          TEST_EH_CREDS: ${{ github.workspace }}/eh_creds.json
           AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
           AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
           AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
@@ -126,3 +142,4 @@ jobs:
           PEERDB_CATALOG_USER: postgres
           PEERDB_CATALOG_PASSWORD: postgres
           PEERDB_CATALOG_DATABASE: postgres
+          ELASTICSEARCH_TEST_ADDRESS: http://localhost:9200
diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
@@ -1,5 +1,12 @@
 version: "3.9"
 
+x-minio-config: &minio-config
+  PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ACCESS_KEY_ID: _peerdb_minioadmin
+  PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_SECRET_ACCESS_KEY: _peerdb_minioadmin
+  PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_REGION: us-east-1
+  PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ENDPOINT_URL_S3: http://host.docker.internal:9001
+  PEERDB_CLICKHOUSE_AWS_S3_BUCKET_NAME: peerdbbucket
+
 x-catalog-config: &catalog-config
   PEERDB_CATALOG_HOST: catalog
   PEERDB_CATALOG_PORT: 5432
@@ -121,7 +128,7 @@ services:
       - 8112:8112
       - 8113:8113
     environment:
-      <<: [*catalog-config, *flow-worker-env]
+      <<: [*catalog-config, *flow-worker-env, *minio-config]
     depends_on:
       temporal-admin-tools:
         condition: service_healthy
@@ -133,7 +140,7 @@ services:
       dockerfile: stacks/flow.Dockerfile
       target: flow-snapshot-worker
     environment:
-      <<: [*catalog-config, *flow-worker-env]
+      <<: [*catalog-config, *flow-worker-env, *minio-config]
     depends_on:
       temporal-admin-tools:
         condition: service_healthy
@@ -145,7 +152,7 @@ services:
       dockerfile: stacks/flow.Dockerfile
       target: flow-worker
     environment:
-      <<: [*catalog-config, *flow-worker-env]
+      <<: [*catalog-config, *flow-worker-env, *minio-config]
     depends_on:
       temporal-admin-tools:
         condition: service_healthy
@@ -157,7 +164,7 @@ services:
       dockerfile: stacks/flow.Dockerfile
       target: flow-worker
     environment:
-      <<: [*catalog-config, *flow-worker-env]
+      <<: [*catalog-config, *flow-worker-env, *minio-config]
     profiles:
       - multi
     depends_on:
@@ -171,7 +178,7 @@ services:
       dockerfile: stacks/flow.Dockerfile
       target: flow-worker
     environment:
-      <<: [*catalog-config, *flow-worker-env]
+      <<: [*catalog-config, *flow-worker-env, *minio-config]
     profiles:
       - multi
     depends_on:
@@ -214,5 +221,26 @@ services:
     depends_on:
       - flow-api
 
+  minio:
+    image: minio/minio
+    volumes:
+      - minio-data:/data
+    ports:
+      - "9001:9000"
+      - "9002:36987"
+    environment:
+      <<: *minio-config
+    entrypoint: >
+      /bin/sh -c "
+      export MINIO_ROOT_USER=$$PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ACCESS_KEY_ID;
+      export MINIO_ROOT_PASSWORD=$$PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_SECRET_ACCESS_KEY;
+      minio server /data --console-address=":36987" &
+      sleep 2;
+      /usr/bin/mc alias set myminiopeerdb http://minio:9000 $$MINIO_ROOT_USER $$MINIO_ROOT_PASSWORD;
+      /usr/bin/mc mb myminiopeerdb/$$PEERDB_CLICKHOUSE_AWS_S3_BUCKET_NAME;
+      wait
+      "
+
 volumes:
   pgdata:
+  minio-data:
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,5 +1,12 @@
 version: "3.9"
 
+x-minio-config: &minio-config
+  PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ACCESS_KEY_ID: _peerdb_minioadmin
+  PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_SECRET_ACCESS_KEY: _peerdb_minioadmin
+  PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_REGION: us-east-1
+  PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ENDPOINT_URL_S3: http://host.docker.internal:9001
+  PEERDB_CLICKHOUSE_AWS_S3_BUCKET_NAME: peerdbbucket
+
 x-catalog-config: &catalog-config
   PEERDB_CATALOG_HOST: catalog
   PEERDB_CATALOG_PORT: 5432
@@ -103,7 +110,7 @@ services:
       - 8112:8112
       - 8113:8113
     environment:
-      <<: [*catalog-config, *flow-worker-env]
+      <<: [*catalog-config, *flow-worker-env, *minio-config]
     depends_on:
       temporal-admin-tools:
         condition: service_healthy
@@ -112,7 +119,7 @@ services:
     container_name: flow-snapshot-worker
     image: ghcr.io/peerdb-io/flow-snapshot-worker:latest-dev
     environment:
-      <<: [*catalog-config, *flow-worker-env]
+      <<: [*catalog-config, *flow-worker-env, *minio-config]
     depends_on:
       temporal-admin-tools:
         condition: service_healthy
@@ -121,7 +128,7 @@ services:
     container_name: flow-worker1
     image: ghcr.io/peerdb-io/flow-worker:latest-dev
     environment:
-      <<: [*catalog-config, *flow-worker-env]
+      <<: [*catalog-config, *flow-worker-env, *minio-config]
     depends_on:
       temporal-admin-tools:
         condition: service_healthy
@@ -130,7 +137,7 @@ services:
     container_name: flow-worker2
     image: ghcr.io/peerdb-io/flow-worker:latest-dev
     environment:
-      <<: [*catalog-config, *flow-worker-env]
+      <<: [*catalog-config, *flow-worker-env, *minio-config]
     profiles:
       - multi
     depends_on:
@@ -141,7 +148,7 @@ services:
     container_name: flow-worker3
     image: ghcr.io/peerdb-io/flow-worker:latest-dev
     environment:
-      <<: [*catalog-config, *flow-worker-env]
+      <<: [*catalog-config, *flow-worker-env, *minio-config]
     profiles:
       - multi
     depends_on:
@@ -179,6 +186,27 @@ services:
     depends_on:
       - flow-api
 
+  minio:
+    image: minio/minio
+    volumes:
+      - minio-data:/data
+    ports:
+      - "9001:9000"
+      - "9002:36987"
+    environment:
+      <<: *minio-config
+    entrypoint: >
+      /bin/sh -c "
+      export MINIO_ROOT_USER=$$PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ACCESS_KEY_ID;
+      export MINIO_ROOT_PASSWORD=$$PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_SECRET_ACCESS_KEY;
+      minio server /data --console-address=":36987" &
+      sleep 2;
+      /usr/bin/mc alias set myminiopeerdb http://minio:9000 $$MINIO_ROOT_USER $$MINIO_ROOT_PASSWORD;
+      /usr/bin/mc mb myminiopeerdb/$$PEERDB_CLICKHOUSE_AWS_S3_BUCKET_NAME;
+      wait
+      "
+
 volumes:
   pgdata:
   prometheusdata:
+  minio-data:
diff --git a/e2e_cleanup/go.mod b/e2e_cleanup/go.mod
@@ -1,6 +1,6 @@
 module github.com/PeerDB-io/peer-flow-cleanup
 
-go 1.22.0
+go 1.22.2
 
 require (
 	cloud.google.com/go/bigquery v1.59.1