Merge remote-tracking branch 'origin/main' into stable

PeerDB-io · Jul 19, 2023 · b0f6a88 · b0f6a88
2 parents e3249c8 + 0dbd626
commit b0f6a88
Show file tree

Hide file tree

Showing 82 changed files with 6,474 additions and 2,025 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -10,6 +10,20 @@ jobs:
   build:
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    services:
+      catalog_peer:
+        image: debezium/postgres:14-alpine
+        ports:
+          - 7132:5432
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
     steps:
       - uses: actions/checkout@v3
         with:
@@ -22,21 +36,39 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install -y protobuf-compiler libssl-dev pkg-config build-essential
+      
+      - name: setup gcp service account
+        id: gcp-service-account
+        uses: jsdaniell/[email protected]
+        with:
+          name: "bq_service_account.json"
+          json: ${{ secrets.GCP_GH_CI_PKEY }}
+          dir: 'nexus/server/tests/assets/'
+
+      - name: setup snowflake credentials
+        id: sf-credentials
+        uses: jsdaniell/[email protected]
+        with:
+          name: "snowflake_creds.json"
+          json: ${{ secrets.SNOWFLAKE_GH_CI_PKEY }}
+          dir: 'nexus/server/tests/assets/'
 
       - name: cargo check
         run: cargo check
         working-directory: ./nexus
 
       - name: cargo test
-        run: cargo test
+        run: cargo test -- --test-threads=1
         working-directory: ./nexus
         env:
           RUST_BACKTRACE: 1
-          PEERDB_CATALOG_HOST: ${{ secrets.NEXUS_CATALOG_HOST }}
-          PEERDB_CATALOG_PORT: ${{ secrets.NEXUS_CATALOG_PORT }}
-          PEERDB_CATALOG_DATABASE: ${{ secrets.NEXUS_CATALOG_DATABASE }}
-          PEERDB_CATALOG_USER: ${{ secrets.NEXUS_CATALOG_USER }}
-          PEERDB_CATALOG_PASSWORD: ${{ secrets.NEXUS_CATALOG_PASSWORD }}
+          PEERDB_CATALOG_HOST: localhost
+          PEERDB_CATALOG_PORT: 7132
+          PEERDB_CATALOG_DATABASE: postgres
+          PEERDB_CATALOG_USER: postgres
+          PEERDB_CATALOG_PASSWORD: postgres
+          TEST_BQ_CREDS: tests/assets/bq_service_account.json
+          TEST_SF_CREDS: tests/assets/snowflake_creds.json
           PEERDB_LOG_DIR: /tmp
 
       - name: dump server.log

diff --git a/.github/workflows/flow.yml b/.github/workflows/flow.yml
@@ -69,3 +69,9 @@ jobs:
           AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
           AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
           AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          ENABLE_SQLSERVER_TESTS: true
+          SQLSERVER_HOST: ${{ secrets.SQLSERVER_HOST }}
+          SQLSERVER_PORT: ${{ secrets.SQLSERVER_PORT }}
+          SQLSERVER_USER: ${{ secrets.SQLSERVER_USER }}
+          SQLSERVER_PASSWORD: ${{ secrets.SQLSERVER_PASSWORD }}
+          SQLSERVER_DB: ${{ secrets.SQLSERVER_DB }}
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -37,7 +37,7 @@ services:
 
   catalog:
     container_name: catalog
-    image: debezium/postgres:14-alpine
+    image: debezium/postgres:15-alpine
     ports:
       # mapping is from host to container
       - 5432:5432
@@ -73,25 +73,9 @@ services:
       - 8112:8112
     environment:
       TEMPORAL_HOST_PORT: temporalite:7233
-      PEERDB_CATALOG_DATABASE: postgres
-      PEERDB_CATALOG_HOST: catalog
-      PEERDB_CATALOG_PASSWORD: postgres
-      PEERDB_CATALOG_PORT: 5432
-      PEERDB_CATALOG_USER: postgres
-      GIN_MODE: release
-      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-""}
-      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-""}
-      AWS_REGION: ${AWS_REGION:-""}
     depends_on:
-      catalog:
-        condition: service_healthy
       temporal-admin-tools:
         condition: service_healthy
-    healthcheck:
-      test: "curl --fail http://flow_api:8112/health || exit 1"
-      interval: 5s
-      timeout: 5s
-      retries: 5
 
   flow_worker:
     container_name: flow_worker
@@ -112,6 +96,7 @@ services:
         condition: service_healthy
 
   peerdb:
+    container_name: peerdb_server
     build:
       context: .
       dockerfile: stacks/nexus.Dockerfile
@@ -123,14 +108,14 @@ services:
       PEERDB_CATALOG_PASSWORD: postgres
       PEERDB_CATALOG_DATABASE: postgres
       PEERDB_PASSWORD: peerdb
-      PEERDB_FLOW_SERVER_ADDRESS: http://flow_api:8112
+      PEERDB_FLOW_SERVER_ADDRESS: grpc://flow_api:8112
+      RUST_LOG: info
+      RUST_BACKTRACE: 1
     ports:
       - 9900:9900
     depends_on:
       catalog:
         condition: service_healthy
-      flow_api:
-        condition: service_healthy
 
 volumes:
   pgdata:

diff --git a/flow/activities/flowable.go b/flow/activities/flowable.go
@@ -253,6 +253,7 @@ func (a *FlowableActivity) StartFlow(ctx context.Context, input *protos.StartFlo
 	log.Info("pushed records")
 
 	if err != nil {
+		log.Warnf("failed to push records: %v", err)
 		return nil, fmt.Errorf("failed to push records: %w", err)
 	}
 
@@ -380,14 +381,14 @@ func (a *FlowableActivity) CleanupQRepFlow(ctx context.Context, config *protos.Q
 	return dst.CleanupQRepFlow(config)
 }
 
-func (a *FlowableActivity) DropFlow(ctx context.Context, config *protos.FlowConnectionConfigs) error {
-	src, err := connectors.GetConnector(ctx, config.Source)
+func (a *FlowableActivity) DropFlow(ctx context.Context, config *protos.ShutdownRequest) error {
+	src, err := connectors.GetConnector(ctx, config.SourcePeer)
 	defer connectors.CloseConnector(src)
 	if err != nil {
 		return fmt.Errorf("failed to get source connector: %w", err)
 	}
 
-	dest, err := connectors.GetConnector(ctx, config.Destination)
+	dest, err := connectors.GetConnector(ctx, config.DestinationPeer)
 	defer connectors.CloseConnector(dest)
 	if err != nil {
 		return fmt.Errorf("failed to get destination connector: %w", err)