flow.yml

name: Flow build and test

on:
  pull_request:
    branches: [main]
  push:
    branches: [main]

jobs:
  flow_test:
    strategy:
      matrix:
        runner: [ubicloud-standard-16-ubuntu-2204-arm]
    runs-on: ${{ matrix.runner }}
    timeout-minutes: 30
    services:
      catalog:
        image: imresamu/postgis:15-3.4-alpine
        ports:
          - 5432:5432
        env:
          PGUSER: postgres
          POSTGRES_PASSWORD: postgres
          POSTGRES_DB: postgres
          POSTGRES_INITDB_ARGS: --locale=C.UTF-8
      elasticsearch:
        image: elasticsearch:8.13.0
        ports:
          - 9200:9200
        env:
          discovery.type: single-node
          xpack.security.enabled: false
          xpack.security.enrollment.enabled: false

    steps:
      - uses: actions/checkout@v4

      - name: generate or hydrate protos
        uses: ./.github/actions/genprotos

      - uses: ubicloud/setup-go@v5
        with:
          go-version: '1.23.0'
          cache-dependency-path: flow/go.sum

      - name: install lib-geos
        run: |
          sudo apt-get update
          sudo apt-get install libgeos-dev

      - run: go mod download
        working-directory: ./flow

      - name: setup gcp service account
        id: gcp-service-account
        uses: jsdaniell/create-json@v1.2.3
        with:
          name: "bq_service_account.json"
          json: ${{ secrets.GCP_GH_CI_PKEY }}

      - name: setup snowflake credentials
        id: sf-credentials
        uses: jsdaniell/create-json@v1.2.3
        with:
          name: "snowflake_creds.json"
          json: ${{ secrets.SNOWFLAKE_GH_CI_PKEY }}

      - name: setup S3 credentials
        id: s3-credentials
        uses: jsdaniell/create-json@v1.2.3
        with:
          name: "s3_creds.json"
          json: ${{ secrets.S3_CREDS }}

      - name: setup GCS credentials
        id: gcs-credentials
        uses: jsdaniell/create-json@v1.2.3
        with:
          name: "gcs_creds.json"
          json: ${{ secrets.GCS_CREDS }}

      - name: setup Eventhubs credentials
        id: eventhubs-credentials
        uses: jsdaniell/create-json@v1.2.3
        with:
          name: "eh_creds.json"
          json: ${{ secrets.EH_CREDS }}

      - name: create hstore extension, increase logical replication limits, and setup catalog database
        run: >
          docker exec "${{ job.services.catalog.id }}" psql -U postgres -c "CREATE EXTENSION hstore;"
          -c "ALTER SYSTEM SET wal_level=logical;"
          -c "ALTER SYSTEM SET max_replication_slots=192;"
          -c "ALTER SYSTEM SET max_wal_senders=256;"
          -c "ALTER SYSTEM SET max_connections=2048;" &&
          (cat ./nexus/catalog/migrations/V{?,??}__* | docker exec -i "${{ job.services.catalog.id }}" psql -U postgres) &&
          docker restart "${{ job.services.catalog.id }}"
        env:
          PGPASSWORD: postgres

      - name: start redpanda
        uses: redpanda-data/github-action@v0.1.4
        with:
          version: "latest"

      - name: start clickhouse
        uses: getsentry/action-clickhouse-in-ci@v1

      - name: Install Temporal CLI
        uses: temporalio/setup-temporal@v0

      - name: run tests
        run: |
          temporal server start-dev --namespace default --headless &
          go build -ldflags="-s -w" -o peer-flow
          temporal operator search-attribute create --name MirrorName --type Text --namespace default
          ./peer-flow worker &
          ./peer-flow snapshot-worker &
          go test -p 32 ./... -timeout 900s
        working-directory: ./flow
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          AWS_REGION: ${{ secrets.AWS_REGION }}
          TEST_BQ_CREDS: ${{ github.workspace }}/bq_service_account.json
          TEST_SF_CREDS: ${{ github.workspace }}/snowflake_creds.json
          TEST_S3_CREDS: ${{ github.workspace }}/s3_creds.json
          TEST_GCS_CREDS: ${{ github.workspace }}/gcs_creds.json
          TEST_EH_CREDS: ${{ github.workspace }}/eh_creds.json
          AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
          AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
          AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
          ENABLE_SQLSERVER_TESTS: "true"
          SQLSERVER_HOST: ${{ secrets.SQLSERVER_HOST }}
          SQLSERVER_PORT: ${{ secrets.SQLSERVER_PORT }}
          SQLSERVER_USER: ${{ secrets.SQLSERVER_USER }}
          SQLSERVER_PASSWORD: ${{ secrets.SQLSERVER_PASSWORD }}
          SQLSERVER_DB: ${{ secrets.SQLSERVER_DB }}
          PEERDB_CATALOG_HOST: localhost
          PEERDB_CATALOG_PORT: 5432
          PEERDB_CATALOG_USER: postgres
          PEERDB_CATALOG_PASSWORD: postgres
          PEERDB_CATALOG_DATABASE: postgres
          PEERDB_QUEUE_FORCE_TOPIC_CREATION: "true"
          ELASTICSEARCH_TEST_ADDRESS: http://localhost:9200