airbyte-sync-parallel-with-dbt.yaml

id: airbyte-sync-parallel-with-dbt
namespace: company.team

tasks:
  - id: data_ingestion
    type: io.kestra.plugin.core.flow.Parallel
    tasks:
      - id: salesforce
        type: io.kestra.plugin.airbyte.connections.Sync
        connectionId: e3b1ce92-547c-436f-b1e8-23b6936c12ab

      - id: google_analytics
        type: io.kestra.plugin.airbyte.connections.Sync
        connectionId: e3b1ce92-547c-436f-b1e8-23b6936c12cd

      - id: facebook_ads
        type: io.kestra.plugin.airbyte.connections.Sync
        connectionId: e3b1ce92-547c-436f-b1e8-23b6936c12ef

  - id: dbt
    type: io.kestra.plugin.core.flow.WorkingDirectory
    tasks:
      - id: clone_repository
        type: io.kestra.plugin.git.Clone
        url: https://github.com/kestra-io/dbt-demo
        branch: main

      - id: dbt_build
        type: io.kestra.plugin.dbt.cli.Build
        taskRunner:
          type: io.kestra.plugin.scripts.runner.docker.Docker
        containerImage: ghcr.io/kestra-io/dbt-bigquery:latest
        dbtPath: /usr/local/bin/dbt
        inputFiles:
          .profile/profiles.yml: |
            jaffle_shop:
              outputs:
                dev:
                  type: bigquery
                  dataset: your_big_query_dataset_name
                  project: your_big_query_project
                  fixed_retries: 1
                  keyfile: sa.json
                  location: EU
                  method: service-account
                  priority: interactive
                  threads: 8
                  timeout_seconds: 300
              target: dev
          sa.json: "{{ secret('GCP_CREDS') }}"

pluginDefaults:
  - type: io.kestra.plugin.airbyte.connections.Sync
    values:
      url: http://host.docker.internal:8000/
      username: "{{ secret('AIRBYTE_USERNAME') }}"
      password: "{{ secret('AIRBYTE_PASSWORD') }}"

extend:
  title: Trigger multiple Airbyte syncs, then run a dbt job
  description: >-
    This flow executes several Airbyte syncs in parallel and then runs dbt
    Core's CLI commands. 

    It's recommended to set GCP and Airbyte server credentials as secrets.
  tags:
    - Parallel
    - Ingest
    - Git
    - dbt
    - BigQuery
  ee: false
  demo: false
  meta_description: "This flow executes several Airbyte syncs in parallel and then
    runs dbt Core's CLI commands. "