dlt-zendesk-to-duckdb.yaml

id: dlt-zendesk-to-duckdb
namespace: company.team

tasks:
  - id: dlt_pipeline
    type: io.kestra.plugin.scripts.python.Script
    taskRunner:
      type: io.kestra.plugin.scripts.runner.docker.Docker
    containerImage: python:3.11
    beforeCommands:
      - pip install dlt[duckdb]
      - dlt --non-interactive init zendesk duckdb
    warningOnStdErr: false
    env:
      SOURCES__ZENDESK__ZENDESK_SUPPORT__CREDENTIALS__PASSWORD: "{{ secret('ZENDESK_PASSWORD') }}"
      SOURCES__ZENDESK__ZENDESK_SUPPORT__CREDENTIALS__SUBDOMAIN: "{{ secret('ZENDESK_SUBDOMAIN') }}"
      SOURCES__ZENDESK__ZENDESK_SUPPORT__CREDENTIALS__EMAIL: "{{ secret('ZENDESK_EMAIL') }}"
    script: |
      import dlt
      from zendesk import zendesk_support

      pipeline = dlt.pipeline(
          pipeline_name="zendesk_pipeline",
          destination="duckdb",
          dataset_name="zendesk",
      )

      zendesk_source = zendesk_support(load_all=False)
      tickets = zendesk_source.tickets

      load_info = pipeline.run(tickets)

extend:
  title: Ingest Zendesk data into DuckDB using dlt
  description: >
    This flow demonstrates how to extract data from Zendesk into a DuckDB
    database using dlt. The entire workflow logic is contained in a single
    Python script that uses the dlt Python library to ingest the data into a
    DuckDB database. 

    The credentials to access the Zendesk API are stored using Secret.
  tags:
    - Ingest
  ee: false
  demo: false
  meta_description: This flow demonstrates how to extract data from Zendesk into a
    DuckDB database using dlt. The entire workflow logic is contained in a
    single Python script that uses the dlt Python library to ingest the data
    into a DuckDB database.