fix: Pinned dbt-core==1.7.13
#1346
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Meltano Pipeline | |
| on: | |
| pull_request: | |
| branches: [main] | |
| paths: | |
| - meltano.yml | |
| - pyproject.toml | |
| - plugins/**/*.lock | |
| - transform/models/**/*.sql | |
| - .github/workflows/melty.yaml | |
| - evidence/sources/**/*.sql | |
| - evidence/pages/**/*.md | |
| - evidence/package.json | |
| - evidence/package-lock.json | |
| push: | |
| branches: [main] | |
| paths: | |
| - meltano.yml | |
| - pyproject.toml | |
| - plugins/**/*.lock | |
| - transform/models/**/*.sql | |
| - .github/workflows/melty.yaml | |
| - evidence/sources/**/*.sql | |
| - evidence/pages/**/*.md | |
| - evidence/package.json | |
| - evidence/package-lock.json | |
| workflow_dispatch: | |
| inputs: | |
| environment: | |
| description: Meltano Environment | |
| required: true | |
| default: prod | |
| logConfig: | |
| description: 'Logging configuration file' | |
| required: true | |
| default: 'logging/dev.yaml' | |
| type: choice | |
| options: | |
| - logging/dev.yaml | |
| schedule: | |
| - cron: '0 12 * * *' # Once a day around 6am CST (12am UTC) | |
| env: | |
| MELTANO_DATABASE_URI: ${{ secrets.MELTANO_DATABASE_URI }} | |
| DUCKDB_PATH: ${{ secrets.DUCKDB_PATH }} | |
| MELTANO_CLI_LOG_CONFIG: ${{ github.event.inputs.environment || 'logging/ci.yaml' }} | |
| MELTANO_ENVIRONMENT: ${{ github.event.inputs.environment || 'prod' }} | |
| MOTHERDUCK_TOKEN: ${{ secrets.MOTHERDUCK_TOKEN }} | |
| jobs: | |
| meltano-run: | |
| name: "Meltano Job: ${{ matrix.job }}" | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ghcr.io/meltano/meltano:latest-python3.11 | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GHCR_IO_TOKEN }} | |
| env: | |
| TAP_GETPOCKET_CONSUMER_KEY: ${{ secrets.TAP_GETPOCKET_CONSUMER_KEY }} | |
| TAP_GETPOCKET_ACCESS_TOKEN: ${{ secrets.TAP_GETPOCKET_ACCESS_TOKEN }} | |
| TAP_GITHUB_AUTH_TOKEN: ${{ secrets.TAP_GITHUB_AUTH_TOKEN }} | |
| TAP_READTHEDOCS_TOKEN: ${{ secrets.TAP_READTHEDOCS_TOKEN }} | |
| TAP_STACKEXCHANGE_KEY: ${{ secrets.TAP_STACKEXCHANGE_KEY }} | |
| TARGET_DUCKDB_PATH: ${{ secrets.TARGET_DUCKDB_PATH }} | |
| strategy: | |
| matrix: | |
| include: | |
| - job: "pocket-to-duckdb" | |
| schedule: "daily_pocket" | |
| - job: "stackoverflow-to-duckdb" | |
| schedule: "daily_stackoverflow" | |
| - job: "rtd-to-duckdb" | |
| schedule: "daily_readthedocs" | |
| - job: "gh-to-duckdb" | |
| schedule: "daily_github" | |
| - job: "pypistats-to-duckdb" | |
| schedule: "daily_pypistats" | |
| fail-fast: false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Check Meltano | |
| run: | | |
| meltano --version | |
| - name: Install Plugins | |
| env: | |
| PIP_PREFER_BINARY: "1" | |
| run: | | |
| meltano install --schedule ${{ matrix.schedule }} | |
| - name: Run | |
| env: | |
| MELTANO_FF_STRICT_ENV_VAR_MODE: true | |
| run: | | |
| meltano run ${{ matrix.job }} | |
| - name: Upload metrics log | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: singer-metrics-${{ matrix.job }} | |
| path: logs/singer_metrics.log | |
| lint: | |
| name: Lint models | |
| runs-on: ubuntu-latest | |
| continue-on-error: true | |
| container: | |
| image: ghcr.io/meltano/meltano:latest-python3.11 | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GHCR_IO_TOKEN }} | |
| env: | |
| DUCKDB_PATH: ${{ secrets.DUCKDB_PATH }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Check Meltano | |
| run: | | |
| meltano --version | |
| - name: Install SQLFluff | |
| run: | | |
| meltano install utility sqlfluff | |
| - name: Lint | |
| env: | |
| MELTANO_CLI_LOG_LEVEL: info | |
| run: | | |
| meltano invoke sqlfluff:lint --format github-annotation-native -v transform/models evidence/sources | |
| transform: | |
| name: Data Transformation with dbt-duckdb | |
| runs-on: ubuntu-latest | |
| needs: [meltano-run, lint] | |
| container: | |
| image: ghcr.io/meltano/meltano:latest-python3.11 | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GHCR_IO_TOKEN }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Check Meltano | |
| run: | | |
| meltano --version | |
| - name: Install dbt-duckdb | |
| run: | | |
| meltano install transformer dbt-duckdb | |
| - name: Transform and Test | |
| run: | | |
| meltano run dbt-duckdb:build | |
| # TODO: Uncomment this block to build the Evidence site with Meltano | |
| # build-evidence: | |
| # name: Build Evidence site locally | |
| # needs: [transform] | |
| # runs-on: ubuntu-latest | |
| # container: | |
| # image: ghcr.io/meltano/meltano:latest-python3.11 | |
| # credentials: | |
| # username: ${{ github.actor }} | |
| # password: ${{ secrets.GHCR_IO_TOKEN }} | |
| # steps: | |
| # - uses: actions/checkout@v4 | |
| # - name: Check Meltano | |
| # run: | | |
| # meltano --version | |
| # - name: Install evidence | |
| # run: | | |
| # meltano install transform dbt-duckdb | |
| # meltano install utility evidence | |
| # - name: Build | |
| # run: | | |
| # meltano invoke evidence npm install | |
| # meltano invoke evidence npm run sources | |
| # meltano invoke evidence build --strict | |
| # - name: Upload Evidence build | |
| # uses: actions/upload-artifact@v4 | |
| # with: | |
| # name: evidence-build | |
| # path: evidence/build | |
| build-evidence: | |
| name: Build Evidence site locally using Node | |
| needs: [transform] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: '18' | |
| cache: npm | |
| cache-dependency-path: evidence/package-lock.json | |
| - run: | | |
| cd evidence | |
| npm install | |
| npm run sources | |
| npm run build:strict | |
| - name: Upload Evidence build | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: evidence-build | |
| path: evidence/build | |
| deploy-evidence: | |
| name: Deploy Evidence site | |
| needs: [build-evidence] | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set Git config | |
| run: | | |
| git config --global --add safe.directory /__w/meltano-dataops/meltano-dataops | |
| - name: Download Evidence build | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: evidence-build | |
| path: evidence/build | |
| - name: Deploy | |
| uses: amondnet/[email protected] | |
| with: | |
| github-comment: ${{ github.event_name == 'pull_request' && 'true' || 'false' }} | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| vercel-token: ${{ secrets.VERCEL_TOKEN }} | |
| vercel-org-id: ${{ secrets.VERCEL_ORG_ID }} | |
| vercel-project-id: ${{ secrets.VERCEL_PROJECT_ID }} | |
| vercel-args: ${{ github.event_name == 'pull_request' && '' || '--prod' }} |