Metadata Extract - Airbyte #9
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Metadata Extract - Airbyte | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| environment: | |
| description: 'The environment to run the workflow in' | |
| required: true | |
| type: choice | |
| default: 'preview' | |
| options: | |
| - 'preview' | |
| - 'production' | |
| schedule: | |
| - cron: '0 9 * * 1' # Run at midnight UTC on Mondays | |
| jobs: | |
| get_variants_list: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| airbyte_matrix: ${{ steps.setmatrix_airbyte.outputs.airbyte_matrix }} | |
| steps: | |
| - uses: actions/[email protected] | |
| - uses: astral-sh/setup-uv@v3 | |
| with: | |
| version: ">=0.4.30" | |
| - name: Install hub-utils | |
| run: uv tool install git+https://github.com/meltano/hub-utils.git@main | |
| - name: Get Variants List - Airbyte | |
| id: get-variants-list-airbyte | |
| run: echo "VARIANTS_AIRBYTE=$(hub-utils get-variant-names $(pwd) --metadata-type=airbyte)" >> $GITHUB_OUTPUT | |
| - name: Set Dynamic Airbyte Matrix | |
| id: setmatrix_airbyte | |
| run: | | |
| matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-airbyte.outputs.VARIANTS_AIRBYTE }}}" | |
| echo "airbyte_matrix=$matrixStringifiedObject" >> $GITHUB_ENV | |
| metadata_extract_airbyte_p1: | |
| name: Airbyte - Part 1 Metadata Extract | |
| needs: get_variants_list | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.get_variants_list.outputs.airbyte_matrix) }} | |
| container: | |
| image: airbyte/${{ matrix.source-name }}:latest | |
| steps: | |
| - name: Get Airbyte Spec | |
| run: | | |
| $AIRBYTE_ENTRYPOINT spec | grep '^.*"type":\s*"SPEC".*$' > extractor--${{ matrix.source-name }}--airbyte.json | |
| id: get-airbyte-stdout | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: extractor--${{ matrix.source-name }}--airbyte.json | |
| path: extractor--${{ matrix.source-name }}--airbyte.json | |
| metadata_extract_airbyte_p2: | |
| name: Airbyte - Part 2 Metadata Extract | |
| if: ${{ always() }} | |
| needs: [metadata_extract_airbyte_p1, get_variants_list] | |
| runs-on: ubuntu-latest | |
| environment: ${{ github.event.inputs.environment || github.event_name == 'schedule' && 'production' || 'preview' }} | |
| env: | |
| AWS_S3_BUCKET: "${{secrets.HUB_METADATA_S3_BUCKET }}" | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.get_variants_list.outputs.airbyte_matrix) }} | |
| permissions: | |
| id-token: write # This is required for requesting the JWT | |
| steps: | |
| - uses: actions/[email protected] | |
| - name: Configure AWS credentials | |
| uses: aws-actions/[email protected] | |
| with: | |
| role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} | |
| aws-region: us-west-2 | |
| role-session-name: "GitHubActions" | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: extractor--${{ matrix.source-name }}--airbyte.json | |
| - uses: astral-sh/setup-uv@v3 | |
| with: | |
| version: ">=0.4.30" | |
| - name: Install hub-utils | |
| run: uv tool install git+https://github.com/meltano/hub-utils.git@main | |
| - name: Upload Metadata | |
| run: hub-utils upload-airbyte "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" extractor--${{ matrix.source-name }}--airbyte.json |