From 1b49ab444be16d0ac0763c5493164b5a97c505be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Tue, 5 Nov 2024 12:02:08 -0600 Subject: [PATCH] ci: Use the correct Python executables for extracting plugin metadata --- .../workflows/metadata-extract-extractors.yml | 52 +++++++------------ .../workflows/metadata-extract-loaders.yml | 50 +++++++----------- 2 files changed, 39 insertions(+), 63 deletions(-) diff --git a/.github/workflows/metadata-extract-extractors.yml b/.github/workflows/metadata-extract-extractors.yml index 0834d63df..3194de5b0 100644 --- a/.github/workflows/metadata-extract-extractors.yml +++ b/.github/workflows/metadata-extract-extractors.yml @@ -70,7 +70,7 @@ jobs: role-session-name: "GitHubActions" - name: Install hub-utils - run: pipx install git+https://github.com/meltano/hub-utils.git + run: pipx install git+https://github.com/meltano/hub-utils.git@edgarrmondragon/fix/custom-python-executable # This starts the attempts to install and extract metadata. Since we don't # know what python versions are accepted we start with the latest and continue @@ -78,59 +78,47 @@ jobs: - name: Install Python 3.12 uses: actions/setup-python@v5 with: - python-version: '3.12' - + python-version: | + 3.13 + 3.12 + 3.11 + 3.10 + 3.9 + 3.8 + + - name: Extract Metadata (3.13) + id: extract_313 + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.13 + continue-on-error: true + - name: Extract Metadata (3.12) + if: always() && (steps.extract_313.outcome == 'failure') id: extract_312 - run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.12 continue-on-error: true - - name: Install Python 3.11 (Only if 3.12 failed) - if: always() && (steps.extract_312.outcome == 'failure') - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: Extract Metadata (3.11) if: always() && (steps.extract_312.outcome == 'failure') id: extract_311 - run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.11 continue-on-error: true - - name: Install Python 3.10 (Only if 3.11 failed) - if: always() && (steps.extract_311.outcome == 'failure') - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - name: Extract Metadata (3.10) if: always() && (steps.extract_311.outcome == 'failure') id: extract_310 - run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.10 continue-on-error: true - - name: Install Python 3.9 (Only if 3.10 failed) - if: always() && (steps.extract_310.outcome == 'failure') - uses: actions/setup-python@v5 - with: - python-version: '3.9' - - name: Extract Metadata (3.9) if: always() && (steps.extract_310.outcome == 'failure') id: extract_39 - run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.9 continue-on-error: true - - name: Install Python 3.8 (Only if 3.9 failed) - if: always() && (steps.extract_39.outcome == 'failure') - uses: actions/setup-python@v5 - with: - python-version: '3.8' - - name: Extract Metadata (3.8) if: always() && (steps.extract_39.outcome == 'failure') id: extract_38 - run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.8 metadata_extract_sdk_ex_p2: name: SDK Extractors - Metadata Extract (p2) diff --git a/.github/workflows/metadata-extract-loaders.yml b/.github/workflows/metadata-extract-loaders.yml index 1c482c750..5dc9f2f0c 100644 --- a/.github/workflows/metadata-extract-loaders.yml +++ b/.github/workflows/metadata-extract-loaders.yml @@ -59,7 +59,7 @@ jobs: role-session-name: "GitHubActions" - name: Install hub-utils - run: pipx install git+https://github.com/meltano/hub-utils.git + run: pipx install git+https://github.com/meltano/hub-utils.git@edgarrmondragon/fix/custom-python-executable # This starts the attempts to install and extract metadata. Since we don't # know what python versions are accepted we start with the latest and continue @@ -67,56 +67,44 @@ jobs: - name: Install Python 3.12 uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: | + 3.13 + 3.12 + 3.11 + 3.10 + 3.9 + 3.8 + + - name: Extract Metadata (3.13) + id: extract_313 + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.13 + continue-on-error: true - name: Extract Metadata (3.12) + if: always() && (steps.extract_313.outcome == 'failure') id: extract_312 - run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.12 continue-on-error: true - - name: Install Python 3.11 (Only if 3.12 failed) - if: always() && (steps.extract_312.outcome == 'failure') - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: Extract Metadata (3.11) if: always() && (steps.extract_312.outcome == 'failure') id: extract_311 - run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.11 continue-on-error: true - - name: Install Python 3.10 (Only if 3.11 failed) - if: always() && (steps.extract_311.outcome == 'failure') - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - name: Extract Metadata (3.10) if: always() && (steps.extract_311.outcome == 'failure') id: extract_310 - run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.10 continue-on-error: true - - name: Install Python 3.9 (Only if 3.10 failed) - if: always() && (steps.extract_310.outcome == 'failure') - uses: actions/setup-python@v5 - with: - python-version: '3.9' - - name: Extract Metadata (3.9) if: always() && (steps.extract_310.outcome == 'failure') id: extract_39 - run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.9 continue-on-error: true - - name: Install Python 3.8 (Only if 3.9 failed) - if: always() && (steps.extract_39.outcome == 'failure') - uses: actions/setup-python@v5 - with: - python-version: '3.8' - - name: Extract Metadata (3.8) if: always() && (steps.extract_39.outcome == 'failure') id: extract_38 - run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.8