From 9f328b82546a55e591d168aaf892ce4b0a2b16b8 Mon Sep 17 00:00:00 2001 From: Colin Casey Date: Fri, 11 Aug 2023 11:11:03 -0300 Subject: [PATCH] Configure npm distribution mirror and inventory syncing Updates the automation workflows to: - use reusable workflows to reduce duplication of steps - adds support for `npm` in the list of distributions to mirror to S3 - adds support for `npm` for synchronizing `inventory.toml` updates - updates the helper binaries used in the automation to support `npm` [W-13916385](https://gus.lightning.force.com/a07EE00001XwfoGYAR) --- .github/workflows/_mirror-distribution.yml | 61 ++++++++-------------- .github/workflows/_update-inventory.yml | 13 +++-- .github/workflows/inventory.yml | 8 ++- .github/workflows/mirror.yml | 6 +-- common/bin/download-verify-npm-package | 6 ++- 5 files changed, 46 insertions(+), 48 deletions(-) diff --git a/.github/workflows/_mirror-distribution.yml b/.github/workflows/_mirror-distribution.yml index 9f037dc1..2d811a84 100644 --- a/.github/workflows/_mirror-distribution.yml +++ b/.github/workflows/_mirror-distribution.yml @@ -6,9 +6,6 @@ env: on: workflow_call: inputs: - name: - type: string - required: true distribution: type: string required: true @@ -22,7 +19,7 @@ on: jobs: get-unmirrored-versions: - name: Get unmirrored versions - ${{ inputs.name }} + name: Get Unmirrored Versions runs-on: ubuntu-22.04 outputs: versions: ${{ steps.get-unmirrored-versions.outputs.versions }} @@ -44,53 +41,41 @@ jobs: mirror-node-distribution: if: inputs.distribution == 'node' - name: Mirror Distribution - ${{ inputs.name }} - ${{ matrix.version }} ${{ matrix.platform }} + name: Mirror v${{ matrix.version }} ${{ matrix.platform && format('({0})', matrix.platform) || '' }} needs: [get-unmirrored-versions] runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: - platform: [ "linux-x64" ] + distribution: [${{ inputs.distribution }}] version: ${{ fromJson(needs.get-unmirrored-versions.outputs.versions) }} + include: + - distribution: node + platform: linux-x64 steps: - name: Checkout uses: actions/checkout@v3 - name: Download and verify distribution - run: common/bin/download-verify-node "${{ matrix.version }}" "${{ matrix.platform }}" - - - name: Upload Node.js distribution to Nodebin S3 bucket - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-east-1 - run: > - aws s3 cp - "node-v${{ matrix.version }}-${{ matrix.platform }}.tar.gz" - "s3://${{ secrets.AWS_S3_BUCKET }}/node/release/${{ matrix.platform}}/node-v${{ matrix.version }}-${{ matrix.platform }}.tar.gz" - - mirror-npm-package-distribution: - if: inputs.distribution != 'node' - name: Mirror Distribution - ${{ inputs.name }} - ${{ matrix.version }} - runs-on: ubuntu-22.04 - needs: [get-unmirrored-versions] - strategy: - fail-fast: false - matrix: - version: ${{ fromJson(needs.get-unmirrored-versions.outputs.versions) }} - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Download and verify npm package - run: common/bin/download-verify-npm-package ${{ inputs.distribution }} "${{ matrix.version }}" + run: | + if [ "${{ matrix.distribution }}" = "node" ]; then + common/bin/download-verify-node "${{ matrix.version }}" "${{ matrix.platform }}" + else + common/bin/download-verify-npm-package ${{ inputs.distribution }} "${{ matrix.version }}" + fi - - name: Upload distribution to S3 bucket + - name: Upload ${{ matrix.version }} to S3 env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: us-east-1 - run: > - aws s3 cp - "${{ inputs.distribution }}-v${{ matrix.version }}.tar.gz" - "s3://${{ secrets.AWS_S3_BUCKET }}/${{ inputs.distribution }}/release/${{ inputs.distribution }}-v${{ matrix.version }}.tar.gz" + run: | + if [ "${{ matrix.distribution }}" = "node" ]; then + aws s3 cp \ + "node-v${{ matrix.version }}-${{ matrix.platform }}.tar.gz" \ + "s3://${{ secrets.AWS_S3_BUCKET }}/node/release/${{ matrix.platform}}/node-v${{ matrix.version }}-${{ matrix.platform }}.tar.gz" + else + aws s3 cp \ + "${{ inputs.distribution }}-v${{ matrix.version }}.tar.gz" \ + "s3://${{ secrets.AWS_S3_BUCKET }}/${{ inputs.distribution }}/release/${{ inputs.distribution }}-v${{ matrix.version }}.tar.gz" + fi diff --git a/.github/workflows/_update-inventory.yml b/.github/workflows/_update-inventory.yml index 19e785ae..cd666cf4 100644 --- a/.github/workflows/_update-inventory.yml +++ b/.github/workflows/_update-inventory.yml @@ -6,22 +6,25 @@ env: on: workflow_call: inputs: - name: + buildpack_id: type: string required: true - distribution: + buildpack_path: type: string required: true - buildpack_id: + distribution: type: string required: true - buildpack_path: + name: type: string required: true + secrets: + LINGUIST_GH_PRIVATE_KEY: + required: true jobs: update-inventory: - name: Update Inventory - ${{ inputs.name }} + name: Update Inventory runs-on: pub-hk-ubuntu-22.04-small steps: - uses: heroku/use-app-token-action@main diff --git a/.github/workflows/inventory.yml b/.github/workflows/inventory.yml index 1daf5db8..7430ddc1 100644 --- a/.github/workflows/inventory.yml +++ b/.github/workflows/inventory.yml @@ -7,25 +7,31 @@ on: jobs: update-nodejs-inventory: + name: Node.js uses: ./.github/workflows/_update-inventory.yml with: - name: Node.js Engine + name: Node.js distribution: node buildpack_id: heroku/nodejs-engine buildpack_path: buildpacks/nodejs-engine + secrets: inherit update-yarn-inventory: + name: Yarn uses: ./.github/workflows/_update-inventory.yml with: name: Yarn distribution: yarn buildpack_id: heroku/nodejs-yarn buildpack_path: buildpacks/nodejs-yarn + secrets: inherit update-npm-inventory: + name: npm uses: ./.github/workflows/_update-inventory.yml with: name: NPM distribution: npm buildpack_id: heroku/nodejs-npm-engine buildpack_path: buildpacks/nodejs-npm-engine + secrets: inherit diff --git a/.github/workflows/mirror.yml b/.github/workflows/mirror.yml index df3a67c8..ab7e6fae 100644 --- a/.github/workflows/mirror.yml +++ b/.github/workflows/mirror.yml @@ -6,22 +6,22 @@ on: jobs: mirror-node: + name: Mirror Node.js uses: ./.github/workflows/_mirror-distribution.yml with: - name: Node.js distribution: node secrets: inherit mirror-yarn: + name: Mirror Yarn uses: ./.github/workflows/_mirror-distribution.yml with: - name: Yarn distribution: yarn secrets: inherit mirror-npm: + name: Mirror npm uses: ./.github/workflows/_mirror-distribution.yml with: - name: NPM distribution: npm secrets: inherit diff --git a/common/bin/download-verify-npm-package b/common/bin/download-verify-npm-package index 211ef3fd..ead0e336 100755 --- a/common/bin/download-verify-npm-package +++ b/common/bin/download-verify-npm-package @@ -25,8 +25,12 @@ if [ "yarn" = "${package_name}" ]; then package_name=$([ "$major_version" -ge 2 ] && echo "@yarnpkg/cli-dist" || echo "yarn") fi +npm_url="https://registry.npmjs.com/${package_name}/${package_version}" + +echo "Determining dist url from ${npm_url}" +url=$(curl -sSf "npm_url" | jq -r '.dist.tarball') + echo "Downloading ${package_name} tarball..." >&2 -url=$(curl -sSf "https://registry.npmjs.com/${package_name}/${package_version}" | jq -r '.dist.tarball') curl -sSf -o "./${package_name}-v${package_version}.tar.gz" "${url}" # Check the file's sha against npm's published sha. This section assumes all