Configure npm distribution mirror and inventory syncing

Updates the automation workflows to: - use reusable workflows to reduce duplication of steps - adds support for `npm` in the list of distributions to mirror to S3 - adds support for `npm` for synchronizing `inventory.toml` updates - updates the helper binaries used in the automation to support `npm` [W-13916385](https://gus.lightning.force.com/a07EE00001XwfoGYAR)
heroku · Aug 11, 2023 · 9f328b8 · 9f328b8
1 parent 3859857
commit 9f328b8
Show file tree

Hide file tree

Showing 5 changed files with 46 additions and 48 deletions.
diff --git a/.github/workflows/_mirror-distribution.yml b/.github/workflows/_mirror-distribution.yml
@@ -6,9 +6,6 @@ env:
 on:
   workflow_call:
     inputs:
-      name:
-        type: string
-        required: true
       distribution:
         type: string
         required: true
@@ -22,7 +19,7 @@ on:
 
 jobs:
   get-unmirrored-versions:
-    name: Get unmirrored versions - ${{ inputs.name }}
+    name: Get Unmirrored Versions
     runs-on: ubuntu-22.04
     outputs:
       versions: ${{ steps.get-unmirrored-versions.outputs.versions }}
@@ -44,53 +41,41 @@ jobs:
 
   mirror-node-distribution:
     if: inputs.distribution == 'node'
-    name: Mirror Distribution - ${{ inputs.name }} - ${{ matrix.version }} ${{ matrix.platform }}
+    name: Mirror v${{ matrix.version }} ${{ matrix.platform && format('({0})', matrix.platform) || '' }}
     needs: [get-unmirrored-versions]
     runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
       matrix:
-        platform: [ "linux-x64" ]
+        distribution: [${{ inputs.distribution }}]
         version: ${{ fromJson(needs.get-unmirrored-versions.outputs.versions) }}
+        include:
+          - distribution: node
+            platform: linux-x64
     steps:
       - name: Checkout
         uses: actions/checkout@v3
 
       - name: Download and verify distribution
-        run: common/bin/download-verify-node "${{ matrix.version }}" "${{ matrix.platform }}"
-
-      - name: Upload Node.js distribution to Nodebin S3 bucket
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          AWS_DEFAULT_REGION: us-east-1
-        run: >
-          aws s3 cp
-          "node-v${{ matrix.version }}-${{ matrix.platform }}.tar.gz"
-          "s3://${{ secrets.AWS_S3_BUCKET }}/node/release/${{ matrix.platform}}/node-v${{ matrix.version }}-${{ matrix.platform }}.tar.gz"
-
-  mirror-npm-package-distribution:
-    if: inputs.distribution != 'node'
-    name: Mirror Distribution - ${{ inputs.name }} - ${{ matrix.version }}
-    runs-on: ubuntu-22.04
-    needs: [get-unmirrored-versions]
-    strategy:
-      fail-fast: false
-      matrix:
-        version: ${{ fromJson(needs.get-unmirrored-versions.outputs.versions) }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Download and verify npm package
-        run: common/bin/download-verify-npm-package ${{ inputs.distribution }} "${{ matrix.version }}"
+        run: |
+          if [ "${{ matrix.distribution }}" = "node" ]; then
+            common/bin/download-verify-node "${{ matrix.version }}" "${{ matrix.platform }}"
+          else
+            common/bin/download-verify-npm-package ${{ inputs.distribution }} "${{ matrix.version }}"
+          fi
 
-      - name: Upload distribution to S3 bucket
+      - name: Upload ${{ matrix.version }} to S3
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_DEFAULT_REGION: us-east-1
-        run: >
-          aws s3 cp
-          "${{ inputs.distribution }}-v${{ matrix.version }}.tar.gz"
-          "s3://${{ secrets.AWS_S3_BUCKET }}/${{ inputs.distribution }}/release/${{ inputs.distribution }}-v${{ matrix.version }}.tar.gz"
+        run: |
+          if [ "${{ matrix.distribution }}" = "node" ]; then
+            aws s3 cp \ 
+              "node-v${{ matrix.version }}-${{ matrix.platform }}.tar.gz" \
+              "s3://${{ secrets.AWS_S3_BUCKET }}/node/release/${{ matrix.platform}}/node-v${{ matrix.version }}-${{ matrix.platform }}.tar.gz"
+          else 
+            aws s3 cp \
+              "${{ inputs.distribution }}-v${{ matrix.version }}.tar.gz" \
+              "s3://${{ secrets.AWS_S3_BUCKET }}/${{ inputs.distribution }}/release/${{ inputs.distribution }}-v${{ matrix.version }}.tar.gz"
+          fi
diff --git a/.github/workflows/_update-inventory.yml b/.github/workflows/_update-inventory.yml
@@ -6,22 +6,25 @@ env:
 on:
   workflow_call:
     inputs:
-      name:
+      buildpack_id:
         type: string
         required: true
-      distribution:
+      buildpack_path:
         type: string
         required: true
-      buildpack_id:
+      distribution:
         type: string
         required: true
-      buildpack_path:
+      name:
         type: string
         required: true
+    secrets:
+      LINGUIST_GH_PRIVATE_KEY:
+        required: true
 
 jobs:
   update-inventory:
-    name: Update Inventory - ${{ inputs.name }}
+    name: Update Inventory
     runs-on: pub-hk-ubuntu-22.04-small
     steps:
       - uses: heroku/use-app-token-action@main

diff --git a/.github/workflows/inventory.yml b/.github/workflows/inventory.yml
@@ -7,25 +7,31 @@ on:
 
 jobs:
   update-nodejs-inventory:
+    name: Node.js
     uses: ./.github/workflows/_update-inventory.yml
     with:
-      name: Node.js Engine
+      name: Node.js
       distribution: node
       buildpack_id: heroku/nodejs-engine
       buildpack_path: buildpacks/nodejs-engine
+    secrets: inherit
 
   update-yarn-inventory:
+    name: Yarn
     uses: ./.github/workflows/_update-inventory.yml
     with:
       name: Yarn
       distribution: yarn
       buildpack_id: heroku/nodejs-yarn
       buildpack_path: buildpacks/nodejs-yarn
+    secrets: inherit
 
   update-npm-inventory:
+    name: npm
     uses: ./.github/workflows/_update-inventory.yml
     with:
       name: NPM
       distribution: npm
       buildpack_id: heroku/nodejs-npm-engine
       buildpack_path: buildpacks/nodejs-npm-engine
+    secrets: inherit
diff --git a/.github/workflows/mirror.yml b/.github/workflows/mirror.yml
@@ -6,22 +6,22 @@ on:
 
 jobs:
   mirror-node:
+    name: Mirror Node.js
     uses: ./.github/workflows/_mirror-distribution.yml
     with:
-      name: Node.js
       distribution: node
     secrets: inherit
 
   mirror-yarn:
+    name: Mirror Yarn
     uses: ./.github/workflows/_mirror-distribution.yml
     with:
-      name: Yarn
       distribution: yarn
     secrets: inherit
 
   mirror-npm:
+    name: Mirror npm
     uses: ./.github/workflows/_mirror-distribution.yml
     with:
-      name: NPM
       distribution: npm
     secrets: inherit
diff --git a/common/bin/download-verify-npm-package b/common/bin/download-verify-npm-package
@@ -25,8 +25,12 @@ if [ "yarn" = "${package_name}" ]; then
 	package_name=$([ "$major_version" -ge 2 ] && echo "@yarnpkg/cli-dist" || echo "yarn")
 fi
 
+npm_url="https://registry.npmjs.com/${package_name}/${package_version}"
+
+echo "Determining dist url from ${npm_url}"
+url=$(curl -sSf "npm_url" | jq -r '.dist.tarball')
+
 echo "Downloading ${package_name} tarball..." >&2
-url=$(curl -sSf "https://registry.npmjs.com/${package_name}/${package_version}" | jq -r '.dist.tarball')
 curl -sSf -o "./${package_name}-v${package_version}.tar.gz" "${url}"
 
 # Check the file's sha against npm's published sha. This section assumes all