Configure npm distribution mirror and inventory syncing

Updates the automation workflows to: - use reusable workflows to reduce duplication of steps - adds support for `npm` in the list of distributions to mirror to S3 - adds support for `npm` for synchronizing `inventory.toml` updates - updates the helper binaries used in the automation to support `npm` [W-13916385](https://gus.lightning.force.com/a07EE00001XwfoGYAR)
heroku · Aug 9, 2023 · 940ca48 · 940ca48
1 parent 0aa417f
commit 940ca48
Show file tree

Hide file tree

Showing 12 changed files with 269 additions and 175 deletions.
diff --git a/.github/workflows/_mirror-distribution.yml b/.github/workflows/_mirror-distribution.yml
@@ -0,0 +1,93 @@
+name: _get-unmirrored-distributions
+
+on:
+  workflow_call:
+    inputs:
+      name:
+        type: string
+        required: true
+      distribution:
+        type: string
+        required: true
+    secrets:
+      AWS_S3_BUCKET:
+        required: true
+      AWS_ACCESS_KEY_ID:
+        required: true
+      AWS_SECRET_ACCESS_KEY:
+        required: true
+
+jobs:
+  get-unmirrored-versions:
+    name: Get unmirrored versions - ${{ inputs.name }}
+    runs-on: ubuntu-22.04
+    outputs:
+      versions: ${{ steps.get-unmirrored-versions.outputs.versions }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+
+      - name: Update Rust toolchain
+        run: rustup update
+
+      - name: Rust Cache
+        uses: Swatinem/[email protected]
+
+      - name: Get unmirrored versions
+        id: get-unmirrored-versions
+        env:
+          AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
+        run: echo "versions=$(cargo run --bin list_unmirrored_versions ${{ inputs.distribution }})" >> $GITHUB_OUTPUT
+
+  mirror-node-distribution:
+    if: inputs.distribution == 'node'
+    name: Mirror Distribution - ${{ inputs.name }} - ${{ matrix.version }} ${{ matrix.platform }}
+    needs: [get-unmirrored-versions]
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [ "linux-x64" ]
+        version: ${{ fromJson(needs.get-unmirrored-versions.outputs.versions) }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Download and verify distribution
+        run: common/bin/download-verify-node "${{ matrix.version }}" "${{ matrix.platform }}"
+
+      - name: Upload Node.js distribution to Nodebin S3 bucket
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: us-east-1
+        run: >
+          aws s3 cp
+          "node-v${{ matrix.version }}-${{ matrix.platform }}.tar.gz"
+          "s3://${{ secrets.AWS_S3_BUCKET }}/node/release/${{ matrix.platform}}/node-v${{ matrix.version }}-${{ matrix.platform }}.tar.gz"
+
+  mirror-npm-package-distribution:
+    if: inputs.distribution != 'node'
+    name: Mirror Distribution - ${{ inputs.name }} - ${{ matrix.version }} ${{ matrix.platform }}
+    runs-on: ubuntu-22.04
+    needs: [get-unmirrored-versions]
+    strategy:
+      fail-fast: false
+      matrix:
+        version: ${{ fromJson(needs.get-unmirrored-yarn-versions.outputs.versions) }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Download and verify npm package
+        run: common/bin/download-verify-npm-package ${{ inputs.distribution }} "${{ matrix.version }}"
+
+      - name: Upload distribution to S3 bucket
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: us-east-1
+        run: >
+          aws s3 cp
+          "${{ inputs.distribution }}-v${{ matrix.version }}.tar.gz"
+          "s3://${{ secrets.AWS_S3_BUCKET }}/${{ inputs.distribution }}/release/${{ inputs.distribution }}-v${{ matrix.version }}.tar.gz"
diff --git a/.github/workflows/_update-inventory.yml b/.github/workflows/_update-inventory.yml
@@ -0,0 +1,81 @@
+name: _update-inventory
+
+env:
+  CARGO_TERM_COLOR: always
+
+on:
+  workflow_call:
+    inputs:
+      name:
+        type: string
+        required: true
+      distribution:
+        type: string
+        required: true
+      buildpack_id:
+        type: string
+        required: true
+      buildpack_path:
+        type: string
+        required: true
+
+jobs:
+  update-inventory:
+    name: Update Inventory - ${{ inputs.name }}
+    runs-on: pub-hk-ubuntu-22.04-small
+    steps:
+      - uses: heroku/use-app-token-action@main
+        id: generate-token
+        with:
+          app_id: ${{ vars.LINGUIST_GH_APP_ID }}
+          private_key: ${{ secrets.LINGUIST_GH_PRIVATE_KEY }}
+
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+        with:
+          token: ${{ steps.generate-token.outputs.app_token }}
+
+      - name: Configure git
+        run: |
+          git config --global user.name ${{ vars.LINGUIST_GH_APP_USERNAME }}
+          git config --global user.email ${{ vars.LINGUIST_GH_APP_EMAIL }}
+
+      - name: Update Rust toolchain
+        run: rustup update
+
+      - name: Rust cache
+        uses: Swatinem/[email protected]
+
+      - name: Set Diff Message
+        id: set-diff-msg
+        run: |
+          delimiter="$(openssl rand -hex 8)"
+          {
+            echo "msg<<${delimiter}"
+            #cargo run --bin diff_versions ${{ inputs.distribution }} ${{ inputs.buildpack_path }}/inventory.toml
+            echo "Added Node.js version 20.3.1.\nAdded Node.js version x.y.z\nRemoved Node.js version a.b.c"
+            echo "${delimiter}"
+          } >> $GITHUB_OUTPUT
+
+      - name: Rebuild Inventory
+        run: cargo run --bin generate_inventory ${{ inputs.distribution }} > ${{ inputs.buildpack_path }}/inventory.toml
+
+      - name: Update Changelog
+        run: echo "${{ steps.set-diff-msg.outputs.msg }}" | xargs -r -I '{}' perl -i -p -e 
+          ${{ inputs.buildpack_path }}/CHANGELOG.md
+
+      - name: Create Pull Request
+        id: pr
+        uses: peter-evans/create-pull-request@v5
+        with:
+          token: ${{ steps.generate-token.outputs.app_token }}
+          title: Update Inventory - ${{ inputs.name }}
+          branch: update-${{ inputs.distribution }}-inventory
+          commit-message: "Update Inventory for ${{ inputs.buildpack_id }}\n\n${{ steps.set-diff-msg.outputs.msg }}"
+          body: "Automated pull-request to update ${{ inputs.buildpack_id }} inventory:\n\n${{ steps.set-diff-msg.outputs.msg }}"
+
+      - name: Configure PR
+        if: steps.pr.outputs.pull-request-operation == 'created'
+        env:
+          GH_TOKEN: ${{ steps.generate-token.outputs.app_token }}
+        run: gh pr merge --squash --auto "${{ steps.pr.outputs.pull-request-number }}"
diff --git a/.github/workflows/inventory.yml b/.github/workflows/inventory.yml
@@ -1,84 +1,31 @@
 name: Update Inventory
+
 on:
   workflow_dispatch:
   schedule:
     - cron: '00 4 * * 1-5'
 
 jobs:
   update-nodejs-inventory:
-    name: Update Node.js Engine Inventory
-    runs-on: pub-hk-ubuntu-22.04-small
-    steps:
-      - name: Checkout Repo
-        uses: actions/checkout@v3
-      - id: install-rust-toolchain
-        name: Install Rust Toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-      - id: set-diff-msg
-        name: Set Diff Message
-        run: echo "::set-output name=msg::$(cargo run --bin diff_versions node buildpacks/nodejs-engine/inventory.toml)"
-      - name: Rebuild Inventory
-        run: "cargo run --bin generate_inventory node > buildpacks/nodejs-engine/inventory.toml"
-      - name: Update Changelog
-        run: echo "${{ steps.set-diff-msg.outputs.msg }}" | xargs -r -I '{}' perl -i -p -e 's/\[Unreleased\]\s+/[Unreleased]\n\n{}/' buildpacks/nodejs-engine/CHANGELOG.md
-      - uses: heroku/use-app-token-action@main
-        id: generate-token
-        with:
-          app_id: ${{ vars.LINGUIST_GH_APP_ID }}
-          private_key: ${{ secrets.LINGUIST_GH_PRIVATE_KEY }}
-      - name: Create Pull Request
-        id: pr
-        uses: peter-evans/create-pull-request@v5
-        with:
-          token: ${{ steps.generate-token.outputs.app_token }}
-          title: "Update Node.js Engine Inventory"
-          commit-message: "Update Inventory for heroku/nodejs-engine\n\n${{ steps.set-diff-msg.outputs.msg }}"
-          branch: update-nodejs-inventory
-          labels: "automation"
-          body: "Automated pull-request to update heroku/nodejs-engine inventory:\n\n${{ steps.set-diff-msg.outputs.msg }}"
-      - name: Configure PR
-        if: steps.pr.outputs.pull-request-operation == 'created'
-        run: gh pr merge --squash --auto "${{ steps.pr.outputs.pull-request-number }}"
-        env:
-          GH_TOKEN: ${{ steps.generate-token.outputs.app_token }}
-
+    uses: ./_update-inventory.yml
+    with:
+      name: Node.js Engine
+      distribution: node
+      buildpack_id: heroku/nodejs-engine
+      buildpack_path: buildpacks/nodejs-engine
+
   update-yarn-inventory:
-    name: Update Node.js Yarn Inventory
-    runs-on: pub-hk-ubuntu-22.04-small
-    steps:
-      - name: Checkout Repo
-        uses: actions/checkout@v3
-      - id: install-rust-toolchain
-        name: Install Rust Toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-      - id: set-diff-msg
-        name: Set Diff Message
-        run: echo "::set-output name=msg::$(cargo run --bin diff_versions yarn buildpacks/nodejs-yarn/inventory.toml)"
-      - name: Rebuild Inventory
-        run: "cargo run --bin generate_inventory yarn > buildpacks/nodejs-yarn/inventory.toml"
-      - name: Update Changelog
-        run: echo "${{ steps.set-diff-msg.outputs.msg }}" | xargs -r -I '{}' perl -i -p -e 's/\[Unreleased\]\s+/[Unreleased]\n\n{}/' buildpacks/nodejs-yarn/CHANGELOG.md
-      - uses: heroku/use-app-token-action@main
-        id: generate-token
-        with:
-          app_id: ${{ vars.LINGUIST_GH_APP_ID }}
-          private_key: ${{ secrets.LINGUIST_GH_PRIVATE_KEY }}   
-      - name: Create Pull Request
-        id: pr
-        uses: peter-evans/create-pull-request@v5
-        with:
-          token: ${{ steps.generate-token.outputs.app_token }}
-          title: "Update Node.js Yarn Inventory"
-          commit-message: "Update Inventory for heroku/nodejs-yarn\n\n${{ steps.set-diff-msg.outputs.msg }}"
-          branch: update-yarn-inventory
-          labels: "automation"
-          body: "Automated pull-request to update heroku/nodejs-yarn inventory:\n\n${{ steps.set-diff-msg.outputs.msg }}"
-      - name: Configure PR
-        if: steps.pr.outputs.pull-request-operation == 'created'
-        run: gh pr merge --squash --auto "${{ steps.pr.outputs.pull-request-number }}"
-        env:
-          GH_TOKEN: ${{ steps.generate-token.outputs.app_token }}
+    uses: ./_update-inventory.yml
+    with:
+      name: Yarn
+      distribution: yarn
+      buildpack_id: heroku/nodejs-yarn
+      buildpack_path: buildpacks/nodejs-yarn
+
+  update-npm-inventory:
+    uses: ./_update-inventory.yml
+    with:
+      name: NPM
+      distribution: npm
+      buildpack_id: heroku/nodejs-npm-engine
+      buildpack_path: buildpacks/nodejs-npm-engine
diff --git a/.github/workflows/mirror.yml b/.github/workflows/mirror.yml
@@ -5,89 +5,23 @@ on:
     - cron: '00 1 * * 1-5'
 
 jobs:
-  get-unmirrored-node-versions:
-    name: Get unmirrored Node.js versions
-    runs-on: ubuntu-22.04
-    outputs:
-      versions: ${{ steps.get-unmirrored-versions.outputs.versions }}
-    steps:
-      - name: Checkout Repo
-        uses: actions/checkout@v3
-      - name: Update Rust toolchain
-        run: rustup update
-      - name: Rust Cache
-        uses: Swatinem/[email protected]
-      - id: get-unmirrored-versions
-        name: Get unmirrored Node.js versions
-        env:
-          AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
-        run: echo "versions=$(cargo run --bin list_unmirrored_versions node)" >> $GITHUB_OUTPUT
-
-  get-unmirrored-yarn-versions:
-    name: Get unmirrored Yarn versions
-    runs-on: ubuntu-22.04
-    outputs:
-      versions: ${{ steps.get-unmirrored-versions.outputs.versions }}
-    steps:
-      - name: Checkout Repo
-        uses: actions/checkout@v3
-      - name: Update Rust toolchain
-        run: rustup update
-      - name: Rust Cache
-        uses: Swatinem/[email protected]
-      - id: get-unmirrored-versions
-        name: Get unmirrored Yarn versions
-        env:
-          AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
-        run: echo "versions=$(cargo run --bin list_unmirrored_versions yarn)" >> $GITHUB_OUTPUT
-
-  mirror-node-distribution:
-    name: Mirror Node.js Distribution ${{ matrix.version }} ${{ matrix.platform }}
-    runs-on: ubuntu-22.04
-    needs: get-unmirrored-node-versions
-    strategy:
-      fail-fast: false
-      matrix:
-        platform: ["linux-x64"]
-        version: ${{ fromJson(needs.get-unmirrored-node-versions.outputs.versions) }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Download and Verify Node.js distribution
-        run: common/bin/download-verify-node "${{ matrix.version }}" "${{ matrix.platform }}"
-
-      - name: Upload Node.js distribution to Nodebin S3 bucket
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          AWS_DEFAULT_REGION: us-east-1
-        run: >
-          aws s3 cp
-          "node-v${{ matrix.version }}-${{ matrix.platform }}.tar.gz"
-          "s3://${{ secrets.AWS_S3_BUCKET }}/node/release/${{ matrix.platform}}/node-v${{ matrix.version }}-${{ matrix.platform }}.tar.gz"
-
-  mirror-yarn-distribution:
-    name: Mirror Yarn Distribution ${{ matrix.version }}
-    runs-on: ubuntu-22.04
-    needs: get-unmirrored-yarn-versions
-    strategy:
-      fail-fast: false
-      matrix:
-        version: ${{ fromJson(needs.get-unmirrored-yarn-versions.outputs.versions) }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Download and verify yarn distribution
-        run: common/bin/download-verify-yarn "${{ matrix.version }}"
-
-      - name: Upload Yarn distribution to S3 bucket
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          AWS_DEFAULT_REGION: us-east-1
-        run: >
-          aws s3 cp
-          "yarn-v${{ matrix.version }}.tar.gz"
-          "s3://${{ secrets.AWS_S3_BUCKET }}/yarn/release/yarn-v${{ matrix.version }}.tar.gz"
+  mirror-node:
+    uses: ./_mirror-distribution.yml
+    with:
+      name: Node.js
+      distribution: node
+    secrets: inherit
+
+  mirror-yarn:
+    uses: ./_mirror-distribution.yml
+    with:
+      name: Yarn
+      distribution: yarn
+    secrets: inherit
+
+  mirror-npm:
+    uses: ./_mirror-distribution.yml
+    with:
+      name: NPM
+      distribution: npm
+    secrets: inherit
diff --git a/buildpacks/nodejs-npm-engine/inventory.toml b/buildpacks/nodejs-npm-engine/inventory.toml