diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 416c246..f7d1702 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -14,3 +14,40 @@ jobs:
 
       - name: build
         run: nix-build -A ci
+
+  # FIXME: For now adding it here so it can be tested in a PR without changing release.yml
+  nixpkgs-pr:
+    runs-on: ubuntu-latest
+    # needs: release
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # FIXME: Update before merging
+          repository: tweag/nixpkgs
+
+      - name: Get version before
+        run: echo "before=$(<pkgs/test/check-by-name/pinned-version.txt)" >> $GITHUB_ENV
+
+      - name: Update the version
+        # We run bash explicitly instead of using the files shebang,
+        # because the shebang is using nix-shell with dependencies we already have by default
+        run: bash pkgs/test/check-by-name/update-pinned-tool.sh
+
+      - name: Get version after
+        run: echo "after=$(<pkgs/test/check-by-name/pinned-version.txt)" >> $GITHUB_ENV
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v6
+        with:
+          # To trigger CI for automated PRs, we use a separate machine account
+          # See https://github.com/peter-evans/create-pull-request/blob/main/docs/concepts-guidelines.md#workarounds-to-trigger-further-workflow-runs
+          # and https://github.com/peter-evans/create-pull-request/blob/main/docs/concepts-guidelines.md#push-pull-request-branches-to-a-fork
+          token: ${{ secrets.MACHINE_USER_PAT }}
+          push-to-fork: infinixbot/nixpkgs
+          committer: infinixbot <infinixbot@infinisil.com>
+          author: infinixbot <infinixbot@infinisil.com>
+          branch: check-by-name-update
+          commit-message: "nixpkgs-check-by-name: ${{ env.before }} -> ${{ env.after }}"
+          title: "nixpkgs-check-by-name: ${{ env.before }} -> ${{ env.after }}"
+          body: |
+            Update the nixpkgs-check-by-name tooling version used by CI.