diff --git a/.github/workflows/create-job-server-pr.yml b/.github/workflows/create-job-server-pr.yml
index e96a8715..502448ba 100644
--- a/.github/workflows/create-job-server-pr.yml
+++ b/.github/workflows/create-job-server-pr.yml
@@ -37,15 +37,27 @@ jobs:
         id: create_pr
         uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6.1.0
         with:
-          token: "${{ secrets.JOB_SERVER_PR_TOKEN }}"
           branch: bot/update-interactive-templates
+          base: main
+          add-paths: requirements.prod.*
+          author: "opensafely-github-bot <opensafely-github-bot@users.noreply.github.com>"
+          committer: "opensafely-github-bot <opensafely-github-bot@users.noreply.github.com>"
           commit-message: "feat: Update interactive templates to ${{ env.UPDATED_VERSION }}"
           title: "Update interactive templates to ${{ env.UPDATED_VERSION }}"
-          body: "Update interactive-templates to ${{ env.UPDATED_VERSION }}"
-
-      # untested
-      #- name: Enable automerge
-      #  if: steps.create_pr.outputs.pull-request-operation == 'created'
-      #  env:
-      #    GH_TOKEN: "${{ secrets.JOB_SERVER_PR_TOKEN }} "
-      #  run: gh pr merge --merge --auto --squash ${{ steps.create_pr.outputs.pull-request-number }}
+          body: |
+            To get tests to run on this PR there's an odd workflow:
+             - Approve it
+             - Close it
+             - Re-open it
+             - Re-enable automerge
+
+            You can read more on why this is needed in the `create-pull-request` [docs][1].
+
+            [1]: https://github.com/peter-evans/create-pull-request/blob/main/docs/concepts-guidelines.md#triggering-further-workflow-runs
+
+      # The PR will still require manual approval, this just reduces it to a one-click process
+      - name: Enable automerge
+        if: steps.create_pr.outputs.pull-request-operation == 'created'
+        run: gh pr merge --auto --squash ${{ steps.create_pr.outputs.pull-request-number }}
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}