Allow auto merge binding PRs (#988)

This PR allows auto merging related binding PRs once the mmtk-core PR is
merged. It also adds some docs for MMTk team about merging process. The
PR closes #205.

.github/scripts/replace-mmtk-dep.py

@@ -2,35 +2,77 @@
 
 import argparse
 import os.path
+import sys
 import tomlkit
 
 parser = argparse.ArgumentParser(
         description='Replace the mmtk-core dependency of a given VM binding',
         )
 
 parser.add_argument('toml_path', help='Path to Cargo.toml')
-parser.add_argument('mmtk_core_path', help='Path to the mmtk_core repo')
+# The following arguments are exclusive. Use either. If both are supplied, we use the local path.
+# 1. Point to a local path
+parser.add_argument('--mmtk-core-path', help='Path to the mmtk-core repo.')
+# 2. Point to a remote repo
+parser.add_argument('--mmtk-core-git', help='URL to the mmtk-core repo.')
+parser.add_argument('--mmtk-core-rev', help='Revision to use')
 
 args = parser.parse_args()
 
+# Check what we should do.
+if args.mmtk_core_path is not None:
+    how = "point_to_local"
+elif args.mmtk_core_git is not None and args.mmtk_core_rev is not None:
+    how = "point_to_repo"
+else:
+    print("No path or git/rev is supplied. We cannot update the toml")
+    sys.exit(1)
+
 print("Reading TOML from '{}'".format(args.toml_path))
 with open(args.toml_path, "rt") as f:
     toml_data = tomlkit.load(f)
 
+if "mmtk" not in toml_data["dependencies"]:
+    print("Cannot find the mmtk dependency in {}".format(args.toml_path))
+    sys.exit(1)
+
+# The mmtk dependency could be an inlined table for some bindings:
+# [dependencies]
+# mmtk = { git = "...", rev = "..." }
+# But it could be a subtable for other bindings:
+# [dependencies.mmtk]
+# git = "..."
+# rev = "..."
 mmtk_node = toml_data["dependencies"]["mmtk"]
 
-# These keys may specify the locations of the dependency. Remove them.
-for key in ["git", "branch", "version", "registry"]:
-    if key in mmtk_node:
-        print("Deleting dependencies.mmtk.{}".format(key))
-        del mmtk_node[key]
-    else:
-        print("Key dependencies.mmtk.{} does not exist.  Ignored.".format(key))
-
-# Use mmtk-core from the specified local directory.
-mmtk_repo_path = os.path.realpath(args.mmtk_core_path)
-print("Setting dependencies.mmtk.path to {}".format(mmtk_repo_path))
-mmtk_node["path"] = mmtk_repo_path
+def remove_keys(item, keys):
+    for key in keys:
+        if key in item:
+            print("Deleting dependencies.mmtk.{}".format(key))
+            del item[key]
+        else:
+            print("Key dependencies.mmtk.{} does not exist.  Ignored.".format(key))
+
+
+if how == "point_to_local":
+    # We are going to update the dependency to a local path. First remove anything we know that would set a version.
+    # Deleting all the keys will mess up the formatting. But it doesn't matter, as the file with
+    # a dependency to the local path is just temporary.
+    remove_keys(mmtk_node, ["git", "branch", "registry", "rev", "tag", "path", "version"])
+
+    # Use mmtk-core from the specified local directory.
+    mmtk_repo_path = os.path.realpath(args.mmtk_core_path)
+    mmtk_node["path"] = mmtk_repo_path
+    print("Setting dependencies.mmtk.path to {}".format(mmtk_repo_path))
+elif how == "point_to_repo":
+    # We assume the file already includes `git` and `rev`. We just update them. This will preserve the
+    # original formatting and comments.
+
+    # Update git/rev
+    mmtk_node["git"] = args.mmtk_core_git
+    mmtk_node["rev"] = args.mmtk_core_rev
+    print("Setting dependencies.mmtk to git={},rev={}".format(args.mmtk_core_git, args.mmtk_core_rev))
+
 
 print("Writing TOML to '{}'".format(args.toml_path))
 with open(args.toml_path, "wt") as f:

.github/workflows/auto-merge-inner.yml

@@ -0,0 +1,95 @@
+name: Merge Binding PR
+
+on:
+  workflow_call:
+    inputs:
+      # The repository from which the binding PR is submitted, such as qinsoon/mmtk-openjdk
+      repo:
+        required: true
+        type: string
+      # The upstream repository where the binding PR is opened, such as mmtk/mmtk-openjdk
+      base_repo:
+        required: true
+        type: string
+      # The branch name for the PR
+      ref:
+        required: true
+        type: string
+      # The core commit hash that the binding should be using.
+      core_commit:
+        required: true
+        type: string
+      # the command line to update lock file once we update Cargo.toml
+      update_lockfile:
+        required: true
+        type: string
+
+env:
+  MMTK_CORE_WORK_DIR: mmtk-core
+  BINDING_WORK_DIR: mmtk-binding-repo
+
+jobs:
+  process-pr:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check input conditions
+        id: check-input
+        run: |
+          if [[ "${{ inputs.repo }}" == ${{ inputs.base_repo }} ]] && [[ "${{ inputs.ref }}" == "master" ]]; then
+            echo "Conditions not met"
+            echo "::set-output name=skip::true"
+          else
+            echo "::set-output name=skip::false"
+          fi
+        shell: bash
+
+      - name: Checkout MMTk Core
+        uses: actions/checkout@v3
+        if: steps.check-input.outputs.skip == 'false'
+        with:
+          path: ${{ env.MMTK_CORE_WORK_DIR }}
+      - name: Checkout repository
+        if: steps.check-input.outputs.skip == 'false'
+        uses: actions/checkout@v3
+        with:
+          repository: ${{ inputs.repo }}
+          path: ${{ env.BINDING_WORK_DIR }}
+          ref: ${{ inputs.ref }}
+          # Check out with CI_ACCESS_TOKEN so we can push to it.
+          token: ${{ secrets.CI_ACCESS_TOKEN }}
+
+      - name: Get PR number
+        if: steps.check-input.outputs.skip == 'false'
+        id: get-pr
+        run: |
+          PR_NUMBER=$(gh pr list --head ${{ inputs.ref }} --repo ${{ inputs.base_repo }} --json number --jq '.[0].number')
+          echo "::set-output name=pr_number::$PR_NUMBER"
+        shell: bash
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Update mmtk dependency
+        if: steps.check-input.outputs.skip == 'false'
+        working-directory: ${{ env.MMTK_CORE_WORK_DIR }}
+        run: |
+          export MANIFEST_PATH=../${{ env.BINDING_WORK_DIR }}/mmtk/Cargo.toml
+          ./.github/scripts/ci-replace-mmtk-dep.sh $MANIFEST_PATH --mmtk-core-git https://github.com/mmtk/mmtk-core.git --mmtk-core-rev ${{ inputs.core_commit }}
+          ${{ inputs.update_lockfile }} --manifest-path $MANIFEST_PATH
+
+      - name: Push a new commit
+        if: steps.check-input.outputs.skip == 'false'
+        working-directory: ${{ env.BINDING_WORK_DIR }}
+        run: |
+          git config user.name "mmtkgc-bot"
+          git config user.email "[email protected]"
+          git add mmtk/Cargo.toml
+          git add mmtk/Cargo.lock
+          git commit -m "Update mmtk-core to ${{ inputs.core_commit }}"
+          git push
+
+      - name: Enable auto-merge for the PR
+        if: steps.check-input.outputs.skip == 'false'
+        run: |
+          gh pr merge ${{ steps.get-pr.outputs.pr_number }} --auto --repo ${{ inputs.base_repo }} --squash
+        env:
+          GITHUB_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }}

.github/workflows/auto-merge.yml

@@ -0,0 +1,87 @@
+name: Auto Merge Binding PRs
+
+# Only trigger on push to master. The workflow needs to use repo secrets.
+# Triggering on master can make sure we have secrets.
+on:
+  push:
+    branches:
+    - master
+
+jobs:
+  # Figure out the PR that is merged.
+  get-merged-pr:
+    runs-on: ubuntu-latest
+    outputs:
+      commit: ${{ steps.get-commit-hash.outputs.commit }}
+      pr: ${{ steps.get-pr.outputs.pr }}
+    steps:
+      - id: get-commit-hash
+        run: echo "commit=${GITHUB_SHA}" >> "$GITHUB_OUTPUT"
+      - id: get-pr
+        run: |
+          PR_NUMBER=$(gh pr list --search "${GITHUB_SHA}" --state merged --repo mmtk/mmtk-core --json number --jq '.[0].number')
+          echo "pr=$PR_NUMBER" >> "$GITHUB_OUTPUT"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  # Figure out binding PRs.
+  binding-refs:
+    uses: ./.github/workflows/pr-binding-refs.yml
+    needs: get-merged-pr
+    with:
+      pull_request: ${{ needs.get-merged-pr.outputs.pr }}
+
+  check-merge-openjdk-pr:
+    uses: ./.github/workflows/auto-merge-inner.yml
+    needs: [get-merged-pr, binding-refs]
+    with:
+      repo: ${{ needs.binding-refs.outputs.openjdk_binding_repo }}
+      base_repo: mmtk/mmtk-openjdk
+      ref: ${{ needs.binding-refs.outputs.openjdk_binding_ref }}
+      core_commit: ${{ needs.get-merged-pr.outputs.commit }}
+      update_lockfile: cargo build
+    secrets: inherit
+
+  check-merge-jikesrvm-pr:
+    uses: ./.github/workflows/auto-merge-inner.yml
+    needs: [get-merged-pr, binding-refs]
+    with:
+      repo: ${{ needs.binding-refs.outputs.jikesrvm_binding_repo }}
+      base_repo: mmtk/mmtk-jikesrvm
+      ref: ${{ needs.binding-refs.outputs.jikesrvm_binding_ref }}
+      core_commit: ${{ needs.get-merged-pr.outputs.commit }}
+      update_lockfile: cargo build --features nogc --target i686-unknown-linux-gnu
+    secrets: inherit
+
+  check-merge-v8-pr:
+    uses: ./.github/workflows/auto-merge-inner.yml
+    needs: [get-merged-pr, binding-refs]
+    with:
+      repo: ${{ needs.binding-refs.outputs.v8_binding_repo }}
+      base_repo: mmtk/mmtk-v8
+      ref: ${{ needs.binding-refs.outputs.v8_binding_ref }}
+      core_commit: ${{ needs.get-merged-pr.outputs.commit }}
+      update_lockfile: cargo build --features nogc
+    secrets: inherit
+
+  check-merge-julia-pr:
+    uses: ./.github/workflows/auto-merge-inner.yml
+    needs: [get-merged-pr, binding-refs]
+    with:
+      repo: ${{ needs.binding-refs.outputs.julia_binding_repo }}
+      base_repo: mmtk/mmtk-julia
+      ref: ${{ needs.binding-refs.outputs.julia_binding_ref }}
+      core_commit: ${{ needs.get-merged-pr.outputs.commit }}
+      update_lockfile: cargo build --features immix
+    secrets: inherit
+
+  check-merge-ruby-pr:
+    uses: ./.github/workflows/auto-merge-inner.yml
+    needs: [get-merged-pr, binding-refs]
+    with:
+      repo: ${{ needs.binding-refs.outputs.ruby_binding_repo }}
+      base_repo: mmtk/mmtk-ruby
+      ref: ${{ needs.binding-refs.outputs.ruby_binding_ref }}
+      core_commit: ${{ needs.get-merged-pr.outputs.commit }}
+      update_lockfile: cargo build
+    secrets: inherit

.github/workflows/merge-check.yml

@@ -20,7 +20,7 @@ jobs:
           # - this action
           # - pre code review checks for stable Rust (we allow them to fail)
           # - binding tests (it may take long to run)
-          ignoreActions: "ready-to-merge,check-public-api-changes,pre-code-review-checks/x86_64-unknown-linux-gnu/stable,pre-code-review-checks/i686-unknown-linux-gnu/stable,pre-code-review-checks/x86_64-apple-darwin/stable,v8-binding-test,openjdk-binding-test,jikesrvm-binding-test,julia-binding-test,ruby-binding-test"
+          ignoreActions: "ready-to-merge,check-public-api-changes,pre-code-review-checks/x86_64-unknown-linux-gnu/stable,pre-code-review-checks/i686-unknown-linux-gnu/stable,pre-code-review-checks/x86_64-apple-darwin/stable,v8-binding-test,openjdk-binding-test,jikesrvm-binding-test,julia-binding-test,ruby-binding-test (release),ruby-binding-test (debug)"
           # This action uses API. We have a quota of 1000 per hour.
           checkInterval: 600
         env:

.github/workflows/post-review-ci.yml

@@ -200,7 +200,7 @@ jobs:
           ref: mmtk
 
       - name: Override mmtk-core dependency for binding
-        run: ./.github/scripts/ci-replace-mmtk-dep.sh ../mmtk-ruby/mmtk/Cargo.toml .
+        run: ./.github/scripts/ci-replace-mmtk-dep.sh ../mmtk-ruby/mmtk/Cargo.toml --mmtk-core-path .
         working-directory: mmtk-core
 
       - name: Setup environment

docs/team/README.md

@@ -0,0 +1,2 @@
+This is the folder for public docs for the MMTk team. The MMTk team refers to the members in the
+MMTk GitHub organization.

docs/team/pull_request.md

@@ -0,0 +1,76 @@
+# Pull Requests
+
+## API Breaking Changes
+
+If a PR includes API breaking changes, it is the responsibility of the MMTk team to make sure
+the officially supported bindings are updated to accomodate the changes.
+A team member may open a PR in the bindings him/herself if the required
+changes are trivial enough. Otherwise he/she may ask the team member that is responsible for a specific
+binding to request an update. The corresponding binding PRs need to be ready to merge before the mmtk-core PR
+can be merged.
+
+### Testing PRs with API breaking changes
+
+If an MMTk core PR should be tested with other bindings PRs, one can specify the binding branch that
+should be tested with by adding a comment like below (see https://github.com/mmtk/mmtk-core/blob/master/.github/workflows/pr-binding-refs.yml).
+If there are multiple comments that match, the first one is effective. If the info is missing for
+a binding, the default repo (`mmtk/mmtk-X`) and branch (`master`) will be used instead.
+```
+binding-refs
+OPENJDK_BINDING_REPO=xx/xx
+OPENJDK_BINDING_REF=xxxxxx
+JIKESRVM_BINDING_REPO=xx/xx
+JIKESRVM_BINDING_REF=xxxxxx
+V8_BINDING_REPO=xx/xx
+V8_BINDING_REF=xxxxxx
+JULIA_BINDING_REPO=xx/xx
+JULIA_BINDING_REF=xxxxxx
+RUBY_BINDING_REPO=xx/xx
+RUBY_BINDING_REF=xxxxxx
+```
+
+### Merging a PR with API breaking changes
+
+If an MMTk core PR includes API breaking changes, the corresponding binding PRs depends on an mmtk-core commit in the PR branch. As we
+use squashing merging, the commit in the PR branch will disappear once the mmtk-core PR is merged. When the mmtk-core PR is merged,
+we will have a new commit in `mmtk-core` master. We will need to fix the mmtk dependency in the binding PRs to point to the new commit,
+and then merge the binding PRs.
+
+#### Auto merging process
+
+This process should be done automatically by [`auto-merge.yml`](https://github.com/mmtk/mmtk-core/blob/master/.github/workflows/auto-merge.yml)
+when an mmtk-core PR is merged and the `binding-refs` comment is present.
+
+1. Make sure there is no other PR in this merging process. If so, resolve those first.
+1. Make sure all the PRs (the mmtk-core PR, the binding PRs, and the associated PRs in the VM repo if any) are ready to merge.
+1. For each binding PR that we need to merge:
+   1. If the binding PR has an assocate PR in the VM repo, merge the VM PR first. Once it is merged, we will have a commit hash (we refer to it as `{vm_commit}`).
+   1. Update `mmtk/Cargo.toml` in the binding:
+      * Find the section `[package.metadata.{binding-name}]`.
+      * Update the field `{binding-name}_repo` if necessary. It should point to our VM fork, such as `https://github.com/mmtk/{binding-name}.git`.
+      * Update the field `{binding-name}_version`. It should point to the new commit hash `{vm_commit}`.
+      * Commit the change.
+1. Merge the mmtk-core PR.
+1. When a new commit is pushed to `master`, `auto-merge.yml` will be triggered.
+1. The binding PRs should be updated and auto merge will be eanbled for the PR. Keep an eye until the PRs are all merged. Resolve any
+   issue that prevents the PR from being auto merged (e.g. flaky tests).
+
+#### Manual merging process
+
+If `auto-merge.yml` failed for any reason, or if we have to manually merge binding PRs, this is the process to follow:
+
+1. Follow Step 1-4 in the auto merging process.
+1. When a new commit is pushed to `master`, we record the commit hash (as `{mmtk_core_commit}`).
+1. For each binding PR that we need to merge:
+   1. Update `mmtk/Cargo.toml` in the binding:
+      * Find the `mmtk` dependency under `[dependencies]`.
+      * Update the field `git` if necessary. It should point to our mmtk-core repo, `https://github.com/mmtk/mmtk-core.git`.
+      * Update the field `rev`. It should point to the new mmtk-core commit hash `{mmtk_core_commit}`.
+      * Update `mmtk/Cargo.lock` by building the Rust project again. If the binding needs to choose a GC plan by feature, use
+        any supported plan. So this step is slightly different for different bindings:
+        * OpenJDK, Ruby: `cargo build`
+        * JikesRVM: `cargo build --features nogc --target i686-unknown-linux-gnu`
+        * V8: `cargo build --features nogc`
+        * Julia: `cargo build --features immix`
+      * Check in both `mmtk/Cargo.toml` and `mmtk/Cargo.lock`, and commit.
+    2. Merge the PR once it can be merged.