diff --git a/.github/workflows/port-issue.yaml b/.github/workflows/port-issue.yaml
index 31ef97cf..9efc7561 100644
--- a/.github/workflows/port-issue.yaml
+++ b/.github/workflows/port-issue.yaml
@@ -10,7 +10,7 @@ on:
 jobs:
   port-issue:
     runs-on: ubuntu-latest
-    if: contains(github.event.comment.body, '/backport') || contains(github.event.comment.body, '/forwardport') && !github.event.issue.pull_request
+    if: ${{ !github.event.issue.pull_request && (contains(github.event.comment.body, '/backport') || contains(github.event.comment.body, '/forwardport')) }}
     steps:
       - name: Check org membership
         env:
diff --git a/.github/workflows/port-pr.yaml b/.github/workflows/port-pr.yaml
index b6365102..1a9123bb 100644
--- a/.github/workflows/port-pr.yaml
+++ b/.github/workflows/port-pr.yaml
@@ -10,7 +10,7 @@ on:
 jobs:
   port-pr:
     runs-on: ubuntu-latest
-    if: (startsWith(github.event.comment.body, '/backport') || startsWith(github.event.comment.body, '/forwardport')) && github.event.issue.pull_request
+    if: ${{ github.event.issue.pull_request && (startsWith(github.event.comment.body, '/backport') || startsWith(github.event.comment.body, '/forwardport')) }}
     steps:
       - name: Check org membership
         env:
@@ -34,7 +34,7 @@ jobs:
           # Sanitize input
           MILESTONE=${BODY_MILESTONE//[^a-zA-Z0-9\-\.]/}
           if gh api repos/${GITHUB_REPOSITORY}/milestones --paginate | jq -e --arg MILESTONE "$MILESTONE" '.[] | select(.title == $MILESTONE)' > /dev/null; then
-              echo "Milestone exists"
+              echo "Milestone ${MILESTONE} exists" >> $GITHUB_STEP_SUMMARY
               echo "milestone_exists=true" >> $GITHUB_ENV
               echo "milestone=${MILESTONE}" >> $GITHUB_ENV
            else
@@ -87,6 +87,7 @@ jobs:
               ERROR_MESSAGE=$(cat error.log)
               FORMATTED_ERROR_MESSAGE=$(printf "\n\`\`\`\n%s\n\`\`\`" "$ERROR_MESSAGE")
               gh issue comment ${ORIGINAL_ISSUE_NUMBER} --body "Not creating port PR, there was an error running git am -3: $FORMATTED_ERROR_MESSAGE"
+              echo "Port PR not created." >> $GITHUB_STEP_SUMMARY
           else
               git push origin $BRANCH
               ORIGINAL_PR=$(gh pr view ${ORIGINAL_ISSUE_NUMBER} --json title,body,assignees)