diff --git a/.github/workflows/docs-issue.yml b/.github/workflows/docs-issue.yml
index 5ec176cdd65..00a098df827 100644
--- a/.github/workflows/docs-issue.yml
+++ b/.github/workflows/docs-issue.yml
@@ -32,8 +32,9 @@ jobs:
     # generating the comment before the other runs.  This lives here instead of the shared workflow because this is where we
     # decide if it should run or not.
     if: |
-      (contains( github.event.pull_request.labels.*.name, 'user docs') && github.event.pull_request.merged == true) ||
-      (github.event.label.name == 'user docs' && github.event.action == 'labeled' )
+      (github.event.pull_request.merged == true) &&
+      ((github.event.action == 'closed' && contains( github.event.pull_request.labels.*.name, 'user docs')) ||
+      (github.event.action == 'labeled' && github.event.label.name == 'user docs'))
     uses: dbt-labs/actions/.github/workflows/open-issue-in-repo.yml@main
     with:
         issue_repository: "dbt-labs/docs.getdbt.com"