fix: update based on feedback to separate into different jobs

.github/workflows/pr.yml

@@ -16,14 +16,21 @@ on:
       # - ingestion-data/staging/dataset-config/*
       - ingestion-data/testing/dataset-config/*
 
+  push:
+    branches:
+      - main
+
+permissions:
+  pull-requests: write
+  contents: read
+
 jobs:
   dataset-publication-and-configuration:
-    permissions:
-      pull-requests: write
-      contents: read
+    if: ${{ github.event_name == 'pull_request' && (github.event.action == 'synchronize' || github.event.action == 'opened') }}
     runs-on: ubuntu-latest
     environment: staging
-
+    outputs:
+      publishedCollections: ${{ steps.publish-collections.outputs.success_collections }}
     steps:
       - uses: actions/checkout@v4
 
@@ -98,7 +105,7 @@ jobs:
       # Used by other steps
       # If none of the requests are successful, workflow fails
       # Updates the PR comment with status of collection publication
-      - name: Publish all newly added collections
+      - name: Publish all newly added collections to staging
         id: publish-collections
         env:
           ADDED_FILES: ${{ steps.changed-files.outputs.added_files }}
@@ -180,17 +187,6 @@ jobs:
           python-version: '3.9'
           cache: 'pip'
 
-      # Creates a slim dataset mdx file for each collection based on the dataset config json
-      - name: Create dataset mdx for given collections
-        env:
-          PUBLISHED_COLLECTION_FILES: ${{ steps.publish-collections.outputs.success_collections }}
-        run: |
-          pip install -r scripts/requirements.txt
-          for file in "${PUBLISHED_COLLECTION_FILES[@]}"
-          do
-            python3 scripts/mdx.py "$file"
-          done
-
       # If the workflow fails at any point, the PR comment will be updated
       - name: Update PR comment on overall workflow failure
         if: failure()
@@ -204,3 +200,34 @@ jobs:
 
           ** ❌ The workflow run failed. [See logs here]($WORKFLOW_URL)**"
           gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
+
+  create-mdx-files:
+    runs-on: ubuntu-latest
+    needs: dataset-publication-and-configuration
+    steps:
+      - name: Use output from dataset-publication-and-configuration
+
+        run: |
+          echo "The output from the previous step is: ${{ needs.dataset-publication-and-configuration.outputs.publishedCollections }}"
+
+      # Creates a slim dataset mdx file for each collection based on the dataset config json
+      - name: Create dataset mdx for given collections
+        env:
+          PUBLISHED_COLLECTION_FILES: ${{ needs.dataset-publication-and-configuration.outputs.publishedCollections }}
+        run: echo "NO-OP step"
+        # run: |
+        #   pip install -r scripts/requirements.txt
+        #   for file in "${PUBLISHED_COLLECTION_FILES[@]}"
+        #   do
+        #     python3 scripts/mdx.py "$file"
+        #   done
+
+  publish-to-prod-on-pr-merge:
+    if: ${{ github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Publish to production on PR merge
+        run: echo "NO-OP. This step runs when a PR is merged."