docs: DOC-1426 Version Branches Automation Improvements (#4314)

* ci: create CI YAML

* chore: update config updater

* docs: DOC-1426

* chore: save

* chore: fix path

* ci: ready for test

* chore: fix CI name

* ci: version logic+

(cherry picked from commit 6c05221)

.github/workflows/versions_robot.yaml

@@ -0,0 +1,105 @@
+name: Versions Robot.txt Check
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+    branches: ["version-*", "backport/version-*"]
+
+
+concurrency:
+  group: versions-robot-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_S3_ACCESS_KEY }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_S3_SECRET_KEY }}
+  AWS_DEFAULT_REGION: us-east-1
+  APPZI_TOKEN: ${{ secrets.APPZI_TOKEN }}
+  FULLSTORY_ORGID: ${{ secrets.FULLSTORY_ORGID }}
+  ALGOLIA_ADMIN_KEY: ${{ secrets.ALGOLIA_ADMIN_KEY }}
+  ALGOLIA_APP_ID: ${{ secrets.ALGOLIA_APP_ID }}
+  ALGOLIA_SEARCH_KEY: ${{ secrets.ALGOLIA_SEARCH_KEY }}
+  ALGOLIA_INDEX_NAME: ${{ secrets.ALGOLIA_INDEX_NAME }}
+  PALETTE_API_KEY: ${{ secrets.PALETTE_API_KEY }}
+  GITHUB_BRANCH: ${{ github.ref_name }} 
+
+jobs:
+  run-ci:
+    # runs-on: ubuntu-latest
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    if: ${{ !github.event.pull_request.draft && github.actor != 'dependabot[bot]' && github.actor != 'dependabot-preview[bot]' }}
+    steps:
+      # If the condition above is not met, aka, the PR is not in draft status, then this step is skipped.
+      # Because this step is part of the critical path, omission of this step will result in remaining CI steps not gettinge executed.
+      # As of 8/8/2022 there is now way to enforce this beahvior in GitHub Actions CI.
+      - run: exit 0
+
+  robots-txt:
+    name: Robots.txt Check
+    needs: [run-ci]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js environment
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: "npm"
+
+      - run: npm ci
+
+      - name: Determine branch name
+        id: extract_branch
+        run: |
+          if [ "${{ github.event_name }}" = "pull_request" ]; then
+            echo "GITHUB_BRANCH=${{ github.head_ref }}" >> $GITHUB_ENV
+          else
+            echo "GITHUB_BRANCH=${{ github.ref_name }}" >> $GITHUB_ENV
+          fi
+
+      - name: Check if robots.txt exists and create if missing
+        shell: bash
+        run: |
+          if [ ! -f "static/robots.txt" ]; then
+            echo "Does not Exist"
+            echo "User-agent: *" > static/robots.txt
+            echo "Disallow: /" >> static/robots.txt
+          else
+            echo "Robots.txt file exists"
+          fi
+      - name: Ensure noIndex is set
+        id: check_noindex
+        run: node scripts/noindex_docusaurus_config.js $RUNNER_TEMP $PWD && mv $RUNNER_TEMP/temp.docusaurus.config.js $PWD/docusaurus.config.js
+
+      - name: Commit Changes
+        id: commit
+        uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          file_pattern: '*.js *.txt **.txt **.js **/*.txt **/*.js'
+          disable_globbing: true
+          commit_message: "ci: adding missing robots.txt or updating noindex in docusaurus.config.js"
+
+      - name: Slack Notification
+        if: ${{ failure() }}
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_PRIVATE_TEAM_WEBHOOK }}
+          SLACK_USERNAME: "spectromate"
+          SLACK_ICON_EMOJI: ":robot_panic:"
+          SLACK_COLOR: ${{ job.status }}
+          SLACK_MESSAGE: ' The PR for version branch ${{env.GITHUB_BRANCH}} failed when attempting add a robot.txt file or update the docusaurus.config.js. Review the GitHub Actions logs for more details.'
+
+      - name: Post Netlify progress
+        if: ${{ steps.commit.conclusion == 'success' && steps.commit.outputs.changes_detected == 'true' }}
+        uses: mshick/add-pr-comment@v2
+        with:
+          message: |
+              🤖 A robot.txt file was auto-generated or the docusaurus.config.js was updated and commited to the branch. All version braches require a robot.txt file.
+          refresh-message-position: false
+

scripts/noindex_docusaurus_config.js

@@ -0,0 +1,67 @@
+const fs = require("fs");
+const parser = require("@babel/parser");
+const generate = require("@babel/generator").default;
+const t = require("@babel/types");
+
+const docusaurusConfigFile = "docusaurus.config.js";
+const tempDirectory = process.argv[2];
+const baseDirectory = process.argv[3];
+
+// This reads the docusaurus.config.js file and parses it into an AST.
+// We need to parse it into an AST so that we can add the new versions to the config object.
+// We then add the noIndex property to the config object.
+// This is only for version branches
+const configCode = fs.readFileSync(`${baseDirectory}/${docusaurusConfigFile}`, "utf8");
+const ast = parser.parse(configCode, {
+  sourceType: "module",
+});
+
+// Function to add noIndex: true after trailingSlash if not already present
+const addNoIndexProperty = () => {
+  // Find the main "config" declaration
+  const configDeclaration = ast.program.body.find(
+    (node) => node.type === "VariableDeclaration" && node.declarations[0].id.name === "config"
+  );
+
+  // If the "config" declaration is not found, log an error and return
+  if (!configDeclaration) {
+    console.error('Could not locate the main "config" declaration.');
+    return;
+  }
+
+  // Check if the "noIndex" property already exists
+  const noIndexExists = configDeclaration.declarations[0].init.properties.some((prop) => prop.key.name === "noIndex");
+
+  if (noIndexExists) {
+    console.log('"noIndex" property already exists in the config.');
+    return; // Do nothing if noIndex already exists
+  }
+
+  // Find the "trailingSlash" property in the "config" declaration
+  const trailingSlashProperty = configDeclaration.declarations[0].init.properties.find(
+    (prop) => prop.key.name === "trailingSlash"
+  );
+
+  // If "trailingSlash" is found, insert "noIndex: true" after it
+  if (trailingSlashProperty) {
+    const noIndexProperty = t.objectProperty(t.identifier("noIndex"), t.booleanLiteral(true));
+
+    // Find the index of the trailingSlash property and insert noIndex after it
+    const index = configDeclaration.declarations[0].init.properties.indexOf(trailingSlashProperty);
+    configDeclaration.declarations[0].init.properties.splice(index + 1, 0, noIndexProperty);
+    console.log('"noIndex" property added to the config.');
+  } else {
+    console.error('Could not locate the "trailingSlash" property.');
+  }
+};
+
+addNoIndexProperty();
+
+// This is where the new config object is converted back into code.
+const updatedCode = generate(ast).code;
+try {
+  // Lastly, this is where the new config object is written to the temp.docusaurus.config.js file.
+  fs.writeFileSync(`${tempDirectory}/temp.docusaurus.config.js`, updatedCode);
+} catch (err) {
+  console.error("Could not write to temp.docusaurus.config.js:", err);
+}