diff --git a/.github/workflows/versions_robot.yaml b/.github/workflows/versions_robot.yaml new file mode 100644 index 0000000000..a73a550cbb --- /dev/null +++ b/.github/workflows/versions_robot.yaml @@ -0,0 +1,105 @@ +name: Versions Robot.txt Check + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + branches: ["version-*", "backport/version-*"] + + +concurrency: + group: versions-robot-${{ github.ref }} + cancel-in-progress: true + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_S3_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_S3_SECRET_KEY }} + AWS_DEFAULT_REGION: us-east-1 + APPZI_TOKEN: ${{ secrets.APPZI_TOKEN }} + FULLSTORY_ORGID: ${{ secrets.FULLSTORY_ORGID }} + ALGOLIA_ADMIN_KEY: ${{ secrets.ALGOLIA_ADMIN_KEY }} + ALGOLIA_APP_ID: ${{ secrets.ALGOLIA_APP_ID }} + ALGOLIA_SEARCH_KEY: ${{ secrets.ALGOLIA_SEARCH_KEY }} + ALGOLIA_INDEX_NAME: ${{ secrets.ALGOLIA_INDEX_NAME }} + PALETTE_API_KEY: ${{ secrets.PALETTE_API_KEY }} + GITHUB_BRANCH: ${{ github.ref_name }} + +jobs: + run-ci: + # runs-on: ubuntu-latest + runs-on: ubuntu-latest + defaults: + run: + shell: bash + if: ${{ !github.event.pull_request.draft && github.actor != 'dependabot[bot]' && github.actor != 'dependabot-preview[bot]' }} + steps: + # If the condition above is not met, aka, the PR is not in draft status, then this step is skipped. + # Because this step is part of the critical path, omission of this step will result in remaining CI steps not gettinge executed. + # As of 8/8/2022 there is now way to enforce this beahvior in GitHub Actions CI. + - run: exit 0 + + robots-txt: + name: Robots.txt Check + needs: [run-ci] + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Setup Node.js environment + uses: actions/setup-node@v4 + with: + node-version: "20" + cache: "npm" + + - run: npm ci + + - name: Determine branch name + id: extract_branch + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + echo "GITHUB_BRANCH=${{ github.head_ref }}" >> $GITHUB_ENV + else + echo "GITHUB_BRANCH=${{ github.ref_name }}" >> $GITHUB_ENV + fi + + - name: Check if robots.txt exists and create if missing + shell: bash + run: | + if [ ! -f "static/robots.txt" ]; then + echo "Does not Exist" + echo "User-agent: *" > static/robots.txt + echo "Disallow: /" >> static/robots.txt + else + echo "Robots.txt file exists" + fi + - name: Ensure noIndex is set + id: check_noindex + run: node scripts/noindex_docusaurus_config.js $RUNNER_TEMP $PWD && mv $RUNNER_TEMP/temp.docusaurus.config.js $PWD/docusaurus.config.js + + - name: Commit Changes + id: commit + uses: stefanzweifel/git-auto-commit-action@v5 + with: + file_pattern: '*.js *.txt **.txt **.js **/*.txt **/*.js' + disable_globbing: true + commit_message: "ci: adding missing robots.txt or updating noindex in docusaurus.config.js" + + - name: Slack Notification + if: ${{ failure() }} + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_PRIVATE_TEAM_WEBHOOK }} + SLACK_USERNAME: "spectromate" + SLACK_ICON_EMOJI: ":robot_panic:" + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: ' The PR for version branch ${{env.GITHUB_BRANCH}} failed when attempting add a robot.txt file or update the docusaurus.config.js. Review the GitHub Actions logs for more details.' + + - name: Post Netlify progress + if: ${{ steps.commit.conclusion == 'success' && steps.commit.outputs.changes_detected == 'true' }} + uses: mshick/add-pr-comment@v2 + with: + message: | + 🤖 A robot.txt file was auto-generated or the docusaurus.config.js was updated and commited to the branch. All version braches require a robot.txt file. + refresh-message-position: false + \ No newline at end of file diff --git a/scripts/noindex_docusaurus_config.js b/scripts/noindex_docusaurus_config.js new file mode 100644 index 0000000000..7a24d5112c --- /dev/null +++ b/scripts/noindex_docusaurus_config.js @@ -0,0 +1,67 @@ +const fs = require("fs"); +const parser = require("@babel/parser"); +const generate = require("@babel/generator").default; +const t = require("@babel/types"); + +const docusaurusConfigFile = "docusaurus.config.js"; +const tempDirectory = process.argv[2]; +const baseDirectory = process.argv[3]; + +// This reads the docusaurus.config.js file and parses it into an AST. +// We need to parse it into an AST so that we can add the new versions to the config object. +// We then add the noIndex property to the config object. +// This is only for version branches +const configCode = fs.readFileSync(`${baseDirectory}/${docusaurusConfigFile}`, "utf8"); +const ast = parser.parse(configCode, { + sourceType: "module", +}); + +// Function to add noIndex: true after trailingSlash if not already present +const addNoIndexProperty = () => { + // Find the main "config" declaration + const configDeclaration = ast.program.body.find( + (node) => node.type === "VariableDeclaration" && node.declarations[0].id.name === "config" + ); + + // If the "config" declaration is not found, log an error and return + if (!configDeclaration) { + console.error('Could not locate the main "config" declaration.'); + return; + } + + // Check if the "noIndex" property already exists + const noIndexExists = configDeclaration.declarations[0].init.properties.some((prop) => prop.key.name === "noIndex"); + + if (noIndexExists) { + console.log('"noIndex" property already exists in the config.'); + return; // Do nothing if noIndex already exists + } + + // Find the "trailingSlash" property in the "config" declaration + const trailingSlashProperty = configDeclaration.declarations[0].init.properties.find( + (prop) => prop.key.name === "trailingSlash" + ); + + // If "trailingSlash" is found, insert "noIndex: true" after it + if (trailingSlashProperty) { + const noIndexProperty = t.objectProperty(t.identifier("noIndex"), t.booleanLiteral(true)); + + // Find the index of the trailingSlash property and insert noIndex after it + const index = configDeclaration.declarations[0].init.properties.indexOf(trailingSlashProperty); + configDeclaration.declarations[0].init.properties.splice(index + 1, 0, noIndexProperty); + console.log('"noIndex" property added to the config.'); + } else { + console.error('Could not locate the "trailingSlash" property.'); + } +}; + +addNoIndexProperty(); + +// This is where the new config object is converted back into code. +const updatedCode = generate(ast).code; +try { + // Lastly, this is where the new config object is written to the temp.docusaurus.config.js file. + fs.writeFileSync(`${tempDirectory}/temp.docusaurus.config.js`, updatedCode); +} catch (err) { + console.error("Could not write to temp.docusaurus.config.js:", err); +}