From 0c4a0cb824a6fdca40c3bf6bb1cb5dd455829ef3 Mon Sep 17 00:00:00 2001
From: "vault-token-factory-spectrocloud[bot]"
 <133815545+vault-token-factory-spectrocloud[bot]@users.noreply.github.com>
Date: Thu, 17 Oct 2024 15:42:51 -0700
Subject: [PATCH] [version-4-4] docs: Version Branches Robots.txt Update
 (#4324) (#4330)

* docs: Version Branches Robots.txt Update (#4324)

* chore: sligh bash modification

* chore: Apply suggestions from code review

Co-authored-by: Lenny Chen <55669665+lennessyy@users.noreply.github.com>

---------

Co-authored-by: Lenny Chen <55669665+lennessyy@users.noreply.github.com>
(cherry picked from commit 79277290022a3a30d9d4a529dd0ada2783651ab2)

* ci: adding missing robots.txt or updating noindex in docusaurus.config.js

---------

Co-authored-by: Karl Cardenas <29551334+karl-cardenas-coding@users.noreply.github.com>
Co-authored-by: vault-token-factory-spectrocloud[bot] <vault-token-factory-spectrocloud[bot]@users.noreply.github.com>
---
 .github/workflows/versions_robot.yaml | 22 +++++++++++++++-------
 static/robots.txt                     |  2 ++
 2 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/versions_robot.yaml b/.github/workflows/versions_robot.yaml
index a73a550cbb..fb69024781 100644
--- a/.github/workflows/versions_robot.yaml
+++ b/.github/workflows/versions_robot.yaml
@@ -66,13 +66,21 @@ jobs:
       - name: Check if robots.txt exists and create if missing
         shell: bash
         run: |
-          if [ ! -f "static/robots.txt" ]; then
-            echo "Does not Exist"
-            echo "User-agent: *" > static/robots.txt
-            echo "Disallow: /" >> static/robots.txt
-          else
-            echo "Robots.txt file exists"
-          fi
+            if [ ! -f "static/robots.txt" ]; then
+              echo "Does not Exist. Creating file with expected content..."
+              echo "User-agent: *" > static/robots.txt
+              echo "Disallow: /" >> static/robots.txt
+            else
+              echo "Robots.txt file exists"
+              if ! grep -Fxq "User-agent: *" static/robots.txt || ! grep -Fxq "Disallow: /" static/robots.txt; then
+                echo "Robots.txt file does not have the expected content. Updating..."
+                echo "User-agent: *" > static/robots.txt
+                echo "Disallow: /" >> static/robots.txt
+              else
+                echo "Robots.txt file has the expected content."
+              fi
+            fi
+            
       - name: Ensure noIndex is set
         id: check_noindex
         run: node scripts/noindex_docusaurus_config.js $RUNNER_TEMP $PWD && mv $RUNNER_TEMP/temp.docusaurus.config.js $PWD/docusaurus.config.js
diff --git a/static/robots.txt b/static/robots.txt
index e69de29bb2..1f53798bb4 100644
--- a/static/robots.txt
+++ b/static/robots.txt
@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /