From 4971634e6ae9472cb86ad77db3ac753cae84ec8e Mon Sep 17 00:00:00 2001
From: Karl Cardenas <29551334+karl-cardenas-coding@users.noreply.github.com>
Date: Thu, 17 Oct 2024 14:36:34 -0700
Subject: [PATCH 1/2] docs: Version Branches Robots.txt Update (#4324)

* chore: sligh bash modification

* chore: Apply suggestions from code review

Co-authored-by: Lenny Chen <55669665+lennessyy@users.noreply.github.com>

---------

Co-authored-by: Lenny Chen <55669665+lennessyy@users.noreply.github.com>
(cherry picked from commit 79277290022a3a30d9d4a529dd0ada2783651ab2)
---
 .github/workflows/versions_robot.yaml | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/versions_robot.yaml b/.github/workflows/versions_robot.yaml
index a73a550cbb..fb69024781 100644
--- a/.github/workflows/versions_robot.yaml
+++ b/.github/workflows/versions_robot.yaml
@@ -66,13 +66,21 @@ jobs:
       - name: Check if robots.txt exists and create if missing
         shell: bash
         run: |
-          if [ ! -f "static/robots.txt" ]; then
-            echo "Does not Exist"
-            echo "User-agent: *" > static/robots.txt
-            echo "Disallow: /" >> static/robots.txt
-          else
-            echo "Robots.txt file exists"
-          fi
+            if [ ! -f "static/robots.txt" ]; then
+              echo "Does not Exist. Creating file with expected content..."
+              echo "User-agent: *" > static/robots.txt
+              echo "Disallow: /" >> static/robots.txt
+            else
+              echo "Robots.txt file exists"
+              if ! grep -Fxq "User-agent: *" static/robots.txt || ! grep -Fxq "Disallow: /" static/robots.txt; then
+                echo "Robots.txt file does not have the expected content. Updating..."
+                echo "User-agent: *" > static/robots.txt
+                echo "Disallow: /" >> static/robots.txt
+              else
+                echo "Robots.txt file has the expected content."
+              fi
+            fi
+            
       - name: Ensure noIndex is set
         id: check_noindex
         run: node scripts/noindex_docusaurus_config.js $RUNNER_TEMP $PWD && mv $RUNNER_TEMP/temp.docusaurus.config.js $PWD/docusaurus.config.js

From 31853841e740f53a625b8a7232b47b9989cd5d9d Mon Sep 17 00:00:00 2001
From: "vault-token-factory-spectrocloud[bot]"
 <vault-token-factory-spectrocloud[bot]@users.noreply.github.com>
Date: Thu, 17 Oct 2024 21:40:22 +0000
Subject: [PATCH 2/2] ci: adding missing robots.txt or updating noindex in
 docusaurus.config.js

---
 static/robots.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/static/robots.txt b/static/robots.txt
index e69de29bb2..1f53798bb4 100644
--- a/static/robots.txt
+++ b/static/robots.txt
@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /