chore: update robots.txt disallow list (#1131)

* chore: update robots.txt disallow list
OpenBeta · May 13, 2024 · 0ac0535 · 0ac0535
1 parent 01663db
commit 0ac0535
Showing 1 changed file with 6 additions and 1 deletion.
diff --git a/public/robots.txt b/public/robots.txt
@@ -1,5 +1,10 @@
 User-agent: PetalBot
-Disallow: /
+User-agent: SemrushBot
+User-agent: AhrefsBot
+User-agent: Dotbot
+User-agent: MJ12bot
+User-agent: rogerbot
+Disallow: /
 
 User-agent: *
 Disallow: /account/