Exclude searchbots in robots.txt

1200wd · Nov 26, 2024 · 8681491 · 8681491
1 parent 79ce611
commit 8681491
Showing 1 changed file with 10 additions and 0 deletions.
diff --git a/blocksmurfer/static/robots.txt b/blocksmurfer/static/robots.txt
@@ -1,6 +1,16 @@
 User-agent: SemrushBot
 User-agent: PetalBot
 User-agent: mj12bot
+User-agent: blowfish
+User-agent: CatchBot
+User-agent: BecomeBot
+User-agent: Bytespider
+User-agent: spider
+User-agent: Amazonbot
+User-agent: bytedance
+User-agent: claudebot
+User-agent: bing
+User-agent: yandex
 Disallow: /
 
 User-agent: *