fix: Update robots.txt

I want (some) search engines to crawl this.
evancharlton · Jan 7, 2025 · 1ce2643 · 1ce2643
1 parent bdac066
commit 1ce2643
Showing 1 changed file with 34 additions and 3 deletions.
diff --git a/public/robots.txt b/public/robots.txt
@@ -1,3 +1,34 @@
-# https://www.robotstxt.org/robotstxt.html
-User-agent: *
-Disallow:
+User-agent: GPTBot
+Disallow: /
+User-agent: ChatGPT-User
+Disallow: /
+User-agent: Google-Extended
+Disallow: /
+User-agent: PerplexityBot
+Disallow: /
+User-agent: Amazonbot
+Disallow: /
+User-agent: ClaudeBot
+Disallow: /
+User-agent: Omgilibot
+Disallow: /
+User-Agent: FacebookBot
+Disallow: /
+User-Agent: Applebot
+Disallow: /
+User-agent: anthropic-ai
+Disallow: /
+User-agent: Bytespider
+Disallow: /
+User-agent: Claude-Web
+Disallow: /
+User-agent: Diffbot
+Disallow: /
+User-agent: ImagesiftBot
+Disallow: /
+User-agent: Omgilibot
+Disallow: /
+User-agent: Omgili
+Disallow: /
+User-agent: YouBot
+Disallow: /