Fixed robots.txt rules to be on separate lines

Each rule in robots.txt should be a separate line based on https://developers.google.com/search/docs/crawling-indexing/robots/create-robots-txt
rawsta · Jun 8, 2024 · 0041971 · 0041971
1 parent ed9b306
commit 0041971
Showing 1 changed file with 16 additions and 2 deletions.
diff --git a/src/common/robots.njk b/src/common/robots.njk
@@ -3,5 +3,19 @@ permalink: /robots.txt
 eleventyExcludeFromCollections: true
 excludeFromSitemap: true
 ---
-User-agent: * Disallow: /404.html User-agent: GPTbot Disallow: / User-agent: ChatGPT-User Disallow: /
-User-agent: Google-Extended Disallow: / User-agent: Omgilibot Disallow: / Sitemap: {{ meta.url }}/sitemap.xml
+User-agent: * 
+Disallow: /404.html 
+
+User-agent: GPTbot 
+Disallow: / 
+
+User-agent: ChatGPT-User 
+Disallow: /
+
+User-agent: Google-Extended 
+Disallow: / 
+
+User-agent: Omgilibot 
+Disallow: / 
+
+Sitemap: {{ meta.url }}/sitemap.xml