disallow common ai crawlers, thanks https://n3wjack.net/2024/07/07/bl…

…ocking-ai-crawlers-with-robots-txt/
lue-bird · Jul 7, 2024 · 3ee36fd · 3ee36fd
1 parent b1b18ce
commit 3ee36fd
Showing 1 changed file with 30 additions and 0 deletions.
diff --git a/robots.txt b/robots.txt
@@ -1,5 +1,35 @@
+User-agent: Amazonbot
+Disallow: /
+User-agent: anthropic-ai
+Disallow: /
+User-agent: Bytespider
+Disallow: /
+User-agent: CCBot
+Disallow: /
+User-agent: ChatGPT-User
+Disallow: /
+User-agent: ClaudeBot
+Disallow: /
+User-agent: Claude-Web
+Disallow: /
+User-agent: cohere-ai
+Disallow: /
+User-agent: Diffbot
+Disallow: /
+User-agent: FacebookBot
+Disallow: /
+User-agent: Google-Extended
+Disallow: /
 User-agent: GPTBot
 Disallow: /
+User-agent: magpie-crawler
+Disallow: /
+User-agent: omgili
+Disallow: /
+User-agent: omgilibot
+Disallow: /
+User-agent: PerplexityBot
+Disallow: /
 
 User-agent: *
 Allow: /