Merge pull request #10397 from internetarchive/robots-txt-crawl-delays

Add crawl delays to robots.txt

Author: mekarpeles

static/robots.txt

@@ -2,13 +2,44 @@
 User-agent: *
 Disallow: /api
 Disallow: /edit
+Disallow: /account
 Disallow: /search
 Disallow: /search/inside
+Disallow: */borrow*
+Disallow: /*.rdf$
 
 Sitemap: https://openlibrary.org/static/sitemaps/siteindex.xml.gz
 
 User-agent: Baiduspider
-Crawl-delay: 0.5
+Crawl-delay: 10
 
 User-agent: Googlebot
 Disallow: /*.rdf$
+Crawl-delay: 10
+
+User-agent: AhrefsBot
+Crawl-delay: 10
+
+User-agent: meta-externalagent
+Crawl-delay: 10
+
+User-agent: anthropic-ai
+Crawl-delay: 10
+
+User-agent: ClaudeBot
+Crawl-delay: 10
+
+User-agent: openai
+Crawl-delay: 10
+
+User-agent: ChatGPT
+Crawl-delay: 10
+
+User-agent: bingbot
+Crawl-delay: 10
+
+User-agent: Yandex
+Crawl-delay: 10
+
+User-agent: *bot
+Crawl-delay: 10