docs: add robots.txt

diff --git a/docs/robots.txt b/docs/robots.txt
new file mode 100644 (file)
index 0000000..864d53b
--- /dev/null
+++ b/docs/robots.txt
@@ -0,0 +1,64 @@
+# robots.txt for bitcoindevkit.org
+# See RFC 9309: https://www.rfc-editor.org/rfc/rfc9309
+
+# Content Signals - Declare AI content usage preferences
+# See https://contentsignals.org/
+Content-Signal: ai-train=yes, search=yes, ai-input=yes
+
+# AI Crawlers - Allow training and search
+User-agent: GPTBot
+Allow: /
+
+User-agent: ChatGPT-User
+Allow: /
+
+User-agent: OAI-SearchBot
+Allow: /
+
+User-agent: Claude-Web
+Allow: /
+
+User-agent: ClaudeBot
+Allow: /
+
+User-agent: Google-Extended
+Allow: /
+
+User-agent: PerplexityBot
+Allow: /
+
+User-agent: Amazonbot
+Allow: /
+
+User-agent: anthropic-ai
+Allow: /
+
+User-agent: Applebot-Extended
+Allow: /
+
+User-agent: CCBot
+Allow: /
+
+User-agent: FacebookBot
+Allow: /
+
+User-agent: cohere-ai
+Allow: /
+
+# Allow all other compliant crawlers to access all content
+User-agent: *
+Allow: /
+
+# Paths to crawl
+Allow: /foundation/
+Allow: /adoption/
+Allow: /blog/
+Allow: /assets/
+
+# Disallow common system files and directories
+Disallow: /.git/
+Disallow: /.github/
+Disallow: /404.html
+
+# Sitemap location
+Sitemap: https://bitcoindevkit.org/sitemap.xml