From: Steve Myers <steve@notmandatory.org>
Date: Wed, 20 May 2026 21:37:06 +0000 (-0500)
Subject: docs: add robots.txt
X-Git-Url: http://internal-gitweb-vhost/%22bdk_chain/io/-invalid-non-witness-utxo/bitcoin/ElectrumClientInterface.html?a=commitdiff_plain;h=d9bfaacc72937b74dc963bc88cd38db009b5dcee;p=bitcoindevkit.org

docs: add robots.txt
---

diff --git a/docs/robots.txt b/docs/robots.txt
new file mode 100644
index 0000000000..864d53b1f3
--- /dev/null
+++ b/docs/robots.txt
@@ -0,0 +1,64 @@
+# robots.txt for bitcoindevkit.org
+# See RFC 9309: https://www.rfc-editor.org/rfc/rfc9309
+
+# Content Signals - Declare AI content usage preferences
+# See https://contentsignals.org/
+Content-Signal: ai-train=yes, search=yes, ai-input=yes
+
+# AI Crawlers - Allow training and search
+User-agent: GPTBot
+Allow: /
+
+User-agent: ChatGPT-User
+Allow: /
+
+User-agent: OAI-SearchBot
+Allow: /
+
+User-agent: Claude-Web
+Allow: /
+
+User-agent: ClaudeBot
+Allow: /
+
+User-agent: Google-Extended
+Allow: /
+
+User-agent: PerplexityBot
+Allow: /
+
+User-agent: Amazonbot
+Allow: /
+
+User-agent: anthropic-ai
+Allow: /
+
+User-agent: Applebot-Extended
+Allow: /
+
+User-agent: CCBot
+Allow: /
+
+User-agent: FacebookBot
+Allow: /
+
+User-agent: cohere-ai
+Allow: /
+
+# Allow all other compliant crawlers to access all content
+User-agent: *
+Allow: /
+
+# Paths to crawl
+Allow: /foundation/
+Allow: /adoption/
+Allow: /blog/
+Allow: /assets/
+
+# Disallow common system files and directories
+Disallow: /.git/
+Disallow: /.github/
+Disallow: /404.html
+
+# Sitemap location
+Sitemap: https://bitcoindevkit.org/sitemap.xml