From d9bfaacc72937b74dc963bc88cd38db009b5dcee Mon Sep 17 00:00:00 2001 From: Steve Myers Date: Wed, 20 May 2026 16:37:06 -0500 Subject: [PATCH] docs: add robots.txt --- docs/robots.txt | 64 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 docs/robots.txt diff --git a/docs/robots.txt b/docs/robots.txt new file mode 100644 index 0000000000..864d53b1f3 --- /dev/null +++ b/docs/robots.txt @@ -0,0 +1,64 @@ +# robots.txt for bitcoindevkit.org +# See RFC 9309: https://www.rfc-editor.org/rfc/rfc9309 + +# Content Signals - Declare AI content usage preferences +# See https://contentsignals.org/ +Content-Signal: ai-train=yes, search=yes, ai-input=yes + +# AI Crawlers - Allow training and search +User-agent: GPTBot +Allow: / + +User-agent: ChatGPT-User +Allow: / + +User-agent: OAI-SearchBot +Allow: / + +User-agent: Claude-Web +Allow: / + +User-agent: ClaudeBot +Allow: / + +User-agent: Google-Extended +Allow: / + +User-agent: PerplexityBot +Allow: / + +User-agent: Amazonbot +Allow: / + +User-agent: anthropic-ai +Allow: / + +User-agent: Applebot-Extended +Allow: / + +User-agent: CCBot +Allow: / + +User-agent: FacebookBot +Allow: / + +User-agent: cohere-ai +Allow: / + +# Allow all other compliant crawlers to access all content +User-agent: * +Allow: / + +# Paths to crawl +Allow: /foundation/ +Allow: /adoption/ +Allow: /blog/ +Allow: /assets/ + +# Disallow common system files and directories +Disallow: /.git/ +Disallow: /.github/ +Disallow: /404.html + +# Sitemap location +Sitemap: https://bitcoindevkit.org/sitemap.xml -- 2.49.0