# robots.txt — 2k Web Developments (https://2kwebdev.com)
#
# Policy: allow everyone, everywhere, except /data/ (source JSON, not pages).
# We WANT to be crawled AND cited by AI answer engines, so the AI citation /
# search crawlers are listed explicitly below and allowed — future-proofing in
# case any engine's default ever changes. We do NOT block training crawlers
# either: this is a small static site, bandwidth is a non-issue, and presence in
# AI parametric knowledge only helps. The only Disallow is /data/.

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /data/

# --- AI answer / citation crawlers (explicitly allowed — we want the citations) ---

# OpenAI — GPTBot (training + search index), OAI-SearchBot (ChatGPT Search
# results/citations), ChatGPT-User (user-triggered page fetches in ChatGPT).
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
Allow: /
Disallow: /data/

# Anthropic — ClaudeBot (crawler) + Claude-SearchBot (powers Claude answer citations).
User-agent: ClaudeBot
User-agent: Claude-SearchBot
Allow: /
Disallow: /data/

# Perplexity — answer-engine crawler.
User-agent: PerplexityBot
Allow: /
Disallow: /data/

# Google — Google-Extended (opt-in for Gemini / AI Overviews to use our content)
# + Googlebot (classic search, which also feeds AI Overviews).
User-agent: Google-Extended
User-agent: Googlebot
Allow: /
Disallow: /data/

# Microsoft — Bingbot (classic search + Copilot answers).
User-agent: Bingbot
Allow: /
Disallow: /data/

Sitemap: https://2kwebdev.com/sitemap_index.xml
Sitemap: https://2kwebdev.com/sitemap-blog.xml