# robots.txt — codecraftbase.com
# Codecraft Innovations W.L.L.
#
# Policy: maximum AI visibility. Every crawler — search-time retrieval,
# training, and user-triggered fetches — is allowed across the entire site.
# There are no admin/cart/checkout/account areas or faceted-search parameters
# on this site, so there is nothing to disallow.
#
# Note: robots.txt controls crawling only, not indexing. Per-page indexing
# directives live in <meta name="robots"> on each template.
#
# Decision recorded: 2026-08-03, Codecraft SEO/GEO/AEO project.
# Re-review when new crawler tokens appear or this policy changes.

User-agent: *
Allow: /

# --- AI search-time crawlers (power answers/citations in AI products) ---
User-agent: OAI-SearchBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: ChatGPT-User
Allow: /

# --- AI training crawlers ---
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Amazonbot
Allow: /

# --- Classic search engines ---
User-agent: Googlebot
Allow: /

User-agent: bingbot
Allow: /

User-agent: Applebot
Allow: /

Sitemap: https://codecraftbase.com/sitemap.xml