# =============================================================
# Solarkiosk.eu — robots.txt
# =============================================================
# Bifurcated AI-crawler policy (Gemini research 28 Jul 2026):
#   ALLOW real-time SEARCH / RAG bots — they drive AI citations
#         and generative-answer visibility.
#   DISALLOW TRAINING bots — they absorb IP into foundation
#         models without giving reciprocal citation traffic.
# Reference: ~/Documents/Scientific-Research/Methodology/
#            2026-07-28_External_Link_Strategy_Pre_and_Post_AI_SEO.md
# =============================================================

# -----------------------------------------------------------
# 1. Traditional search engines — full crawl
# -----------------------------------------------------------
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# -----------------------------------------------------------
# 2. AI real-time SEARCH / RAG crawlers — allowed
#    These drive citations in ChatGPT Search, Claude Search,
#    Perplexity, Google AI Overviews.
# -----------------------------------------------------------
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# -----------------------------------------------------------
# 3. AI TRAINING crawlers — disallowed
#    These harvest content for foundation-model training
#    without reciprocal citation. IP protection.
# -----------------------------------------------------------
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: DiffBot
Disallow: /

# -----------------------------------------------------------
# 4. Default — everything else may crawl
# -----------------------------------------------------------
User-agent: *
Allow: /
Disallow: /api/
Disallow: /design-system/

# -----------------------------------------------------------
# Sitemap
# -----------------------------------------------------------
Sitemap: https://www.solarkiosk.eu/sitemap-index.xml