# robots.txt für lexyhost.de
# Diese Datei regelt den Zugriff von Suchmaschinen- und KI-Crawlern.

User-agent: *
Disallow: /admin/
Disallow: /private/
Disallow: /cart/
Disallow: /checkout/
Disallow: /api/
Disallow: /cgi-bin/
Disallow: /install000/
Allow: /

# --- KI / LLM Crawler Erlaubnis ---

# OpenAI / ChatGPT
User-agent: ChatGPT-User
Allow: /
User-agent: ChatGPT-UserAgent
Allow: /
User-agent: OAI-SearchBot
Allow: /
User-agent: OAI-Agent
Allow: /
User-agent: OAI_bot
Allow: /

# Anthropic Claude
User-agent: ClaudeBot
Allow: /

# Google (Gemini / Bard)
User-agent: Google-Extended
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# You.com
User-agent: YouBot
Allow: /

# NeevaAI (ehem. You.com-Partner)
User-agent: NeevaAI
Allow: /

# DuckDuckGo (DuckAssist / Chat)
User-agent: DuckAssistBot
Allow: /

# Bing (Copilot)
User-agent: Bingbot
Allow: /
User-agent: BingPreview
Allow: /
User-agent: bingai
Allow: /

# Applebot (Siri / Spotlight)
User-agent: Applebot
Allow: /

# Meta (Facebook / Threads / Llama)
User-agent: FacebookBot
Allow: /
User-agent: Meta-ExternalAgent
Allow: /

# CCBot (Common Crawl - Basis für viele KI-Datensätze)
User-agent: CCBot
Allow: /

# DataForSeo / Diffbot (Datensammler für KI)
User-agent: Diffbot
Allow: /
User-agent: DataForSeoBot
Allow: /

# --- Sitemap ---
Sitemap: https://lexyhost.de/sitemap.xml