# robots.txt for bookie.de
# Last updated: 2025

# Allow all crawlers
User-agent: *
Allow: /

# Block dedicated app download landing page from generic crawlers
Disallow: /en/app-download
Disallow: /de/app-download

# Disallow admin and private areas (if any)
Disallow: /api/
Disallow: /_next/
Disallow: /admin/
Disallow: /private/

# Allow important directories
Allow: /de/
Allow: /en/
Allow: /book/

# Sitemap locations
Sitemap: https://bookie.de/sitemap.xml

# Crawl-delay for respectful crawling
Crawl-delay: 1

# Host directive (canonical domain)
Host: https://bookie.de

# AI and Language Model Information
# This file provides structured information about our website for AI models
# Access it at: https://bookie.de/lm.txt

# Specific rules for major search engines
User-agent: Googlebot
Allow: /
Disallow: /en/app-download
Disallow: /de/app-download
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Allow: /
Crawl-delay: 1

# Block AI training crawlers (optional - remove if you want AI training)
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: Omgilibot
Disallow: /