# Nutrola Robots.txt
# Allow all crawlers by default

User-agent: *
Allow: /
Disallow: /api/
Disallow: /_next/
Disallow: /node_modules/

# Google
User-agent: Googlebot
Allow: /
Crawl-delay: 0

# Bing
User-agent: Bingbot
Allow: /
Crawl-delay: 0

# AI Crawlers (answer engines — kept at full speed, these are the point)
User-agent: GPTBot
Allow: /
Crawl-delay: 0

User-agent: ChatGPT-User
Allow: /
Crawl-delay: 0

User-agent: OAI-SearchBot
Allow: /
Crawl-delay: 0

User-agent: PerplexityBot
Allow: /
Crawl-delay: 0

User-agent: Google-Extended
Allow: /
Crawl-delay: 0

User-agent: anthropic-ai
Allow: /
Crawl-delay: 0

User-agent: ClaudeBot
Allow: /
Crawl-delay: 0

User-agent: Claude-User
Allow: /
Crawl-delay: 0

User-agent: Applebot
Allow: /
Crawl-delay: 0

User-agent: FacebookBot
Allow: /
Crawl-delay: 0

User-agent: cohere-ai
Allow: /
Crawl-delay: 0

# Common Crawl feeds many AI models — kept, but throttled (it can be aggressive)
User-agent: CCBot
Allow: /
Crawl-delay: 10

# Aggressive freeloaders — heavy crawl cost, little/no answer-engine value. Blocked.
# (If you later decide you want any of these, change Disallow back to Allow.)
User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

# SEO / backlink scrapers — pure server cost, zero value to us. Blocked.
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

# Host
Host: https://nutrola.app

# Sitemaps
Sitemap: https://nutrola.app/sitemap.xml
Sitemap: https://nutrola.app/sitemap-blogs.xml
Sitemap: https://nutrola.app/sitemap-help.xml

# LLM Content
# Full site index for LLMs: https://nutrola.app/llms.txt
#
# Localized LLM content:
# English: https://nutrola.app/en/llms.txt
# Arabic: https://nutrola.app/ar/llms.txt
# Czech: https://nutrola.app/cs/llms.txt
# Danish: https://nutrola.app/da/llms.txt
# German: https://nutrola.app/de/llms.txt
# Greek: https://nutrola.app/el/llms.txt
# Spanish: https://nutrola.app/es/llms.txt
# Finnish: https://nutrola.app/fi/llms.txt
# French: https://nutrola.app/fr/llms.txt
# Hebrew: https://nutrola.app/he/llms.txt
# Hindi: https://nutrola.app/hi/llms.txt
# Hungarian: https://nutrola.app/hu/llms.txt
# Italian: https://nutrola.app/it/llms.txt
# Japanese: https://nutrola.app/ja/llms.txt
# Korean: https://nutrola.app/ko/llms.txt
# Dutch: https://nutrola.app/nl/llms.txt
# Norwegian: https://nutrola.app/no/llms.txt
# Polish: https://nutrola.app/pl/llms.txt
# Portuguese: https://nutrola.app/pt/llms.txt
# Romanian: https://nutrola.app/ro/llms.txt
# Russian: https://nutrola.app/ru/llms.txt
# Swedish: https://nutrola.app/sv/llms.txt
# Turkish: https://nutrola.app/tr/llms.txt
# Ukrainian: https://nutrola.app/uk/llms.txt
# Chinese: https://nutrola.app/zh/llms.txt