# == Buscadores principales ==
User-agent: Googlebot
Disallow:
User-agent: Googlebot-Image
Allow: /
User-agent: Adsbot-Google
Allow: /
User-agent: Googlebot-Mobile
Allow: /
User-agent: Bingbot
Disallow:
User-agent: DuckDuckBot
Disallow:
User-agent: Screaming Frog SEO Spider
Disallow:

# == AI Crawlers - Permitir los que generan visibilidad (GEO) ==
User-agent: GPTBot
Allow: /
User-agent: ChatGPT-User
Allow: /
User-agent: ClaudeBot
Allow: /
User-agent: Claude-Web
Allow: /
User-agent: PerplexityBot
Allow: /
User-agent: Google-Extended
Allow: /
User-agent: Applebot-Extended
Allow: /
User-agent: Amazonbot
Allow: /
User-agent: FacebookBot
Allow: /
# CCBot (Common Crawl) alimenta los datasets de entrenamiento de los modelos:
# abierto a proposito el 16-sep-2026 para trabajar la memoria del modelo (factor 3).
User-agent: CCBot
Allow: /

# == AI Crawlers - Bloquear scrapers agresivos ==
User-agent: Bytespider
Disallow: /
User-agent: cohere-ai
Disallow: /

# == Scrapers y herramientas de copia ==
User-agent: HTTrack
Disallow: /
User-agent: Wget
Disallow: /
User-agent: TeleportPro
Disallow: /
User-agent: WebCopier
Disallow: /
User-agent: WebZIP
Disallow: /
User-agent: Offline Explorer
Disallow: /
User-agent: SiteSnagger
Disallow: /

# == Buscadores regionales (no relevantes) ==
User-agent: Baiduspider
Disallow: /
User-agent: Yandex
Disallow: /
User-agent: YandexBot
Disallow: /
User-agent: YandexTurbo
Disallow: /

# == Ajustes de crawl ==
User-agent: archive.org_bot
Visit-time: 0200-0600
User-agent: msnbot
Crawl-delay: 30
User-agent: Slurp
Crawl-delay: 10

# == Resto de bots: permitir ==
# OJO: /mailing/ NO se bloquea aquí a propósito. Se desindexa con la cabecera
# X-Robots-Tag: noindex de netlify.toml, y para leerla los bots necesitan poder
# rastrear la URL. Bloquearla aquí dejaría esas páginas indexadas para siempre.
User-agent: *
Disallow:

Sitemap: https://ondho.com/sitemap.xml
Sitemap: https://ondho.com/ca/sitemap.xml
Sitemap: https://ondho.com/diccionario-de-marketing/sitemap.xml
Sitemap: https://ondho.com/diccionario-de-branding/sitemap.xml