User-agent: *
Allow: /

# ---------------------------------------------------------------------------
# AI crawler policy (reviewed 2026-07-19). Rationale: 90DJ's content IS the
# product, so we block pure TRAINING crawlers (content used for model weights,
# no traffic back) but explicitly ALLOW citation/search crawlers (they quote us
# WITH a link). Blocking a vendor's training bot does not affect its search bot.
# Do NOT copy this to the services/brand sites (BPHQ, RennDriver) — they benefit
# from being found/recommended and should stay open.
# ---------------------------------------------------------------------------

# --- Training crawlers: BLOCKED (no traffic value, donates the product) ---
User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CloudflareBrowserRenderingCrawler
Disallow: /

User-agent: GPTBot
Disallow: /

# Google-Extended controls Gemini TRAINING *and* grounding (Google bundles them,
# unlike OpenAI). Kept blocked: Googlebot already covers Search + AI Overviews
# regardless, so the grounding given up is small. Revisit if Gemini referrals
# ever show up in analytics.
User-agent: Google-Extended
Disallow: /

User-agent: meta-externalagent
Disallow: /

# --- Citation / search crawlers: EXPLICITLY ALLOWED (they cite us with a link) ---
User-agent: OAI-SearchBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Applebot
Allow: /

Sitemap: https://90dayjapanese.com/sitemap-index.xml