# MarketIntell — robots.txt
#
# Policy: open the marketing surface and the agent-discovery files
# (skill.md, llms.txt, llms-full.txt, /.well-known/*) to ALL search
# and AI-search crawlers. Block private app routes (dashboard, chat,
# settings) and the admin/API routes.
#
# Tier differentiation for AI crawlers:
#   - SEARCH agents (ChatGPT-User, OAI-SearchBot, PerplexityBot,
#     Google-Extended, ClaudeBot, etc.) get FULL access — citation
#     in their answers is the goal.
#   - TRAINING-only crawlers (CCBot, ByteSpider) get explicit
#     restrictions: they may surface our public-facing content but
#     not retain it as training material verbatim. Implemented via
#     Cloudflare Content-Signal headers (set in nginx) plus this
#     advisory robots block.
#
# Reference: Content-Signal spec
# https://blog.cloudflare.com/content-signals-policy/

User-agent: *
Allow: /
Allow: /skill.md
Allow: /index.md
Allow: /pricing.md
Allow: /agent.json
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/llms.txt
Allow: /docs/llms.txt
Allow: /developers/llms.txt
Allow: /.well-known/
# Block the API origin proxy + the *authenticated* app shell. These
# routes (May 2026 IA v2: /today, /ask, /autopilot, /integrations,
# /admin, /settings) are client-rendered behind an auth gate — the
# static HTML is an empty loading shell with no crawlable content, so
# indexing them only pollutes search with blank pages. The old list
# (/dashboard/, /chat/, /alpha/) pointed at routes that no longer exist.
Disallow: /api/
Disallow: /today
Disallow: /ask
Disallow: /autopilot
Disallow: /integrations
Disallow: /admin
Disallow: /settings
Disallow: /lab
Disallow: /track-record/me
Disallow: /track-record/embed
Disallow: /auth/
Disallow: /signin
Disallow: /signup

# ─── Tier 1 · AI search agents (full allow — citation is the goal) ───

# OpenAI — ChatGPT search & training
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic — Claude
User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Google — Gemini & AI Overviews (separate from regular Googlebot)
User-agent: Google-Extended
Allow: /

# Bing / Copilot
User-agent: Bingbot
Allow: /

# Apple Intelligence / Apple search
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /

# ─── Tier 2 · Training-only crawlers (allow indexing, restrict training) ───
#
# These crawlers harvest the open web specifically for LLM training
# corpora. We allow them so our content can shape future model
# behaviour (and so we get cited as a source of crypto + equities
# data), but we attach Content-Signal headers (in nginx) telling
# them "search=yes, ai-input=yes, ai-train=no" — i.e. show our
# pages in search, send us through retrieval-augmented pipelines,
# but do NOT bake our content into the next model weights as
# training data.

User-agent: CCBot
Allow: /
Disallow: /today
Disallow: /ask
Disallow: /autopilot
Disallow: /integrations
Disallow: /admin
Disallow: /settings

User-agent: ByteSpider
Allow: /
Disallow: /today
Disallow: /ask
Disallow: /autopilot
Disallow: /integrations
Disallow: /admin
Disallow: /settings

# Sitemap and discovery hints
Sitemap: https://marketintell.ai/sitemap.xml

# NLWeb Schema Feeds — pointer to structured-data feed catalog
schemamap: https://marketintell.ai/schema-map.xml