# Robots.txt for AIssential
# https://aissential.tech

User-agent: *
Allow: /
Allow: /api/og/
Disallow: /api/

# /articles is the authenticated app entry — not a public SEO surface.
# The bare path is fine (it redirects to auth) but any query-string variant
# (filters, search, role) is a thin/duplicate page. Block them explicitly.
# This also kills the SearchAction template URL /articles?search=...
Disallow: /articles?
Disallow: /articles/*?

# NOTE: /articles/* is deliberately NOT disallowed for search engines. Those
# pages serve `noindex, follow`, and a Disallow would stop crawlers reading
# the noindex — leaving them indexed indefinitely. Let them crawl and drop.

# ─── AI crawlers ───────────────────────────────────────────────────────
# The article corpus went registered-only on 2026-07-31. These crawlers do
# not honour `noindex` (a search-index directive), so without an explicit
# Disallow they keep ingesting full article text that no human can read
# without an account — the same leak the closure was meant to stop.
#
# Only /articles/ is closed. The Counsel verdicts stay open on purpose:
# they are the surface we want cited, and they are free to read.

User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: claude-web
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: CCBot
User-agent: Google-Extended
User-agent: Amazonbot
User-agent: meta-externalagent
User-agent: Applebot-Extended
User-agent: Bytespider
User-agent: YouBot
User-agent: MistralAI-User
Allow: /verdict
Allow: /verdicts
Disallow: /articles/
Disallow: /api/

# Sitemap location
Sitemap: https://aissential.tech/sitemap.xml

# Curated entry point for LLMs — what AIssential is, which pages are open,
# and which entity the name refers to (several unrelated orgs share it).
# https://llmstxt.org/
LLM-Content: https://aissential.tech/llms.txt

# Crawl-delay for respectful crawling
Crawl-delay: 1