# Lantern welcomes AI agents and crawlers.
#
# Every group below repeats the same policy on purpose. RFC 9309 section 2.2.1:
# a crawler obeys only the most specific group that matches its token, and
# groups do not merge. A named group containing just "Allow: /" would grant that
# crawler the whole site, including the authenticated app. This mirrors the
# template Lantern generates for customers in
# libs/shared-actions/src/lib/robots-txt-template.ts.
#
# Guarded by scripts/ci/check-robots-groups.mjs (npm run validate:robots-groups).

User-Agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /settings/
Disallow: /admin/
Disallow: /auth/
Disallow: /onboard
Disallow: /onboarding
Disallow: /dev
Disallow: /oauth/consent
Disallow: /slack/link
Disallow: /connect/
Disallow: /pricing/success

User-Agent: GPTBot
User-Agent: OAI-SearchBot
User-Agent: ChatGPT-User
User-Agent: ClaudeBot
User-Agent: anthropic-ai
User-Agent: Claude-User
User-Agent: PerplexityBot
User-Agent: Perplexity-User
User-Agent: Google-Extended
User-Agent: Applebot-Extended
User-Agent: CCBot
User-Agent: Meta-ExternalAgent
User-Agent: Amazonbot
User-Agent: Bytespider
User-Agent: cohere-ai
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /settings/
Disallow: /admin/
Disallow: /auth/
Disallow: /onboard
Disallow: /onboarding
Disallow: /dev
Disallow: /oauth/consent
Disallow: /slack/link
Disallow: /connect/
Disallow: /pricing/success

# Machine-readable site guide for LLM agents (served from the published content manifest):
#   https://lantern.is/llms.txt
#   https://lantern.is/llms-full.txt
#
# RSS 2.0 feed over published /resources/* articles (final-branch review
# finding 4): no standard robots.txt directive announces a feed, so this is
# a discovery comment, not a crawl directive.
#   https://lantern.is/resources/feed.xml

Sitemap: https://lantern.is/sitemap.xml