# ClearList robots.txt
# Allow all standard and AI crawlers.
#
# RFC 9309 gotcha: a crawler that matches a NAMED group ignores the "*" group
# entirely — it does not inherit its Disallow lines. Every named group below
# therefore shares the SAME rule block as "*" (multiple User-agent lines per
# group is standard). A named group containing only "Allow: /" would have
# granted that crawler access to the private paths.

User-agent: *
Allow: /
Allow: /api/og
Allow: /api/health
Allow: /api/status
Content-Signal: ai-train=no, search=yes, ai-input=yes
Disallow: /dashboard
Disallow: /items
Disallow: /inbox
Disallow: /settings
Disallow: /publish
Disallow: /upgrade
Disallow: /pickup
Disallow: /pickups
Disallow: /api/auth/
Disallow: /api/admin/
Disallow: /api/webhooks/
Disallow: /api/cron/
Disallow: /api/settings/
Disallow: /api/account/
Disallow: /api/ai-jobs/
Disallow: /api/preview-email/

# Social media crawlers (link previews) + AI crawlers, explicitly welcomed.
# Same rules as "*" — see the RFC 9309 note above.
User-agent: facebookexternalhit
User-agent: WhatsApp
User-agent: Twitterbot
User-agent: LinkedInBot
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: PerplexityBot
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Google-Extended
User-agent: Bytespider
User-agent: CCBot
User-agent: cohere-ai
Allow: /
Allow: /api/og
Allow: /api/health
Allow: /api/status
Content-Signal: ai-train=no, search=yes, ai-input=yes
Disallow: /dashboard
Disallow: /items
Disallow: /inbox
Disallow: /settings
Disallow: /publish
Disallow: /upgrade
Disallow: /pickup
Disallow: /pickups
Disallow: /api/auth/
Disallow: /api/admin/
Disallow: /api/webhooks/
Disallow: /api/cron/
Disallow: /api/settings/
Disallow: /api/account/
Disallow: /api/ai-jobs/
Disallow: /api/preview-email/

# Sitemap
Sitemap: https://clearlist.me/sitemap.xml