# Serac — robots.txt
# Marketing apex (serac.build) is fully indexable for search engines AND
# for AI/LLM crawlers. We want serac.build content to be discoverable
# both in classic search results and inside LLM training data and
# retrieval-time answers (Perplexity, ChatGPT search, Claude search,
# Google AI Overviews, etc.).
#
# The portal host (dashboard.serac.build) is excluded from indexing via
# a separate robots rule on that host (Cloudflare worker / _headers).
#
# IMPORTANT: do NOT also enable Cloudflare's "AI Audit > managed
# robots.txt" — it prepends a Content-Signal directive that Google's
# robots.txt validator flags as an unknown directive.

User-agent: *
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /portal/
Disallow: /login

# --- Per-bot explicit allow rules ---
# Same disallows as the wildcard block (api/admin/portal stay off-limits),
# but each AI / LLM crawler is named so the signal isn't dependent on
# wildcard interpretation.

User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /portal/

User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /portal/

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /portal/

User-agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /portal/

User-agent: anthropic-ai
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /portal/

User-agent: Claude-Web
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /portal/

User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /portal/

User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /portal/

User-agent: Applebot-Extended
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /portal/

User-agent: Amazonbot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /portal/

User-agent: CCBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /portal/

User-agent: cohere-ai
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /portal/

User-agent: Meta-ExternalAgent
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /portal/

User-agent: YouBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /portal/

User-agent: DuckAssistBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /portal/

User-agent: Bytespider
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /portal/

Sitemap: https://serac.build/sitemap.xml