# TuneVault — robots.txt
# Last updated: 2026-08-13
#
# STRUCTURE NOTE — read before editing:
# A crawler obeys ONLY the most specific user-agent group that matches it, and
# ignores "User-agent: *" entirely once it has matched a named group. Until
# 2026-08-13 each AI crawler had its own group containing nothing but Allow
# lines, which meant GPTBot/ClaudeBot/PerplexityBot inherited NO Disallow rules
# at all and were being invited to crawl /admin/, /api/ and the authenticated
# app pages. The per-bot Allow lists had also drifted apart (Googlebot listed
# the /features/* deep-dives, Bingbot did not) while adding nothing: everything
# not disallowed is already allowed by default.
#
# So: one shared rule body, three groups. If you add a path to one Disallow
# list, add it to all three, or the AI crawlers will keep crawling it.

# ─── Search engine crawlers ──────────────────────────────────────────────────
User-agent: Googlebot
User-agent: Bingbot
User-agent: DuckDuckBot
User-agent: Applebot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /admin-schedules
Disallow: /healthcheck
Disallow: /dashboard
Disallow: /performance
Disallow: /db-ops
Disallow: /ebs-deep
Disallow: /ebs-middleware
Disallow: /ebs-concurrent
Disallow: /patches
Disallow: /sanity-check
Disallow: /report
Disallow: /report-ebs
Disallow: /fleet
Disallow: /clients
Disallow: /autopilot
Disallow: /agent
Disallow: /sql-console
Disallow: /terminal
Disallow: /fndload
Disallow: /ebs-ops
Disallow: /ebs-patches
Disallow: /ebs-status-sources
Disallow: /ebs-log-tail
Disallow: /clone
Disallow: /db-clone
Disallow: /ebs-clone
Disallow: /manager
Disallow: /patches-status
Disallow: /activity
Disallow: /settings
Disallow: /settings-billing
Disallow: /settings-api
Disallow: /settings-security
Disallow: /settings-sso
Disallow: /settings-alerts
Disallow: /settings-compliance
Disallow: /signin
Disallow: /invite-accept
Disallow: /mfa-challenge
Disallow: /payment-test
Disallow: /validator

# ─── AI crawlers and answer engines ──────────────────────────────────────────
# Two distinct kinds are listed together because they get identical access:
#   - Training/index crawlers: GPTBot, ClaudeBot, Google-Extended, Applebot-Extended,
#     meta-externalagent, Amazonbot, cohere-ai
#   - Answer-time fetchers, which retrieve a page while composing a reply to a
#     user and are what actually decides whether TuneVault appears in an AI
#     answer today: OAI-SearchBot, ChatGPT-User, Claude-User, Claude-SearchBot,
#     PerplexityBot, Perplexity-User, DuckAssistBot
# GPTBot and OAI-SearchBot are separate tokens; so are ClaudeBot, Claude-User
# and Claude-SearchBot. Allowing one does not allow the others.
#
# Ground-truth files for these crawlers: /llms.txt and /llms-full.txt
# (routes/ai-discoverability.js). Keep those accurate — an LLM will quote them.
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Applebot-Extended
User-agent: meta-externalagent
User-agent: Amazonbot
User-agent: cohere-ai
User-agent: DuckAssistBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /admin-schedules
Disallow: /healthcheck
Disallow: /dashboard
Disallow: /performance
Disallow: /db-ops
Disallow: /ebs-deep
Disallow: /ebs-middleware
Disallow: /ebs-concurrent
Disallow: /patches
Disallow: /sanity-check
Disallow: /report
Disallow: /report-ebs
Disallow: /fleet
Disallow: /clients
Disallow: /autopilot
Disallow: /agent
Disallow: /sql-console
Disallow: /terminal
Disallow: /fndload
Disallow: /ebs-ops
Disallow: /ebs-patches
Disallow: /ebs-status-sources
Disallow: /ebs-log-tail
Disallow: /clone
Disallow: /db-clone
Disallow: /ebs-clone
Disallow: /manager
Disallow: /patches-status
Disallow: /activity
Disallow: /settings
Disallow: /settings-billing
Disallow: /settings-api
Disallow: /settings-security
Disallow: /settings-sso
Disallow: /settings-alerts
Disallow: /settings-compliance
Disallow: /signin
Disallow: /invite-accept
Disallow: /mfa-challenge
Disallow: /payment-test
Disallow: /validator

# ─── Global rules (all other crawlers) ───────────────────────────────────────
User-agent: *
Allow: /

# Block API and admin endpoints — not indexable content
Disallow: /api/
Disallow: /admin/
Disallow: /admin-schedules
Disallow: /healthcheck

# Block authenticated app pages (require login)
Disallow: /dashboard
Disallow: /performance
Disallow: /db-ops
Disallow: /ebs-deep
Disallow: /ebs-middleware
Disallow: /ebs-concurrent
Disallow: /patches
Disallow: /sanity-check
Disallow: /report
Disallow: /report-ebs
Disallow: /fleet
Disallow: /clients
Disallow: /autopilot
Disallow: /agent
Disallow: /sql-console
Disallow: /terminal
Disallow: /fndload
Disallow: /ebs-ops
Disallow: /ebs-patches
Disallow: /ebs-status-sources
Disallow: /ebs-log-tail
Disallow: /clone
Disallow: /db-clone
Disallow: /ebs-clone
Disallow: /manager
Disallow: /patches-status
Disallow: /activity

# Block settings pages
Disallow: /settings
Disallow: /settings-billing
Disallow: /settings-api
Disallow: /settings-security
Disallow: /settings-sso
Disallow: /settings-alerts
Disallow: /settings-compliance

# Block auth flow pages
Disallow: /signin
Disallow: /invite-accept
Disallow: /mfa-challenge
Disallow: /payment-test

# Block debug/test pages
Disallow: /validator

Sitemap: https://tunevault.app/sitemap.xml
