# negotiateit.ai — crawl policy.
#
# AI crawlers are EXPLICITLY allowed (GEO research 2026-06-10): assistant
# recommendations are an acquisition channel, and some AI crawlers treat a
# bare wildcard as ambiguous. Each bot gets its own block because a UA-specific
# block REPLACES the * rules — so every block must re-state the /app/ + /api/ +
# /admin disallows. Notes:
#  - OAI-SearchBot powers ChatGPT search citations; GPTBot is training-only.
#  - Google-Extended is a token, not a crawler; absence of a disallow = opted in.
#  - PerplexityBot indexes for Perplexity answers (not training).

User-agent: *
Allow: /
Disallow: /app/
Disallow: /api/
Disallow: /admin

User-agent: GPTBot
Allow: /
Disallow: /app/
Disallow: /api/
Disallow: /admin

User-agent: OAI-SearchBot
Allow: /
Disallow: /app/
Disallow: /api/
Disallow: /admin

User-agent: ChatGPT-User
Allow: /
Disallow: /app/
Disallow: /api/
Disallow: /admin

User-agent: PerplexityBot
Allow: /
Disallow: /app/
Disallow: /api/
Disallow: /admin

User-agent: ClaudeBot
Allow: /
Disallow: /app/
Disallow: /api/
Disallow: /admin

User-agent: Claude-SearchBot
Allow: /
Disallow: /app/
Disallow: /api/
Disallow: /admin

User-agent: Claude-User
Allow: /
Disallow: /app/
Disallow: /api/
Disallow: /admin

User-agent: Bingbot
Allow: /
Disallow: /app/
Disallow: /api/
Disallow: /admin

User-agent: Applebot
Allow: /
Disallow: /app/
Disallow: /api/
Disallow: /admin

User-agent: Applebot-Extended
Allow: /
Disallow: /app/
Disallow: /api/
Disallow: /admin

User-agent: Meta-ExternalAgent
Allow: /
Disallow: /app/
Disallow: /api/
Disallow: /admin

User-agent: Amazonbot
Allow: /
Disallow: /app/
Disallow: /api/
Disallow: /admin

User-agent: CCBot
Allow: /
Disallow: /app/
Disallow: /api/
Disallow: /admin

Sitemap: https://negotiateit.ai/sitemap.xml