# AEPT-AI Web Crawler Policy -- zephyradv.com
# Set 2026-07-15 (Alexander Moore, CPA).
#
# Rationale: AI *training* crawlers are NOT additive for SEO, GEO/AEO
# citations, referral traffic, or backlinks. Those benefits come from the
# *search* and *user-fetch* agents, which are SEPARATE bots from the training
# crawlers and are governed by their own robots.txt directives. Training use
# is uncompensated, unattributed, and costs bandwidth with no visibility
# return -- so we opt OUT of training while staying fully eligible for
# AI-search citations and classic search ranking.
#
# Policy: BLOCK AI training crawlers; ALLOW AI search + user-fetch agents and
# all classic search engines. Review the training-bot list and verify current
# user-agent strings against each vendor's crawler docs periodically
# (Anthropic: docs.anthropic.com/en/docs/build-with-claude/web-crawling).

# --- Default: classic search engines (Googlebot, Bingbot, ...) + anything
#     not named below are ALLOWED. This is what drives traditional SEO. ---
User-agent: *
Allow: /

# --- AI SEARCH / USER-FETCH agents: EXPLICITLY ALLOWED ---
# These are the AI-citation channel (GEO/AEO). Keeping them open is the whole
# point -- blocking them would delete us from AI answers.
User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Applebot
Allow: /

# --- AI TRAINING crawlers: BLOCKED (not additive; opt out of training) ---
# Note: Google-Extended blocks Gemini *training* only and does NOT affect
# Google Search ranking (Googlebot is separate, allowed above). Same pattern
# for Applebot-Extended vs Applebot.
User-agent: ClaudeBot
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: cohere-ai
Disallow: /

Sitemap: https://zephyradv.com/sitemap.xml