# soldatov-sem.ru
#
# Training crawlers and answer crawlers are DIFFERENT agents. Blocking a training
# bot does not remove you from AI answers, and blocking an answer bot removes you
# from citations entirely. Everything is allowed here on purpose: for a personal
# brand living on lead generation, being cited is worth more than withholding text
# from training. To opt out of training later, flip the training block to Disallow.

# --- answer / citation crawlers: keep these open or lose AI citations ---
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# --- classic search. Googlebot also builds AI Overviews; Bingbot feeds ChatGPT search ---
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: YandexBot
Allow: /

# --- training crawlers ---
# Google-Extended controls Gemini TRAINING only, never AI Overviews.
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

# --- everyone else ---
User-agent: *
Allow: /
Disallow: /api/

Sitemap: https://soldatov-sem.ru/sitemap-index.xml