User-Agent: *
# ❌ Служебные директории Битрикса
Disallow: /bitrix/
Disallow: /local/
Disallow: /auth*
Disallow: /basket*
Disallow: /order*
Disallow: /personal/
Disallow: /compare/
Disallow: /rss/
Disallow: /webstat/
Disallow: /desktop_app/
Disallow: /ajax/
Disallow: /test/
Disallow: /404.php
Disallow: /search/
Disallow: /personal/order/make/
Disallow: /include/
Disallow: /catalog/compare.php
Disallow: /?logout=yes
# ❌ Технические параметры
Disallow: /*?clear_cache
Disallow: /*clear_cache_session=
Disallow: /*bitrix_include_areas=
Disallow: /*show_include_exec_time=
Disallow: /*show_page_exec_time=
Disallow: /*show_sql_stat=
Disallow: /*conversion/ajax_counter.php
Disallow: /*?categ=
Disallow: /*?p=shop
Disallow: /*BLOCK=assoc
# ❌ Формы и служебные экшены Аспро
Disallow: /info/more/
Disallow: /form/
Disallow: /*/action/*
# ❌ Мусорные параметры фильтра
Disallow: /*?*FILTER*
Disallow: /*?*set_filter*
Disallow: /*?*del_filter*
Disallow: /*?*arrFilter*
Disallow: /*?q=
Disallow: /*&q=
Disallow: /*?sort=
Disallow: /*&sort=
Disallow: /*?order=
Disallow: /*&order=
Disallow: /*?view_mode=
Disallow: /*ADD_TO_COMPARE_LIST
Disallow: /*ORDER_BY
Disallow: /*?display=
Disallow: /*&display=
Disallow: /*?linerow=
Disallow: /*&linerow=
Disallow: /*?LIST=
Disallow: /*&LIST=
Disallow: /*?spell=
Disallow: /*&spell=
# ❌ Ajax-параметры (защита от дублей в Google)
Disallow: /*?bxajaxid=
Disallow: /*&bxajaxid=
# ❌ Баг arturgolubev (исправлено)
Disallow: /b14521976/
Disallow: /*cdn-ru.bitrix24.ru
Disallow: /*mc.yandex.ru
Disallow: /*www.google-analytics.com
Disallow: /*www.googletagmanager.com
# ✅ Разрешаем пагинацию (чтобы индексировались все товары в категории)
Allow: /*?PAGEN_
# ✅ Разрешаем ЧПУ-посадочные Аспро (критично для SEO)
Allow: /catalog/*/filter/*/apply/
Allow: /catalog/*/filter/*/
# ✅ Разрешаем статику и системные ресурсы
Allow: /bitrix/templates/
Allow: /bitrix/js/
Allow: /bitrix/css/
Allow: /local/templates/
Allow: /local/js/
Allow: /upload/
Allow: /upload/iblock/
Allow: /upload/medialibrary/
Allow: /bitrix/cache/
Allow: /bitrix/templates/aspro_max/js/setTheme.php
Allow: /catalog/*/
# ✅ Разрешаем статические файлы
Allow: /*.js
Allow: /*.css
Allow: /*.woff
Allow: /*.woff2
Allow: /*.svg
Allow: /*.png
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.gif
Allow: /*.webp
# ✅ LLMs.txt для ИИ-ботов
Allow: /llms.txt
Allow: /llms-full.txt

# ============================================================
# ЯНДЕКС (с Clean-param)
# ============================================================

User-Agent: Yandex
# ❌ Служебные директории Битрикса
Disallow: /bitrix/
Disallow: /local/
Disallow: /auth*
Disallow: /basket*
Disallow: /order*
Disallow: /personal/
Disallow: /compare/
Disallow: /rss/
Disallow: /webstat/
Disallow: /desktop_app/
Disallow: /ajax/
Disallow: /test/
Disallow: /404.php
Disallow: /search/
Disallow: /personal/order/make/
Disallow: /include/
Disallow: /?logout=yes
# ❌ Технические параметры
Disallow: /*?clear_cache
Disallow: /*clear_cache_session=
Disallow: /*bitrix_include_areas=
Disallow: /*show_include_exec_time=
Disallow: /*show_page_exec_time=
Disallow: /*show_sql_stat=
Disallow: /*conversion/ajax_counter.php
Disallow: /*?categ=
Disallow: /*?p=shop
Disallow: /*BLOCK=assoc
# ❌ Формы и служебные экшены Аспро
Disallow: /info/more/
Disallow: /form/
Disallow: /*/action/*
# ❌ Мусорные параметры фильтра
Disallow: /*?*FILTER*
Disallow: /*?*set_filter*
Disallow: /*?*del_filter*
Disallow: /*?*arrFilter*
Disallow: /*?q=
Disallow: /*&q=
Disallow: /*?sort=
Disallow: /*&sort=
Disallow: /*?order=
Disallow: /*&order=
Disallow: /*?view_mode=
Disallow: /*ADD_TO_COMPARE_LIST
Disallow: /*ORDER_BY
Disallow: /*?display=
Disallow: /*&display=
Disallow: /*?linerow=
Disallow: /*&linerow=
Disallow: /*?LIST=
Disallow: /*&LIST=
Disallow: /*?spell=
Disallow: /*&spell=
# ❌ Баг arturgolubev
Disallow: /b14521976/
Disallow: /*cdn-ru.bitrix24.ru
Disallow: /*mc.yandex.ru
Disallow: /*www.google-analytics.com
Disallow: /*www.googletagmanager.com
# ✅ Разрешаем пагинацию
Allow: /*?PAGEN_
# ✅ Разрешаем ЧПУ-посадочные Аспро
Allow: /catalog/*/filter/*/apply/
Allow: /catalog/*/filter/*/
# ✅ Разрешаем статику
Allow: /bitrix/templates/
Allow: /bitrix/js/
Allow: /bitrix/css/
Allow: /local/templates/
Allow: /local/js/
Allow: /upload/
Allow: /upload/iblock/
Allow: /upload/medialibrary/
Allow: /bitrix/cache/
Allow: /bitrix/templates/aspro_max/js/setTheme.php
Allow: /catalog/*/
Allow: /*.js
Allow: /*.css
Allow: /*.woff
Allow: /*.woff2
Allow: /*.svg
Allow: /*.png
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.gif
Allow: /*.webp
Allow: /llms.txt
Allow: /llms-full.txt
# ⚙️ Clean-param (только Яндекс)
Clean-param: utm_source&utm_medium&utm_campaign&utm_term&utm_content&gclid&yclid&fbclid /
Clean-param: bxajaxid&RID&PageSpeed&ls&q /
Clean-param: clear_cache&clear_cache_session&ADD_TO_COMPARE_LIST&ORDER_BY&list_style&sort_by&arrFilter /
Clean-param: PAGEN_1&PAGEN_2&PAGEN_3&PAGEN_4&PAGEN_5&PAGEN_6&PAGEN_7&PAGEN_8&PAGEN_9&PAGEN_10 /
Clean-param: print_course&register&forgot_password&change_password&login&logout&auth&backurl&back_url /
Clean-param: BACKURL&BACK_URL&back_url_admin&display&view_mode&min_price&max_price&path&route /
Clean-param: products_on_page&new&edit&SHOWALL&sort&direction /
Clean-param: LIST&linerow&spell /

# ============================================================
# ИИ-БОТЫ (управление)
# ============================================================
# --- OpenAI: поисковый бот (даёт трафик из ChatGPT Search) ---

User-Agent: OAI-SearchBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /bitrix/
Disallow: /ajax/
Disallow: /b14521976/
Disallow: /*cdn-ru.bitrix24.ru
Disallow: /*mc.yandex.ru
Disallow: /*www.google-analytics.com
Disallow: /*www.googletagmanager.com

# --- OpenAI: браузинг пользователя ChatGPT ---

User-Agent: ChatGPT-User
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /bitrix/
Disallow: /ajax/

# --- OpenAI: обучение моделей ---

User-Agent: GPTBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /bitrix/
Disallow: /ajax/
Disallow: /bitrix/tools/
Disallow: /bitrix/cache/

# --- Anthropic: Claude ---

User-Agent: ClaudeBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /bitrix/
Disallow: /ajax/
Disallow: /b14521976/

User-Agent: Claude-User
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

User-Agent: Claude-SearchBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /bitrix/
Disallow: /ajax/

# --- Perplexity ---

User-Agent: PerplexityBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /bitrix/
Disallow: /ajax/
Disallow: /b14521976/

User-Agent: Perplexity-User
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

# --- ByteDance: Bytespider (очень жадный, ограничиваем) ---

User-Agent: Bytespider
Crawl-delay: 10
Allow: /
Disallow: /bitrix/
Disallow: /ajax/
Disallow: /bitrix/tools/
Disallow: /bitrix/cache/
Disallow: /*.svg?*

# --- Meta: meta-externalagent (ограничиваем AJAX) ---

User-Agent: meta-externalagent
Allow: /
Disallow: /bitrix/
Disallow: /ajax/
Disallow: /bitrix/tools/
Disallow: /bitrix/tools/conversion/
Disallow: /bitrix/tools/concept.banner/
Disallow: /b14521976/
Disallow: /*cdn-ru.bitrix24.ru
Disallow: /*mc.yandex.ru

# --- Google: Google-Extended (для Gemini) ---

User-Agent: Google-Extended
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /bitrix/
Disallow: /ajax/

# ============================================================
# КОНЕЦ
# ============================================================

Sitemap: https://zdorov.by/sitemap.xml