Array
(
[yandex] => Array
(
[disallow] => Array
(
[0] => /web/
[1] => /cgi-bin/
[2] => /questions/interests
[3] => /questions/favorites
[4] => /questions/bonus
[5] => /questions/popular
[6] => /questions/unanswered
[7] => /questions/actual
[8] => /questions/actual/cat*_p*.html
[9] => /tags
[10] => /badges
[11] => /profile
[12] => /community
[13] => *ua_checksum
)
[allow] => Array
(
[0] => /questions/actual.html
[1] => /questions/actual_p2.html
[2] => /questions/actual_p3.html
[3] => /questions/actual_p4.html
[4] => /questions/actual_p5.html
[5] => /questions/actual_p6.html
[6] => /questions/actual_p7.html
[7] => /questions/actual_p8.html
[8] => /questions/actual_p9.html
[9] => /questions/actual_p10.html
[10] => /questions/actual_p11.html
[11] => /questions/actual_p12.html
[12] => /questions/actual_p13.html
[13] => /questions/actual_p14.html
[14] => /questions/actual_p15.html
[15] => /questions/actual_p16.html
[16] => /questions/actual_p17.html
[17] => /questions/actual_p18.html
[18] => /questions/actual_p19.html
[19] => /questions/actual_p20.html
[20] => /questions/actual/cat*.html
[21] => /questions/actual/cat*_p2.html
[22] => /questions/actual/cat*_p3.html
[23] => /questions/actual/cat*_p4.html
[24] => /questions/actual/cat*_p6.html
[25] => /questions/actual/cat*_p7.html
[26] => /questions/actual/cat*_p8.html
[27] => /questions/actual/cat*_p9.html
[28] => /questions/actual/cat*_p10.html
[29] => /web/sitemap.cgi
)
[crawl-delay] => 0.1
[clean-param] => Array
(
[0] => fr&suggest_reqid
)
)
[008] => Array
(
[disallow] => Array
(
[0] => /
)
)
[ahrefsbot] => Array
(
[disallow] => Array
(
[0] => /
)
)
[ccbot] => Array
(
[disallow] => Array
(
[0] => /
)
)
[sogou web spider] => Array
(
[disallow] => Array
(
[0] => /
)
)
[gptbot] => Array
(
[disallow] => Array
(
[0] => /
)
)
[chatgpt-user] => Array
(
[disallow] => Array
(
[0] => /
)
)
[google-extended] => Array
(
[disallow] => Array
(
[0] => /
)
)
[facebookbot] => Array
(
[disallow] => Array
(
[0] => /
)
)
[omgilibot] => Array
(
[disallow] => Array
(
[0] => /
)
)
[mj12bot] => Array
(
[disallow] => Array
(
[0] => /
)
)
[amazonbot] => Array
(
[disallow] => Array
(
[0] => /
)
)
[mediapartners-google] => Array
(
[clean-param] => Array
(
[0] => fr&suggest_reqid
)
)
[*] => Array
(
[disallow] => Array
(
[0] => /web/
[1] => /cgi-bin/
[2] => /questions/interests
[3] => /questions/favorites
[4] => /questions/bonus
[5] => /questions/popular
[6] => /questions/unanswered
[7] => /questions/actual
[8] => /tags
[9] => /badges
[10] => /profile
[11] => /community
[12] => /infopage_privacy.html
[13] => *ua_checksum
)
[allow] => Array
(
[0] => /questions/actual.html
[1] => /questions/actual_p2.html
[2] => /questions/actual_p3.html
[3] => /questions/actual_p4.html
[4] => /questions/actual_p5.html
[5] => /questions/actual_p6.html
[6] => /questions/actual_p7.html
[7] => /questions/actual_p8.html
[8] => /questions/actual_p9.html
[9] => /questions/actual_p10.html
[10] => /questions/actual_p11.html
[11] => /questions/actual_p12.html
[12] => /questions/actual_p13.html
[13] => /questions/actual_p14.html
[14] => /questions/actual_p15.html
[15] => /questions/actual_p16.html
[16] => /questions/actual_p17.html
[17] => /questions/actual_p18.html
[18] => /questions/actual_p19.html
[19] => /questions/actual_p20.html
)
[clean-param] => Array
(
[0] => fr&suggest_reqid
)
[sitemap] => Array
(
[0] => https://www.bolshoyvopros.ru/sitemap.xml
)
[host] => www.bolshoyvopros.ru
)
)
User-agent: Yandex
Disallow: /web/
Disallow: /cgi-bin/
Disallow: /questions/interests
Disallow: /questions/favorites
Disallow: /questions/bonus
Disallow: /questions/popular
Disallow: /questions/unanswered
Disallow: /questions/actual
Allow: /questions/actual.html
Allow: /questions/actual_p2.html
Allow: /questions/actual_p3.html
Allow: /questions/actual_p4.html
Allow: /questions/actual_p5.html
Allow: /questions/actual_p6.html
Allow: /questions/actual_p7.html
Allow: /questions/actual_p8.html
Allow: /questions/actual_p9.html
Allow: /questions/actual_p10.html
Allow: /questions/actual_p11.html
Allow: /questions/actual_p12.html
Allow: /questions/actual_p13.html
Allow: /questions/actual_p14.html
Allow: /questions/actual_p15.html
Allow: /questions/actual_p16.html
Allow: /questions/actual_p17.html
Allow: /questions/actual_p18.html
Allow: /questions/actual_p19.html
Allow: /questions/actual_p20.html
Allow: /questions/actual/cat*.html
Disallow: /questions/actual/cat*_p*.html
Allow: /questions/actual/cat*_p2.html
Allow: /questions/actual/cat*_p3.html
Allow: /questions/actual/cat*_p4.html
Allow: /questions/actual/cat*_p6.html
Allow: /questions/actual/cat*_p7.html
Allow: /questions/actual/cat*_p8.html
Allow: /questions/actual/cat*_p9.html
Allow: /questions/actual/cat*_p10.html
Allow: /web/sitemap.cgi
Disallow: /tags
Disallow: /badges
Disallow: /profile
Disallow: /community
Disallow: *ua_checksum
Crawl-Delay: 0.1
Clean-param: fr&suggest_reqid
User-agent: 008
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: CCBot
Disallow: /
# ---------- AI bots block -------------
# Used for many other (non-commercial) purposes as well
User-agent: CCBot
Disallow: /
# some AI spider
User-agent: Sogou web spider
Disallow: /
# For new training only
User-agent: GPTBot
Disallow: /
# Not for training, only for user requests
User-agent: ChatGPT-User
Disallow: /
# Marker for disabling Bard and Vertex AI
User-agent: Google-Extended
Disallow: /
# Speech synthesis only?
User-agent: FacebookBot
Disallow: /
# Multi-purpose, commercial uses; including LLMs
User-agent: Omgilibot
Disallow: /
# -----------------------
User-agent: MJ12bot
Disallow: /
User-agent: Amazonbot
Disallow: /
# empty dissalow (allow all) for MediaPartners-Google (?)
User-agent: Mediapartners-Google
Disallow:
Clean-param: fr&suggest_reqid
User-agent: *
Disallow: /web/
Disallow: /cgi-bin/
Disallow: /questions/interests
Disallow: /questions/favorites
Disallow: /questions/bonus
Disallow: /questions/popular
Disallow: /questions/unanswered
Disallow: /questions/actual
Allow: /questions/actual.html
Allow: /questions/actual_p2.html
Allow: /questions/actual_p3.html
Allow: /questions/actual_p4.html
Allow: /questions/actual_p5.html
Allow: /questions/actual_p6.html
Allow: /questions/actual_p7.html
Allow: /questions/actual_p8.html
Allow: /questions/actual_p9.html
Allow: /questions/actual_p10.html
Allow: /questions/actual_p11.html
Allow: /questions/actual_p12.html
Allow: /questions/actual_p13.html
Allow: /questions/actual_p14.html
Allow: /questions/actual_p15.html
Allow: /questions/actual_p16.html
Allow: /questions/actual_p17.html
Allow: /questions/actual_p18.html
Allow: /questions/actual_p19.html
Allow: /questions/actual_p20.html
Disallow: /tags
Disallow: /badges
Disallow: /profile
Disallow: /community
Disallow: /infopage_privacy.html
Disallow: *ua_checksum
Clean-param: fr&suggest_reqid
Sitemap: https://www.bolshoyvopros.ru/sitemap.xml
Host: www.bolshoyvopros.ru
-
код 200
http://bolshoyvopros.ru/robots.txt