Array
(
[googlebot] => Array
(
[disallow] => Array
(
[0] => /?*
)
)
[baiduspider] => Array
(
[disallow] => Array
(
[0] => /?*
)
)
[yandexbot] => Array
(
[disallow] => Array
(
[0] => /?*
)
)
[ichiro] => Array
(
[disallow] => Array
(
[0] => /?*
)
)
[sogou spider] => Array
(
[disallow] => Array
(
[0] => /?*
)
)
[sosospider] => Array
(
[disallow] => Array
(
[0] => /?*
)
)
[youdaobot] => Array
(
[disallow] => Array
(
[0] => /?*
)
)
[yetibot] => Array
(
[disallow] => Array
(
[0] => /?*
)
)
[bingbot] => Array
(
[crawl-delay] => 2
[disallow] => Array
(
[0] => /?*
)
)
[yahoo! slurp] => Array
(
[crawl-delay] => 2
[disallow] => Array
(
[0] => /?*
)
)
[rdfbot] => Array
(
[disallow] => Array
(
[0] => /?*
)
)
[seznambot] => Array
(
[disallow] => Array
(
[0] => /?*
)
)
[ia_archiver] => Array
(
)
[mediapartners-google] => Array
(
)
[*] => Array
(
[disallow] => Array
(
[0] => /
)
)
)
User-agent: Googlebot
Disallow: /?*
User-agent: Baiduspider
Disallow: /?*
User-agent: YandexBot
Disallow: /?*
User-agent: ichiro
Disallow: /?*
User-agent: sogou spider
Disallow: /?*
User-agent: Sosospider
Disallow: /?*
User-agent: YoudaoBot
Disallow: /?*
User-agent: YetiBot
Disallow: /?*
User-agent: bingbot
Crawl-delay: 2
Disallow: /?*
User-Agent: Yahoo! Slurp
Crawl-delay: 2
Disallow: /?*
User-agent: rdfbot
Disallow: /?*
User-agent: Seznambot
Request-rate: 1/2s
Disallow: /?*
User-agent: ia_archiver
Disallow:
User-agent: Mediapartners-Google
Disallow:
User-agent: *
Disallow: /
-
код 200
http://kinoprice.ru/robots.txt