Array
(
[*] => Array
(
[disallow] => Array
(
[0] => /cgi-bin/
[1] => /wp-admin/
[2] => /readme.html
[3] => /comments/feed/
[4] => /trackback/
[5] => /index.php
[6] => /xmlrpc.php
[7] => /wp-content/plugins/
[8] => /feed/
[9] => */feed/
[10] => */?
[11] => /?
)
[allow] => Array
(
[0] => /wp-content/uploads/
)
)
[mediapartners-google] => Array
(
[allow] => Array
(
[0] => /
)
)
[googlebot-image] => Array
(
[allow] => Array
(
[0] => /wp-content/uploads/
)
)
[googlebot] => Array
(
[allow] => Array
(
[0] => /wp-content/uploads/
)
)
[adsbot-google] => Array
(
[allow] => Array
(
[0] => /
)
)
[googlebot-mobile] => Array
(
[allow] => Array
(
[0] => /
)
)
)
User-agent: *
Disallow: /cgi-bin/
Disallow: /wp-admin/
Disallow: /readme.html
Disallow: /comments/feed/
Disallow: /trackback/
Disallow: /index.php
Disallow: /xmlrpc.php
Disallow: /wp-content/plugins/
Disallow: /feed/
Disallow: */feed/
Disallow: */?
Disallow: /?
Allow: /wp-content/uploads/
User-agent: Mediapartners-Google
Allow: /
User-agent: Googlebot-Image
Allow: /wp-content/uploads/
User-agent: Googlebot
Allow: /wp-content/uploads/
User-agent: Adsbot-Google
Allow: /
User-agent: Googlebot-Mobile
Allow: /
-
код 301
http://frettboard.com/robots.txt
-
код 200
https://frettboard.com/robots.txt