Array
(
[*] => Array
(
[crawl-delay] => 5
[disallow] => Array
(
[0] => /3rdparty/
[1] => /admin/
[2] => /admin/admin_index.php
[3] => /backup/
[4] => /cache/
[5] => /install/
[6] => /internal/
[7] => /languages/
[8] => /libs/
[9] => /live/
[10] => /LICENSE.txt
[11] => /logs/
[12] => /modules/
[13] => /plugins/
[14] => /readme.html
[15] => /search.php
[16] => /search/
[17] => /searchurl/
[18] => /tag/
[19] => /templates/
[20] => /new/recent/
[21] => /new/yesterday/
[22] => /new/today/
[23] => /new/week/
[24] => /new/month/
[25] => /new/year/
[26] => /new/alltime/
[27] => /recent/
[28] => /yesterday/
[29] => /today/
[30] => /week/
[31] => /month/
[32] => /year/
[33] => /alltime/
[34] => /upvoted/
[35] => /downvoted/
[36] => /commented/
)
)
)
# 1) this filename (robots.txt) must stay lowercase
# 2) this file must be in the servers root directory
# ex: http://www.mydomain.com/pliggsubfolder/ -- you must move the robots.txt from
# /pliggsubfolder/ to the root folder for http://www.mydomain.com/
# you must then add your subfolder to each 'Disallow' below
# ex: Disallow: /cache/ becomes Disallow: /pliggsubfolder/cache/
User-agent: *
Crawl-delay: 5
Disallow:
Disallow: /3rdparty/
Disallow: /admin/
Disallow: /admin/admin_index.php
Disallow: /backup/
Disallow: /cache/
Disallow: /install/
Disallow: /internal/
Disallow: /languages/
Disallow: /libs/
Disallow: /live/
Disallow: /LICENSE.txt
Disallow: /logs/
Disallow: /modules/
Disallow: /plugins/
Disallow: /readme.html
Disallow: /search.php
Disallow: /search/
Disallow: /searchurl/
Disallow: /tag/
Disallow: /templates/
Disallow: /new/recent/
Disallow: /new/yesterday/
Disallow: /new/today/
Disallow: /new/week/
Disallow: /new/month/
Disallow: /new/year/
Disallow: /new/alltime/
Disallow: /recent/
Disallow: /yesterday/
Disallow: /today/
Disallow: /week/
Disallow: /month/
Disallow: /year/
Disallow: /alltime/
Disallow: /upvoted/
Disallow: /downvoted/
Disallow: /commented/
-
код 301
http://gifts.bookmarking.site/robots.txt
-
код 200
http://www.gifts.bookmarking.site/robots.txt