Array
(
[*] => Array
(
[disallow] => Array
(
[0] => /lithe/
)
[crawl-delay] => 5
)
)
User-agent: *
Disallow: /lithe/
Crawl-delay: 5
-
код 301
http://ptv.vic.gov.au/robots.txt
-
код 301
http://www.ptv.vic.gov.au/robots.txt
-
код 200
https://www.ptv.vic.gov.au/robots.txt