Newsru.co.il has robots.txt file. Http response code is 200.
Robots size is small. It is about 66 kb and 30 rows.

Main host for newsru.co.il is doska.newsru.co.il.

Array
(
    [yandexnews] => Array
        (
            [allow] => Array
                (
                    [0] => /bin/export/yandex/
                    [1] => /pict/
                )

            [disallow] => Array
                (
                    [0] => /
                )

            [crawl-delay] => 2
        )

    [ia_archiver] => Array
        (
            [disallow] => Array
                (
                    [0] => /
                )

        )

    [*] => Array
        (
            [disallow] => Array
                (
                    [0] => /search
                    [1] => /bin
                    [2] => /cgi
                    [3] => /css
                    [4] => /img
                    [5] => /plain
                    [6] => /*_save.html$
                    [7] => /*_blog.html$
                    [8] => /*_print.html$
                    [9] => /*_email.html$
                )

            [crawl-delay] => 2
            [host] => doska.newsru.co.il
            [allow] => Array
                (
                    [0] => /ads.txt
                )

        )

)
User-agent: YandexNews
Allow: /bin/export/yandex/
Allow: /pict/
Disallow: /
Crawl-delay: 2

User-agent: ia_archiver
Disallow: /

User-agent: *
Disallow: /search
Disallow: /bin
Disallow: /cgi
Disallow: /css
Disallow: /img
Disallow: /plain
Disallow: /*_save.html$
Disallow: /*_blog.html$
Disallow: /*_print.html$
Disallow: /*_email.html$
Crawl-delay: 2

User-Agent: *
Disallow:
Host: doska.newsru.co.il

User-Agent: *
Disallow:
Host: dxt.newsru.co.il

User-Agent: *	
Disallow:
Host: old.newsru.co.il

User-agent: *
Allow: /ads.txt
  • код 200 http://newsru.co.il/robots.txt