Dailytv.ru has robots.txt file. Http response code is 200.
Robots size is small. It is about 70 kb and 25 rows.

Main host for dailytv.ru is https://dailytv.ru/.

dailytv.ru has separate rules for googlebot and yandexbot and also has rules for all bots (User-Agent=*).
Array
(
    [yandex] => Array
        (
            [crawl-delay] => 10
            [disallow] => Array
                (
                    [0] => /admin/
                    [1] => /relatedclips.php
                    [2] => /fpembed.php
                    [3] => /embed.php
                    [4] => /embed/
                )

        )

    [googlebot] => Array
        (
            [allow] => Array
                (
                    [0] => /*.js*
                    [1] => /*.css*
                    [2] => /*.png*
                    [3] => /*.jpg*
                    [4] => /*.gif*
                    [5] => /templates/*.css
                    [6] => /templates/*.tpl
                    [7] => /templates/*.js
                )

            [disallow] => Array
                (
                    [0] => /admin/
                    [1] => /relatedclips.php
                    [2] => /fpembed.php
                    [3] => /embed.php
                    [4] => /embed/
                )

        )

    [*] => Array
        (
            [host] => https://dailytv.ru/
            [sitemap] => Array
                (
                    [0] => https://dailytv.ru/video-sitemap-index.xml
                    [1] => https://dailytv.ru/sitemap-index.xml
                )

        )

)
User-agent: Yandex
Crawl-delay: 10

User-agent: Yandex
Disallow: /admin/
Disallow: /relatedclips.php
Disallow: /fpembed.php
Disallow: /embed.php
Disallow: /embed/

User-agent: Googlebot
Allow: /*.js*
Allow: /*.css*
Allow: /*.png*
Allow: /*.jpg*
Allow: /*.gif*
Allow: /templates/*.css
Allow: /templates/*.tpl
Allow: /templates/*.js
Disallow: /admin/
Disallow: /relatedclips.php
Disallow: /fpembed.php
Disallow: /embed.php
Disallow: /embed/

Host: https://dailytv.ru/
Sitemap: https://dailytv.ru/video-sitemap-index.xml
Sitemap: https://dailytv.ru/sitemap-index.xml
  • код 301 http://dailytv.ru/robots.txt
  • код 200 https://dailytv.ru/robots.txt