Mirror.co.uk has robots.txt file. Http response status is 200.
Robots size is small. It is about 238 kb and 71 rows.

Robots.txt for User-Agent=* contains only disallow directives.

Host directive is not presented for domain mirror.co.uk in robots.txt file.
Array
(
    [*] => Array
        (
            [sitemap] => Array
                (
                    [0] => https://www.mirror.co.uk/map_news.xml
                    [1] => https://www.mirror.co.uk/sitemaps/sitemap_index.xml
                )

            [disallow] => Array
                (
                    [0] => /topics/*
                    [1] => /topics
                    [2] => /*token=*
                    [3] => /search/
                    [4] => /comm-part-test/
                    [5] => /*service=ajax
                    [6] => /centenary-fund/
                    [7] => /3am/weird-celeb-news/xxx-1341448
                    [8] => /3am/weird-celeb-news/tamara-ecclestone-watched-boyfriends-sex-1341448
                    [9] => /comm-part-test/oh-like-beside-seaside-uk-1722203
                    [10] => /resources/js/s_code.js
                    [11] => /template/
                    [12] => /tv/tv-news/jean-alexander-dies-coronation-streets-3764827
                    [13] => /3am/celebrity-news/anne-kirkbride-dead-bill-roache-5013775
                    [14] => /lifestyle/cartoons/andy-capp/andy-capp
                    [15] => /lifestyle/cartoons/the-gag-vault/gag-vault
                    [16] => /lifestyle/cartoons/perishers/perishers
                    [17] => /lifestyle/cartoons/horace/horace
                    [18] => /lifestyle/cartoons/garth/garth
                    [19] => /lifestyle/cartoons/mandy/mandy
                    [20] => /lifestyle/cartoons/kerber-black/
                    [21] => /regression-test-home/
                    [22] => /5293/
                    [23] => /exclusive-offers/
                )

            [crawl-delay] => 10
        )

    [daumoa] => Array
        (
            [disallow] => Array
                (
                    [0] => /
                )

        )

    [sosospider] => Array
        (
            [disallow] => Array
                (
                    [0] => /
                )

        )

    [rogerbot] => Array
        (
            [disallow] => Array
                (
                    [0] => /
                )

        )

    [semetrical] => Array
        (
            [disallow] => Array
                (
                    [0] => /
                )

        )

    [googlebot-news] => Array
        (
            [disallow] => Array
                (
                    [0] => /thedavedesk/
                )

        )

    [ahrefsbot] => Array
        (
            [disallow] => Array
                (
                    [0] => /
                )

        )

    [gptbot] => Array
        (
            [disallow] => Array
                (
                    [0] => /
                )

        )

    [oai-searchbot] => Array
        (
            [disallow] => Array
                (
                    [0] => /
                )

        )

    [applebot-extended] => Array
        (
            [disallow] => Array
                (
                    [0] => /
                )

        )

    [perplexitybot] => Array
        (
            [disallow] => Array
                (
                    [0] => /
                )

        )

    [perplexity-ai] => Array
        (
            [disallow] => Array
                (
                    [0] => /
                )

        )

    [ccbot] => Array
        (
            [disallow] => Array
                (
                    [0] => /
                )

        )

    [youbot] => Array
        (
            [disallow] => Array
                (
                    [0] => /
                )

        )

    [claudebot] => Array
        (
            [disallow] => Array
                (
                    [0] => /
                )

        )

    [anthropic-ai] => Array
        (
            [disallow] => Array
                (
                    [0] => /
                )

        )

    [claude-web] => Array
        (
            [disallow] => Array
                (
                    [0] => /
                )

        )

    [grapeshot] => Array
        (
        )

    [bingbot] => Array
        (
            [crawl-delay] => 1
        )

    [amazonadbot] => Array
        (
        )

    [ozone] => Array
        (
        )

    [meta-externalagent] => Array
        (
            [disallow] => Array
                (
                    [0] => /
                )

        )

)
Sitemap: https://www.mirror.co.uk/map_news.xml
Sitemap: https://www.mirror.co.uk/sitemaps/sitemap_index.xml
User-agent: *
Disallow: /topics/*
Disallow: /topics
Disallow: /*token=*
Crawl-delay: 10.0

Disallow: /search/
Disallow: /comm-part-test/
Disallow: /*service=ajax
Disallow: /centenary-fund/
Disallow: /3am/weird-celeb-news/xxx-1341448
Disallow: /3am/weird-celeb-news/tamara-ecclestone-watched-boyfriends-sex-1341448
Disallow: /comm-part-test/oh-like-beside-seaside-uk-1722203
Disallow: /resources/js/s_code.js
Disallow: /template/
Disallow: /tv/tv-news/jean-alexander-dies-coronation-streets-3764827
Disallow: /3am/celebrity-news/anne-kirkbride-dead-bill-roache-5013775
Disallow: /lifestyle/cartoons/andy-capp/andy-capp
Disallow: /lifestyle/cartoons/the-gag-vault/gag-vault
Disallow: /lifestyle/cartoons/perishers/perishers
Disallow: /lifestyle/cartoons/horace/horace
Disallow: /lifestyle/cartoons/garth/garth
Disallow: /lifestyle/cartoons/mandy/mandy
Disallow: /lifestyle/cartoons/kerber-black/
Disallow: /regression-test-home/
Disallow: /5293/
Disallow: /exclusive-offers/

#Agent Specific Disallowed Sections

User-agent: daumoa
Disallow: /

User-agent: Sosospider
Disallow: /

User-agent: rogerbot
Disallow: /

User-agent: semetrical
Disallow: /

User-agent: Googlebot-News
Disallow: /thedavedesk/

User-agent: AhrefsBot 
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Perplexity-ai
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: grapeshot
Crawl-delay: 0

User-agent: bingbot
Crawl-delay: 1

User-agent: AmazonAdBot
Crawl-delay: 0

User-agent: ozone
Crawl-delay: 0

User-agent: Meta-ExternalAgent
Disallow: /
  • код 301 http://mirror.co.uk/robots.txt
  • код 200 https://www.mirror.co.uk/robots.txt