reuters.com robots.txt

Array
(
    [ai2bot] => Array
        (
        )

    [ai2bot-dolma] => Array
        (
        )

    [alibaba.security.heimdall] => Array
        (
        )

    [aliyunsecbot/aliyun] => Array
        (
        )

    [anthropic-ai] => Array
        (
        )

    [applebot-extended] => Array
        (
        )

    [archive.org_bot] => Array
        (
        )

    [bedrockbot] => Array
        (
        )

    [bytespider] => Array
        (
        )

    [ccbot] => Array
        (
        )

    [chatglm-spider] => Array
        (
        )

    [claude-searchbot] => Array
        (
        )

    [claude-web] => Array
        (
        )

    [claudebot] => Array
        (
        )

    [cloudflarebrowserrenderingcrawler] => Array
        (
        )

    [cohere-ai] => Array
        (
        )

    [cohere-training-data-crawler] => Array
        (
        )

    [crawlspace] => Array
        (
        )

    [deepseekbot] => Array
        (
        )

    [diffbot] => Array
        (
        )

    [facebookbot] => Array
        (
        )

    [feedfetcher-google] => Array
        (
        )

    [gemini-deep-research] => Array
        (
        )

    [google-cloudvertexbot] => Array
        (
        )

    [google-extended] => Array
        (
        )

    [googleother] => Array
        (
        )

    [grok] => Array
        (
        )

    [ia_archiver] => Array
        (
        )

    [ia_archiver-web.archive.org] => Array
        (
        )

    [img2dataset] => Array
        (
        )

    [mistralai-user] => Array
        (
        )

    [meta-externalagent] => Array
        (
        )

    [meta-externalfetcher] => Array
        (
        )

    [pangubot] => Array
        (
        )

    [perplexitybot] => Array
        (
        )

    [piplbot] => Array
        (
        )

    [timpibot] => Array
        (
        )

    [velenpublicwebcrawler] => Array
        (
        )

    [youbot] => Array
        (
            [disallow] => Array
                (
                    [0] => /
                )

        )

    [*] => Array
        (
            [disallow] => Array
                (
                    [0] => /finance/stocks/option
                    [1] => /finance/stocks/financialHighlights
                    [2] => /search
                    [3] => /site-search/
                    [4] => /beta
                    [5] => /designtech
                    [6] => /featured-optimize
                    [7] => /energy-test
                    [8] => /article/beta
                    [9] => /sponsored/previewcampaign
                    [10] => /sponsored/previewarticle
                    [11] => /test/
                    [12] => /news/archive/commentary
                    [13] => /brandfeatures/venture-capital
                    [14] => /assets/siteindex
                    [15] => /article/api/
                    [16] => /practical-law-the-journal/search/
                    [17] => /pf/api/
                    [18] => /fr/
                    [19] => /it/
                    [20] => /es/
                    [21] => /pt/
                    [22] => /de/
                    [23] => /account/subscribe/payment/
                )

            [sitemap] => Array
                (
                    [0] => https://www.reuters.com/arc/outboundfeeds/sitemap-index/?outputType=xml
                    [1] => https://www.reuters.com/arc/outboundfeeds/news-sitemap-index/?outputType=xml
                    [2] => https://www.reuters.com/plus/sitemap-index.xml
                    [3] => https://www.reuters.com/arc/outboundfeeds/sitemap-plj-index/?outputType=xml
                    [4] => https://www.reuters.com/graphics/sitemap.xml
                    [5] => https://www.reuters.com/arc/outboundfeeds/sitemap-index/pictures/?outputType=xml
                    [6] => https://www.reuters.com/static/video-sitemap/us/sitemap_video_index.xml
                    [7] => https://www.reuters.com/arc/outboundfeeds/topic-sitemap/?outputType=xml
                    [8] => https://www.reuters.com/arc/outboundfeeds/author-sitemap/?outputType=xml
                    [9] => https://www.reuters.com/arc/outboundfeeds/pressrelease-sitemap/?outputType=xml
                )

        )

)