Tweakers.net has robots.txt file. Http response status is 200.
Robots size is small. It is about 333 kb and 102 rows.

tweakers.net has great amount of sitemap files — 11: https://tweakers.net/nieuws/news.xml, https://tweakers.net/nieuws/sitemap.xml, https://tweakers.net/plan/sitemap.xml and others. Robots.txt for User-Agent=* contains only disallow directives.

Host directive is not presented for domain tweakers.net in robots.txt file.
Array
(
    [fasterfox] => Array
        (
            [disallow] => Array
                (
                    [0] => /
                )

        )

    [008] => Array
        (
            [disallow] => Array
                (
                    [0] => /
                )

        )

    [mj12bot] => Array
        (
            [disallow] => Array
                (
                    [0] => /
                )

        )

    [exabot] => Array
        (
            [disallow] => Array
                (
                    [0] => /
                )

        )

    [proximic] => Array
        (
            [disallow] => Array
                (
                    [0] => /
                )

        )

    [youdaobot] => Array
        (
            [disallow] => Array
                (
                    [0] => /
                )

        )

    [*] => Array
        (
            [sitemap] => Array
                (
                    [0] => https://tweakers.net/nieuws/news.xml
                    [1] => https://tweakers.net/nieuws/sitemap.xml
                    [2] => https://tweakers.net/plan/sitemap.xml
                    [3] => https://tweakers.net/downloads/sitemap.xml
                    [4] => https://tweakers.net/reviews/sitemap.xml
                    [5] => https://tweakers.net/video/sitemap.xml
                    [6] => https://tweakers.net/pricewatch/sitemap.xml
                    [7] => https://tweakers.net/product/sitemap.xml
                    [8] => https://tweakers.net/productimage/sitemap.xml
                    [9] => https://tweakers.net/productreview/sitemap.xml
                    [10] => https://tweakers.net/carriere/sitemap.xml
                )

            [disallow] => Array
                (
                    [0] => /7455/tweakers
                    [1] => /7594/tweakers-be
                    [2] => /aanbod/admanager
                    [3] => /aanbod/beoordeel
                    [4] => /aanbod/bookmarks
                    [5] => /aanbod/cat/user
                    [6] => /aanbod/user
                    [7] => /aanbod/rating
                    [8] => /aanbod/edit_bod
                    [9] => /aanbod/meldpunt
                    [10] => /aanbod/reageer
                    [11] => /aanbod/placeRating
                    [12] => /aanbod/editRating
                    [13] => /aanbod/reactivateAd
                    [14] => /aanbod/deleteBid
                    [15] => /aanbod/placeBid
                    [16] => /aanbod/indebuurt
                    [17] => /aanbod/search
                    [18] => /aanbod/search_results
                    [19] => /aanbod/zoeken
                    [20] => /aanbod/nieuw
                    [21] => /aanbod/wijzigen
                    [22] => /aboshop/*
                    [23] => /abonnementen/*
                    [24] => /ajax
                    [25] => /app.php
                    [26] => /camo
                    [27] => /captcha
                    [28] => /csp-report
                    [29] => /devicedetect
                    [30] => /ext/logincheck
                    [31] => /ext/lt
                    [32] => /ext/pricewatch
                    [33] => /ext/reageer
                    [34] => /external
                    [35] => /gallery
                    [36] => /images
                    [37] => /instellingen
                    [38] => /my.tnet
                    [39] => /nieuws/dag
                    [40] => /notificatie
                    [41] => /pricewatch/addproduct
                    [42] => /pricewatch/compare
                    [43] => /pricewatch/improve
                    [44] => /pricewatch/zoeken
                    [45] => /pricewatch/bestelkosten
                    [46] => /pricewatch/winkelmand_rml
                    [47] => /pricewatch/unsorted/zoeken
                    [48] => /productcollectie
                    [49] => /productsurvey/manager
                    [50] => /productsurvey/user
                    [51] => /productreview/manager
                    [52] => /productreview/user
                    [53] => /pwlev
                    [54] => /shopreview/*/vergelijken/
                    [55] => /shopreview/schrijven
                    [56] => /shopreview/wijzigen
                    [57] => /shopreview/reageer
                    [58] => /shopreview/wijzig-reactie
                    [59] => /reacties
                    [60] => /search
                    [61] => /shopwidget
                    [62] => /stats
                    [63] => /submit
                    [64] => /xmlhttp
                    [65] => /wot
                    [66] => /zoeken
                )

        )

    [googlebot-news] => Array
        (
            [disallow] => Array
                (
                    [0] => /*
                )

            [allow] => Array
                (
                    [0] => /$
                    [1] => /nieuws
                    [2] => /geek
                    [3] => /redactieblogs
                    [4] => /reviews
                    [5] => /downloads
                    [6] => /plan
                    [7] => /feeds
                    [8] => /g/api
                )

        )

)
User-agent: Fasterfox
Disallow: /

User-Agent: 008
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: proximic
Disallow: /

User-Agent: YoudaoBot
Disallow: /

User-agent: *
Sitemap: https://tweakers.net/nieuws/news.xml
Sitemap: https://tweakers.net/nieuws/sitemap.xml
Sitemap: https://tweakers.net/plan/sitemap.xml
Sitemap: https://tweakers.net/downloads/sitemap.xml
Sitemap: https://tweakers.net/reviews/sitemap.xml
Sitemap: https://tweakers.net/video/sitemap.xml
Sitemap: https://tweakers.net/pricewatch/sitemap.xml
Sitemap: https://tweakers.net/product/sitemap.xml
Sitemap: https://tweakers.net/productimage/sitemap.xml
Sitemap: https://tweakers.net/productreview/sitemap.xml
Sitemap: https://tweakers.net/carriere/sitemap.xml
Disallow: /7455/tweakers
Disallow: /7594/tweakers-be
Disallow: /aanbod/admanager
Disallow: /aanbod/beoordeel
Disallow: /aanbod/bookmarks
Disallow: /aanbod/cat/user
Disallow: /aanbod/user
Disallow: /aanbod/rating
Disallow: /aanbod/edit_bod
Disallow: /aanbod/meldpunt
Disallow: /aanbod/reageer
Disallow: /aanbod/placeRating
Disallow: /aanbod/editRating
Disallow: /aanbod/reactivateAd
Disallow: /aanbod/deleteBid
Disallow: /aanbod/placeBid
Disallow: /aanbod/indebuurt
Disallow: /aanbod/search
Disallow: /aanbod/search_results
Disallow: /aanbod/zoeken
Disallow: /aanbod/nieuw
Disallow: /aanbod/wijzigen
Disallow: /aboshop/*
Disallow: /abonnementen/*
Disallow: /ajax
Disallow: /app.php
Disallow: /camo
Disallow: /captcha
Disallow: /csp-report
Disallow: /devicedetect
Disallow: /ext/logincheck
Disallow: /ext/lt
Disallow: /ext/pricewatch
Disallow: /ext/reageer
Disallow: /external
Disallow: /gallery
Disallow: /images
Disallow: /instellingen
Disallow: /my.tnet
Disallow: /nieuws/dag
Disallow: /notificatie
Disallow: /pricewatch/addproduct
Disallow: /pricewatch/compare
Disallow: /pricewatch/improve
Disallow: /pricewatch/zoeken
Disallow: /pricewatch/bestelkosten
Disallow: /pricewatch/winkelmand_rml
Disallow: /pricewatch/unsorted/zoeken
Disallow: /productcollectie
Disallow: /productsurvey/manager
Disallow: /productsurvey/user
Disallow: /productreview/manager
Disallow: /productreview/user
Disallow: /pwlev
Disallow: /shopreview/*/vergelijken/
Disallow: /shopreview/schrijven
Disallow: /shopreview/wijzigen
Disallow: /shopreview/reageer
Disallow: /shopreview/wijzig-reactie
Disallow: /reacties
Disallow: /search
Disallow: /shopwidget
Disallow: /stats
Disallow: /submit
Disallow: /xmlhttp
Disallow: /wot
Disallow: /zoeken

User-agent: Googlebot-News
Disallow: /*
Allow: /$
Allow: /nieuws
Allow: /geek
Allow: /redactieblogs
Allow: /reviews
Allow: /downloads
Allow: /plan
Allow: /feeds
Allow: /g/api
  • код 301 http://tweakers.net/robots.txt
  • код 200 https://tweakers.net/robots.txt