Alfa.lt has robots.txt file. Http response code is 200.
Robots size is small. It is about 110 kb and 37 rows.

Robots.txt for User-Agent=* contains only disallow directives.

Host directive is not presented for domain alfa.lt in robots.txt file.
Array
(
    [*] => Array
        (
            [crawl-delay] => 10
            [disallow] => Array
                (
                    [0] => /adClick
                    [1] => /stat
                    [2] => /getMediaFile.do
                    [3] => /print
                    [4] => /printLocation
                    [5] => /printEvent
                    [6] => /emailArticle
                    [7] => /emailVideo
                    [8] => /emailEvent
                    [9] => /emailLocation
                    [10] => /statistics.do
                    [11] => /rubricExport.do
                    [12] => /subscriptionEmail.do
                    [13] => /galleryDataExporter
                    [14] => /komentarai/videoSou
                    [15] => /Lifestyles/dalyves/
                    [16] => /forumas
                    [17] => /*pageMapName
                    [18] => /*pageMapName*
                    [19] => /*feed/*
                    [20] => /article/bump/
                    [21] => /adocean/meta/
                )

        )

    [facebookexternalhit] => Array
        (
            [crawl-delay] => 1
        )

    [googlebot/2.1] => Array
        (
            [crawl-delay] => 1
        )

    [yandex] => Array
        (
            [crawl-delay] => 2
        )

    [exabot] => Array
        (
            [crawl-delay] => 10
        )

    [sentibot] => Array
        (
            [disallow] => Array
                (
                    [0] => /
                )

        )

    [sentibot www.sentibot.eu (compatible with googlebot)] => Array
        (
            [disallow] => Array
                (
                    [0] => /
                )

        )

)
#Sitemap: http://www.alfa.lt/sitemap.xml
User-Agent: *
Crawl-delay: 10
Disallow: /adClick
Disallow: /stat
Disallow: /getMediaFile.do
Disallow: /print
Disallow: /printLocation
Disallow: /printEvent
Disallow: /emailArticle
Disallow: /emailVideo
Disallow: /emailEvent
Disallow: /emailLocation
Disallow: /statistics.do
Disallow: /rubricExport.do
Disallow: /subscriptionEmail.do
Disallow: /galleryDataExporter
Disallow: /komentarai/videoSou
Disallow: /Lifestyles/dalyves/
Disallow: /forumas 
Disallow: /*pageMapName
Disallow: /*pageMapName*
Disallow: /*feed/*
Disallow: /article/bump/
Disallow: /adocean/meta/

User-agent: facebookexternalhit
Crawl-delay: 1

User-agent: Googlebot/2.1
Crawl-delay: 1

User-agent: Yandex
Crawl-delay: 2

User-agent: Exabot
Crawl-delay: 10

User-agent: SentiBot
Disallow: /

User-agent: SentiBot www.sentibot.eu (compatible with Googlebot)
Disallow: /
  • код 301 http://alfa.lt/robots.txt
  • код 200 https://www.alfa.lt/robots.txt