Smartpetrol.ru has robots.txt file. Http response code is 200.
Robots size is small. It is about 184 kb and 75 rows.

smartpetrol.ru has sitemap file https://smartpetrol.ru/sitemap.xml. Main host for smartpetrol.ru is https://smartpetrol.ru.

smartpetrol.ru has separate rules for googlebot and yandexbot and also has rules for all bots (User-Agent=*).
Array
(
    [*] => Array
        (
            [disallow] => Array
                (
                    [0] => /wp-
                    [1] => /wp/
                    [2] => /wp-login.php
                    [3] => /wp-register.php
                    [4] => *?s=
                    [5] => *&s=
                    [6] => */embed
                    [7] => */feed
                    [8] => /cgi-bin
                    [9] => /?
                    [10] => /team/*
                    [11] => /testimonials/*
                    [12] => /xmlrpc.php
                    [13] => *utm*=
                    [14] => *openstat=
                )

            [allow] => Array
                (
                    [0] => /wp-admin/admin-ajax.php
                    [1] => */uploads
                )

            [sitemap] => Array
                (
                    [0] => https://smartpetrol.ru/sitemap.xml
                )

            [host] => https://smartpetrol.ru
        )

    [googlebot] => Array
        (
            [disallow] => Array
                (
                    [0] => /cgi-bin
                    [1] => /?
                    [2] => /wp-
                    [3] => /wp/
                    [4] => *?s=
                    [5] => *&s=
                    [6] => /author/
                    [7] => /users/
                    [8] => */trackback
                    [9] => */feed
                    [10] => */rss
                    [11] => */embed
                    [12] => */wlwmanifest.xml
                    [13] => /xmlrpc.php
                    [14] => *utm*=
                    [15] => *openstat=
                    [16] => /team/*
                    [17] => /testimonials/*
                )

            [allow] => Array
                (
                    [0] => */uploads
                    [1] => /*/*.js
                    [2] => /*/*.css
                    [3] => /wp-*.png
                    [4] => /wp-*.jpg
                    [5] => /wp-*.jpeg
                    [6] => /wp-*.gif
                    [7] => /wp-admin/admin-ajax.php
                )

        )

    [yandex] => Array
        (
            [disallow] => Array
                (
                    [0] => /cgi-bin
                    [1] => /?
                    [2] => /wp-
                    [3] => /wp/
                    [4] => *?s=
                    [5] => *&s=
                    [6] => /author/
                    [7] => /users/
                    [8] => */trackback
                    [9] => */feed
                    [10] => */rss
                    [11] => */embed
                    [12] => */wlwmanifest.xml
                    [13] => /xmlrpc.php
                    [14] => /team/*
                    [15] => /testimonials/*
                )

            [allow] => Array
                (
                    [0] => */uploads
                    [1] => /*/*.js
                    [2] => /*/*.css
                    [3] => /wp-*.png
                    [4] => /wp-*.jpg
                    [5] => /wp-*.jpeg
                    [6] => /wp-*.gif
                    [7] => /wp-admin/admin-ajax.php
                )

        )

)
User-agent: *
Disallow: /wp-
Disallow: /wp/
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: *?s=
Disallow: *&s=
Disallow: */embed
Disallow: */feed
Disallow: /cgi-bin
Disallow: /?
Disallow: /team/*
Disallow: /testimonials/*
Disallow: /xmlrpc.php
Disallow: *utm*=
Disallow: *openstat= 
Allow: /wp-admin/admin-ajax.php
Allow: */uploads

User-agent: GoogleBot       
Disallow: /cgi-bin
Disallow: /?
Disallow: /wp-
Disallow: /wp/
Disallow: *?s=
Disallow: *&s=
Disallow: /author/
Disallow: /users/
Disallow: */trackback
Disallow: */feed
Disallow: */rss
Disallow: */embed
Disallow: */wlwmanifest.xml
Disallow: /xmlrpc.php
Disallow: *utm*=
Disallow: *openstat=
Disallow: /team/*
Disallow: /testimonials/*
Allow: */uploads
Allow: /*/*.js 
Allow: /*/*.css
Allow: /wp-*.png
Allow: /wp-*.jpg
Allow: /wp-*.jpeg
Allow: /wp-*.gif 
Allow: /wp-admin/admin-ajax.php 

User-agent: Yandex
Disallow: /cgi-bin
Disallow: /?
Disallow: /wp-
Disallow: /wp/
Disallow: *?s=
Disallow: *&s=
Disallow: /author/
Disallow: /users/
Disallow: */trackback
Disallow: */feed
Disallow: */rss
Disallow: */embed
Disallow: */wlwmanifest.xml
Disallow: /xmlrpc.php
Disallow: /team/*
Disallow: /testimonials/*
Allow: */uploads
Allow: /*/*.js
Allow: /*/*.css
Allow: /wp-*.png
Allow: /wp-*.jpg
Allow: /wp-*.jpeg
Allow: /wp-*.gif
Allow: /wp-admin/admin-ajax.php

Sitemap: https://smartpetrol.ru/sitemap.xml
Host: https://smartpetrol.ru
  • код 301 http://smartpetrol.ru/robots.txt
  • код 200 https://smartpetrol.ru/robots.txt