nike.com robots.txt
Nike.com has robots.txt file. Http response status is 200.
Robots size is small. It is about 435 kb and 100 rows.
nike.com has great amount of sitemap files — 7: https://www.nike.com/sitemap-us-help.xml, https://www.nike.com/sitemap-v2-landingpage-index.xml, https://www.nike.com/sitemap-v2-pdp-index.xml and others. Robots.txt for User-Agent=* contains only disallow directives.
Host directive is not presented for domain nike.com in robots.txt file.
Robots size is small. It is about 435 kb and 100 rows.
nike.com has great amount of sitemap files — 7: https://www.nike.com/sitemap-us-help.xml, https://www.nike.com/sitemap-v2-landingpage-index.xml, https://www.nike.com/sitemap-v2-pdp-index.xml and others. Robots.txt for User-Agent=* contains only disallow directives.
Host directive is not presented for domain nike.com in robots.txt file.
Array
(
[*] => Array
(
[disallow] => Array
(
[0] => */member/inbox
[1] => */member/settings
[2] => */p/
[3] => */checkout/
[4] => /*.swf$
[5] => /*.pdf$
[6] => /pdf/
[7] => /ar/help/
[8] => /br/help/
[9] => /hk/help/
[10] => /uy/help/
[11] => /xf/help/
[12] => /xl/help/
[13] => /xm/help/
[14] => /fragments/recommendations-carousel
[15] => /kr/en$
[16] => /kr/en/
)
[sitemap] => Array
(
[0] => https://www.nike.com/sitemap-us-help.xml
[1] => https://www.nike.com/sitemap-v2-landingpage-index.xml
[2] => https://www.nike.com/sitemap-v2-pdp-index.xml
[3] => https://www.nike.com/sitemap-v2-snkrsweb-index.xml
[4] => https://www.nike.com/sitemap-v2-gridwall-index.xml
[5] => https://www.nike.com/sitemap-v2-article-index.xml
[6] => https://www.nike.com/sitemap-locator-index.xml
)
)
[baiduspider] => Array
(
[allow] => Array
(
[0] => /cn$
[1] => /cn/
[2] => /CN$
[3] => /CN/
[4] => /assets/
[5] => /static/
[6] => /styleguide/
)
[disallow] => Array
(
[0] => */w?q=
[1] => */w/?q=
[2] => /
)
)
[haosouspider] => Array
(
[allow] => Array
(
[0] => /cn$
[1] => /cn/
[2] => /CN$
[3] => /CN/
[4] => /assets/
[5] => /static/
[6] => /styleguide/
)
[disallow] => Array
(
[0] => */w?q=
[1] => */w/?q=
[2] => /
)
)
[sogou web spider] => Array
(
[allow] => Array
(
[0] => /cn$
[1] => /cn/
[2] => /CN$
[3] => /CN/
[4] => /assets/
[5] => /static/
[6] => /styleguide/
)
[disallow] => Array
(
[0] => */w?q=
[1] => */w/?q=
[2] => /
)
)
[sogou inst spider] => Array
(
[allow] => Array
(
[0] => /cn$
[1] => /cn/
[2] => /CN$
[3] => /CN/
[4] => /assets/
[5] => /static/
[6] => /styleguide/
)
[disallow] => Array
(
[0] => */w?q=
[1] => */w/?q=
[2] => /
)
)
[sogou spider2] => Array
(
[allow] => Array
(
[0] => /cn$
[1] => /cn/
[2] => /CN$
[3] => /CN/
[4] => /assets/
[5] => /static/
[6] => /styleguide/
)
[disallow] => Array
(
[0] => */w?q=
[1] => */w/?q=
[2] => /
)
)
)
# www.nike.com robots.txt -- just crawl it. User-agent: * Disallow: */member/inbox Disallow: */member/settings Disallow: */p/ Disallow: */checkout/ Disallow: /*.swf$ Disallow: /*.pdf$ Disallow: /pdf/ Disallow: /ar/help/ Disallow: /br/help/ Disallow: /hk/help/ Disallow: /uy/help/ Disallow: /xf/help/ Disallow: /xl/help/ Disallow: /xm/help/ Disallow: /fragments/recommendations-carousel Disallow: /kr/en$ Disallow: /kr/en/ User-agent: Baiduspider Allow: /cn$ Allow: /cn/ Allow: /CN$ Allow: /CN/ Allow: /assets/ Allow: /static/ Allow: /styleguide/ Disallow: */w?q= Disallow: */w/?q= Disallow: / User-agent: HaoSouSpider Allow: /cn$ Allow: /cn/ Allow: /CN$ Allow: /CN/ Allow: /assets/ Allow: /static/ Allow: /styleguide/ Disallow: */w?q= Disallow: */w/?q= Disallow: / User-agent: Sogou web spider Allow: /cn$ Allow: /cn/ Allow: /CN$ Allow: /CN/ Allow: /assets/ Allow: /static/ Allow: /styleguide/ Disallow: */w?q= Disallow: */w/?q= Disallow: / User-agent: Sogou inst spider Allow: /cn$ Allow: /cn/ Allow: /CN$ Allow: /CN/ Allow: /assets/ Allow: /static/ Allow: /styleguide/ Disallow: */w?q= Disallow: */w/?q= Disallow: / User-agent: Sogou spider2 Allow: /cn$ Allow: /cn/ Allow: /CN$ Allow: /CN/ Allow: /assets/ Allow: /static/ Allow: /styleguide/ Disallow: */w?q= Disallow: */w/?q= Disallow: / Sitemap: https://www.nike.com/sitemap-us-help.xml Sitemap: https://www.nike.com/sitemap-v2-landingpage-index.xml Sitemap: https://www.nike.com/sitemap-v2-pdp-index.xml Sitemap: https://www.nike.com/sitemap-v2-snkrsweb-index.xml Sitemap: https://www.nike.com/sitemap-v2-gridwall-index.xml Sitemap: https://www.nike.com/sitemap-v2-article-index.xml Sitemap: https://www.nike.com/sitemap-locator-index.xml # # `` ```.` # `+/ ``.-/+o+:-. # `/mo ``.-:+syhdhs/-` # -hMd `..:+oyhmNNmds/-` # `oNMM/ ``.-/oyhdmMMMMNdy+:. # .hMMMM- `.-/+shdmNMMMMMMNdy+:. # :mMMMMM+ `.-:+sydmNMMMMMMMMMNmho:.` # :NMMMMMMN: `.-:/oyhmmNMMMMMMMMMMMNmho:.` # .NMMMMMMMMNy:` `.-/oshdmNMMMMMMMMMMMMMMMmhs/-` # hMMMMMMMMMMMMmhysooosyhdmNMMMMMMMMMMMMMMMMMMmds/-` # .MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMNdy+-.` # -MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMNdy+-.` # `NMMMMMMMMMMMMMMMMMMMMMMMMMMMMMmyo:.` # /NMMMMMMMMMMMMMMMMMMMMMMMmho:.` # .yNMMMMMMMMMMMMMMMMmhs/.` # ./shdmNNmmdhyo/-`` # `````
- код 301 http://nike.com/robots.txt
- код 200 https://www.nike.com/robots.txt