# www.robotstxt.org/ # www.google.com/support/webmasters/bin/answer.py?hl=en&answer=156449 User-agent: SemrushBot Disallow: / User-agent: SemrushBot-SA Disallow: / User-agent: SemrushBot-BA Disallow: / User-agent: SemrushBot-SI Disallow: / User-agent: SemrushBot-SWA Disallow: / User-agent: SemrushBot-CT Disallow: / User-agent: SemrushBot-BM Disallow: / User-agent: AhrefsBot Disallow: / User-Agent: serpstatbot Disallow: / User-agent: SeznamBot disallow: / User-agent: VigLink disallow: / User-agent: DotBot disallow: / User-agent: MJ12bot disallow: / User-agent: SputnikBot Disallow: / User-agent: Sputnik Disallow: / User-agent: SputnikBot/2.3 Disallow: / User-agent: Curious George Disallow: / User-agent: Curious George - www.analyticsseo.com Disallow: / User-agent: Curious George - www.analyticsseo.com/crawler Disallow: / User-agent: http://site.ru Disallow: / User-agent: site.ru Disallow: / User-agent: AhrefsBot Disallow:/ User-agent: MauiBot Disallow:/ User-agent: SEOkicks Disallow:/ User-agent: BLEXBot Disallow: / User-agent: BUbiNG Disallow: / User-agent: ScoutJet Disallow: / User-agent: grapeshot Disallow: / User-agent: GrapeshotCrawler Disallow: / User-agent: Seekport Crawler Disallow: / User-agent: zoominfobot Disallow: / User-agent: Barkrowler Disallow: / User-agent: Datanyze Disallow: / User-agent: YisouSpider Disallow: / User-agent: adscanner Disallow: / User-agent: Genieo Disallow: / User-agent: Genieo/1.0 Disallow: / User-Agent: The Knowledge AI Disallow: / User-agent: Zgrab Disallow: / User-Agent: ZoominfoBot Disallow: / User-agent: sogou spider Disallow: / User-agent: Blekkobot Disallow: / User-agent: BLEXBot Disallow: / User-agent: Seobility Disallow: / User-agent: Baiduspider Disallow: / User-agent: Baiduspider/2.0 Disallow: / User-agent: Yandex Disallow: / User-agent: YandexBot/3.0 Disallow: / User-agent: IstellaBot Disallow: / User-agent: IstellaBot/1.01.18 Disallow: / User-agent: IstellaBot/1.01.18 +http://www.tiscali.it/ Disallow: / User-agent: IstellaBot/1.10.2 +http://www.tiscali.it/ Disallow: / User-agent: Mozilla/5.0 (compatible; IstellaBot/1.01.18 +http://www.tiscali.it/) Disallow: / User-agent: Cliqzbot Disallow: / User-agent: Sogou web spider/4.0 Disallow: / User-agent: GroupHigh Disallow: / User-agent: GroupHigh/1.0 Disallow: / User-agent: coccocbot-web Disallow: / User-agent: ccbot Disallow: / User-agent: Mail.Ru Disallow: / User-agent: grapeshotcrawler Disallow: / User-agent: ia_archiver Disallow: / User-agent: james bot Disallow: / User-agent: leikibot Disallow: / User-agent: libcurl Disallow: / User-agent: linkdexbot Disallow: / User-agent: lipperhey Disallow: / User-agent: livelap Disallow: / User-agent: lssrocket Disallow: / User-agent: magpie Disallow: / User-agent : Uptimebot Disallow: / User-agent: Gluten Free Crawler/1.0 Disallow: / User-agent : serpstatbot/1.0 Disallow: / User-agent: DomainCrawler Disallow: / User-agent: Steeler disallow: / User-agent: Steeler/3.5 disallow: / User-agent: Daum Disallow: / User-agent :arquivo-web-crawler Disallow: / User-agent: AwarioSmartBot Disallow: / User-agent: MojeekBot Disallow: / User-agent: Yeti Disallow: / User-agent: Yeti-Mobile Disallow: / User-agent :Mr.4x3 Powered Disallow: / User-agent: SJUUPBot Disallow: / User-agent: VigLink Disallow: / User-agent: Pi-Monster Disallow : / User-agent: tracemyfile/1.0 Disallow: / User-agent: Xenu's Link Sleuth 1.1c Disallow: / User-agent: oBot/2.3.1 Disallow: / User-agent: CowBot/1.0 Disallow: / User-agent: Deskyobot Disallow: / User-agent: Deskyobot/1.0 Disallow: / User-agent: ltx71+-+(http://ltx71.com/) Disallow: / User-agent: CCBot/2.0 Disallow: / User-agent: SurdotlyBot Disallow: /