User-agent: !Susie (http://www.sync2it.com/susie)   
Disallow: /
User-agent: ( Robots.txt Validator http://www.searchengineworld.com/cgi-bin/robotcheck.cgi )  
Disallow: /
User-agent: (DreamPassport/3.0; isao/MyDiGiRabi)  
Disallow: /
User-agent: (Privoxy/1.0)  
Disallow: /
User-agent: */Nutch-0.9-dev  
Disallow: /
User-agent: +SitiDi.net/SitiDiBot/1.0 (+Have Good Day)  
Disallow: /
User-agent: -DIE-KRAEHE- META-SEARCH-ENGINE/1.1 http://www.die-kraehe.de  
Disallow: /
User-agent: 123spider-Bot (Version: 1.02, powered by www.123spider.de  
Disallow: /
User-agent: 192.comAgent  
Disallow: /
User-agent: 1st ZipCommander (Net) - http://www.zipcommander.com/  
Disallow: /
User-agent: 2Bone_LinkChecker/1.0 libwww-perl/5.64  
Disallow: /
User-agent: 4anything.com LinkChecker v2.0  
Disallow: /
User-agent: 8484 Boston Project v 1.0  
Disallow: /
User-agent: :robot/1.0 (linux) ( admin e-mail: undefined http://www.neofonie.de/loesungen/search/robot.html )  
Disallow: /
User-agent: <a href='http://www.unchaos.com/'> UnChaos </a> From Chaos To Order Hybrid Web Search Engine.(vadim_gonchar@unchaos.com)  
Disallow: /
User-agent: <a href='http://www.unchaos.com/'> UnChaos Bot Hybrid Web Search Engine. </a> (vadim_gonchar@unchaos.com)  
Disallow: /
User-agent: <b> UnChaosBot From Chaos To Order UnChaos Hybrid Web Search Engine at www.unchaos.com </b> (info@unchaos.com)  
Disallow: /
User-agent: <http://www.sygol.com/> http://www.sygol.com  
Disallow: /
User-agent: A-Online Search  
Disallow: /
User-agent: A1 Keyword Research/1.0.2 (+http://www.micro-sys.dk/products/keyword-research/) miggibot/2007.03.27  
Disallow: /
User-agent: A1 Sitemap Generator/1.0 (+http://www.micro-sys.dk/products/sitemap-generator/) miggibot/2006.01.24  
Disallow: /
User-agent: ABCdatos BotLink/5.xx.xxx#BBL  
Disallow: /
User-agent: AESOP_com_SpiderMan  
Disallow: /
User-agent: AIBOT/2.1 By +(www.21seek.com A Real artificial intelligence search engine China)  
Disallow: /
User-agent: ANTFresco/x.xx  
Disallow: /
User-agent: ASAHA Search Engine Turkey V.001 (http://www.asaha.com/)  
Disallow: /
User-agent: ASPSeek/1.2.5  
Disallow: /
User-agent: ASPSeek/1.2.x  
Disallow: /
User-agent: ASPSeek/1.2.xa  
Disallow: /
User-agent: ASPSeek/1.2.xxpre  
Disallow: /
User-agent: ASPseek/1.2.9d  
Disallow: /
User-agent: ASPseek/1.2.xx  
Disallow: /
User-agent: ASSORT/0.10  
Disallow: /
User-agent: AU-MIC/2.0 MMP/2.0  
Disallow: /
User-agent: AUDIOVOX-SMT5600  
Disallow: /
User-agent: AV Fetch 1.0  
Disallow: /
User-agent: AVSearch-1.0(peter.turney@nrc.ca)  
Disallow: /
User-agent: AVSearch-2.0-fusionIdx-14-CompetitorWebSites  
Disallow: /
User-agent: AVSearch-3.0(AltaVista/AVC)  
Disallow: /
User-agent: AWeb  
Disallow: /
User-agent: AbachoBOT  
Disallow: /
User-agent: AbachoBOT (Mozilla compatible)  
Disallow: /
User-agent: Aberja Checkomat  
Disallow: /
User-agent: About/0.1libwww-perl/5.47  
Disallow: /
User-agent: Accelatech RSSCrawler/0.4  
Disallow: /
User-agent: Accoona-AI-Agent/1.1.1 (crawler at accoona dot com)  
Disallow: /
User-agent: Accoona-AI-Agent/1.1.2 (aicrawler at accoonabot dot com)  
Disallow: /
User-agent: Ace Explorer  
Disallow: /
User-agent: Ack (http://www.ackerm.com/)  
Disallow: /
User-agent: AcoiRobot  
Disallow: /
User-agent: Acoon Robot v1.50.001  
Disallow: /
User-agent: Acoon Robot v1.52 (http://www.acoon.de)  
Disallow: /
User-agent: Acoon-Robot 4.0.x.[xx] (http://www.acoon.de)  
Disallow: /
User-agent: Acoon-Robot v3.xx (http://www.acoon.de and http://www.acoon.com)  
Disallow: /
User-agent: Acorn/Nutch-0.9 (Non-Profit Search Engine; acorn.isara.org; acorn at isara dot org)  
Disallow: /
User-agent: ActiveBookmark 1.x  
Disallow: /
User-agent: ActiveWorlds/3.xx (xxx)  
Disallow: /
User-agent: Activeworlds  
Disallow: /
User-agent: Ad Muncher v4.xx.x  
Disallow: /
User-agent: Ad Muncher v4x Build xxxxx  
Disallow: /
User-agent: Adaxas Spider (http://www.adaxas.net/)  
Disallow: /
User-agent: Advanced Browser (http://www.avantbrowser.com)  
Disallow: /
User-agent: Agent-SharewarePlazaFileCheckBot/2.0+(+http://www.SharewarePlaza.com)  
Disallow: /
User-agent: AgentName/0.1 libwww-perl/5.48  
Disallow: /
User-agent: AideRSS/1.0 (aiderss.com)  
Disallow: /
User-agent: Aladin/3.324  
Disallow: /
User-agent: Alcatel-BG3/1.0 UP.Browser/5.0.3.1.2  
Disallow: /
User-agent: Aleksika Spider/1.0 (+http://www.aleksika.com/)  
Disallow: /
User-agent: AlkalineBOT/1.3  
Disallow: /
User-agent: AlkalineBOT/1.4 (1.4.0326.0 RTM)  
Disallow: /
User-agent: Allesklar/0.1 libwww-perl/5.46  
Disallow: /
User-agent: Alligator 1.31 (www.nearsoftware.com)  
Disallow: /
User-agent: AltaVista Intranet V2.0 AVS EVAL search@freeit.com  
Disallow: /
User-agent: AltaVista Intranet V2.0 Compaq Altavista Eval sveand@altavista.net  
Disallow: /
User-agent: AltaVista Intranet V2.0 evreka.com crawler@evreka.com  
Disallow: /
User-agent: AltaVista V2.0B crawler@evreka.com  
Disallow: /
User-agent: AmfibiBOT  
Disallow: /
User-agent: Amfibibot/0.06 (Amfibi Web Search; http://www.amfibi.com; agent@amfibi.com)  
Disallow: /
User-agent: Amfibibot/0.07 (Amfibi Robot; http://www.amfibi.com; agent@amfibi.com)  
Disallow: /
User-agent: AmiTCP Miami (AmigaOS 2.04)  
Disallow: /
User-agent: Amiga-AWeb/3.4.167SE  
Disallow: /
User-agent: AmigaVoyager/3.4.4 (MorphOS/PPC native)  
Disallow: /
User-agent: Amoi 8512/R21.0 NF-Browser/3.3  
Disallow: /
User-agent: AnnoMille spider 0.1 alpha - http://www.annomille.it  
Disallow: /
User-agent: Anonymized by ProxyOS: http://www.megaproxy.com  
Disallow: /
User-agent: Anonymizer/1.1  
Disallow: /
User-agent: AnswerBus (http://www.answerbus.com/)  
Disallow: /
User-agent: AnswerChase PROve x.0  
Disallow: /
User-agent: AnswerChase x.0  
Disallow: /
User-agent: AnzwersCrawl/2.0 (anzwerscrawl@anzwers.com.au;Engine)  
Disallow: /
User-agent: Apexoo Spider 1.x  
Disallow: /
User-agent: Aplix HTTP/1.0.1  
Disallow: /
User-agent: Aplix_SANYO_browser/1.x (Japanese)  
Disallow: /
User-agent: Aplix_SEGASATURN_browser/1.x (Japanese)  
Disallow: /
User-agent: Aport  
Disallow: /
User-agent: Apple iPhone v1.1.4 CoreMedia v1.0.0.4A102  
Disallow: /
User-agent: ArabyBot (compatible; Mozilla/5.0; GoogleBot; FAST Crawler 6.4; http://www.araby.com;)  
Disallow: /
User-agent: ArachBot  
Disallow: /
User-agent: Arachnoidea (arachnoidea@euroseek.com)  
Disallow: /
User-agent: ArchitextSpider  
Disallow: /
User-agent: Argus/1.1 (Nutch; http://www.simpy.com/bot.html; feedback at simpy dot com)  
Disallow: /
User-agent: Arikus_Spider  
Disallow: /
User-agent: Arquivo-web-crawler (compatible; heritrix/1.12.1 +http://arquivo-web.fccn.pt)  
Disallow: /
User-agent: Asahina-Antenna/1.x  
Disallow: /
User-agent: Asahina-Antenna/1.x (libhina.pl/x.x ; libtime.pl/x.x)  
Disallow: /
User-agent: AskAboutOil/0.06-rcp (Nutch; http://www.nutch.org/docs/en/bot.html; nutch-agent@askaboutoil.com)  
Disallow: /
User-agent: AtlocalBot/1.1 +(http://www.atlocal.com/local-web-site-owner.html)  
Disallow: /
User-agent: Atomic_Email_Hunter/4.0  
Disallow: /
User-agent: Atomz/1.0  
Disallow: /
User-agent: Attentio/Nutch-0.9-dev (Attentio's beta blog crawler; www.attentio.com; info@attentio.com)  
Disallow: /
User-agent: Avant Browser (http://www.avantbrowser.com)  
Disallow: /
User-agent: AxmoRobot - Crawling your site for better indexing on www.axmo.com search engine.  
Disallow: /
User-agent: Azureus 2.x.x.x  
Disallow: /
User-agent: BDFetch  
Disallow: /
User-agent: BDNcentral Crawler v2.3 [en] (http://www.bdncentral.com/robot.html) (X11; I; Linux 2.0.44 i686)  
Disallow: /
User-agent: BIGLOTRON (Beta 2;GNU/Linux)  
Disallow: /
User-agent: BMCLIENT  
Disallow: /
User-agent: BMLAUNCHER  
Disallow: /
User-agent: BPImageWalker/2.0 (www.bdbrandprotect.com)  
Disallow: /
User-agent: BSDSeek/1.0  
Disallow: /
User-agent: BStop.BravoBrian.it Agent Detector  
Disallow: /
User-agent: BTWebClient/180B(9704)  
Disallow: /
User-agent: BTbot/0.x (+http://www.btbot.com/btbot.html)  
Disallow: /
User-agent: BW-C-2.0  
Disallow: /
User-agent: BaboomBot/1.x.x (+http://www.baboom.us)  
Disallow: /
User-agent: BackStreet Browser 3.x  
Disallow: /
User-agent: BaiDuSpider  
Disallow: /
User-agent: BaiduImagespider+(+http://www.baidu.jp/search/s308.html)  
Disallow: /
User-agent: Baiduspider+(+http://help.baidu.jp/system/05.html)  
Disallow: /
User-agent: Baiduspider+(+http://www.baidu.com/search/spider.htm)  
Disallow: /
User-agent: Baiduspider+(+http://www.baidu.com/search/spider_jp.html)  
Disallow: /
User-agent: Balihoo/Nutch-1.0-dev (Crawler for Balihoo.com search engine - obeys robots.txt and robots meta tags ; http://balihoo.com/index.aspx; robot at balihoo dot com)  
Disallow: /
User-agent: BanBots/1.2 (spider@banbots.com)  
Disallow: /
User-agent: Barca/2.0.xxxx  
Disallow: /
User-agent: BarcaPro/1.4.xxxx  
Disallow: /
User-agent: BarraHomeCrawler (albertof@barrahome.org)  
Disallow: /
User-agent: BeamMachine/0.5 (dead link remover of www.beammachine.net)  
Disallow: /
User-agent: BebopBot/2.5.1 ( crawler http://www.apassion4jazz.net/bebopbot.html )  
Disallow: /
User-agent: BeebwareDirectory/v0.01  
Disallow: /
User-agent: Big Brother (http://pauillac.inria.fr/~fpottier/)  
Disallow: /
User-agent: Big Fish v1.0  
Disallow: /
User-agent: BigBrother/1.6e  
Disallow: /
User-agent: BigCliqueBOT/1.03-dev (bigclicbot; http://www.bigclique.com; bot@bigclique.com)  
Disallow: /
User-agent: Bigsearch.ca/Nutch-x.x-dev (Bigsearch.ca Internet Spider; http://www.bigsearch.ca/; info@enhancededge.com)  
Disallow: /
User-agent: Bilbo/2.3b-UNIX  
Disallow: /
User-agent: BilgiBetaBot/0.8-dev (bilgi.com (Beta) ; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)  
Disallow: /
User-agent: BilgiBot/1.0(beta) (http://www.bilgi.com/; bilgi at bilgi dot com)  
Disallow: /
User-agent: Bitacle Robot (V:1.0;) (http://www.bitacle.com)  
Disallow: /
User-agent: Bitacle bot/1.1  
Disallow: /
User-agent: Biyubi/x.x (Sistema Fenix; G11; Familia Toledo; es-mx)  
Disallow: /
User-agent: BlackBerry7520/4.0.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/5.0.3.3 UP.Link/5.1.2.12 (Google WAP Proxy/1.0)  
Disallow: /
User-agent: BlackWidow  
Disallow: /
User-agent: Blaiz-Bee/1.0 (+http://www.blaiz.net)  
Disallow: /
User-agent: Blaiz-Bee/2.00.8222 (BE Internet Search Engine http://www.rawgrunt.com)  
Disallow: /
User-agent: Blaiz-Bee/2.00.xxxx (+http://www.blaiz.net)  
Disallow: /
User-agent: BlitzBOT@tricus.net  
Disallow: /
User-agent: BlitzBOT@tricus.net (Mozilla compatible)  
Disallow: /
User-agent: BlockNote.Net  
Disallow: /
User-agent: BlogBot/1.x  
Disallow: /
User-agent: BlogBridge 2.13 (http://www.blogbridge.com/)  
Disallow: /
User-agent: BlogMap (http://www.feedmap.net)  
Disallow: /
User-agent: BlogPulseLive (support@blogpulse.com)  
Disallow: /
User-agent: BlogSearch/1.x +http://www.icerocket.com/  
Disallow: /
User-agent: BlogVibeBot-v1.1 (spider@blogvibe.nl)  
Disallow: /
User-agent: Bloglines Title Fetch/1.0 (http://www.bloglines.com)  
Disallow: /
User-agent: Bloglines-Images/0.1 (http://www.bloglines.com)  
Disallow: /
User-agent: Blogpulse (info@blogpulse.com)  
Disallow: /
User-agent: BlogsNowBot, V 2.01 (+http://www.blogsnow.com/)  
Disallow: /
User-agent: BlogzIce/1.0 (+http://icerocket.com; rhodes@icerocket.com)  
Disallow: /
User-agent: BlogzIce/1.0 +http://www.icerocket.com/  
Disallow: /
User-agent: BloobyBot  
Disallow: /
User-agent: Bloodhound/Nutch-0.9 (Testing Crawler for Research - obeys robots.txt and robots meta tags ; http://balihoo.com/index.aspx; robot at balihoo dot com)  
Disallow: /
User-agent: Bobby/4.0.x RPT-HTTPClient/0.3-3E  
Disallow: /
User-agent: Bookdog/x.x  
Disallow: /
User-agent: Bookmark Buddy bookmark checker (http://www.bookmarkbuddy.net/)  
Disallow: /
User-agent: Bookmark Renewal Check Agent [http://www.bookmark.ne.jp/]  
Disallow: /
User-agent: Bookmark Renewal Check Agent [http://www.bookmark.ne.jp/] (Version 2.0beta)  
Disallow: /
User-agent: BookmarkBase(2/;http://bookmarkbase.com)  
Disallow: /
User-agent: Bot mailto:craftbot@yahoo.com  
Disallow: /
User-agent: BravoBrian SpiderEngine MarcoPolo  
Disallow: /
User-agent: BravoBrian bstop.bravobrian.it  
Disallow: /
User-agent: BrightCrawler (http://www.brightcloud.com/brightcrawler.asp)  
Disallow: /
User-agent: BruinBot (+http://webarchive.cs.ucla.edu/bruinbot.html)   
Disallow: /
User-agent: BuildCMS crawler (http://www.buildcms.com/crawler)  
Disallow: /
User-agent: Bulkfeeds/r1752 (http://bulkfeeds.net/)  
Disallow: /
User-agent: BullsEye  
Disallow: /
User-agent: BunnySlippers  
Disallow: /
User-agent: BurstFindCrawler/1.1 (crawler.burstfind.com; http://crawler.burstfind.com; crawler@burstfind.com)  
Disallow: /
User-agent: Buscaplus Robi/1.0 (http://www.buscaplus.com/robi/)  
Disallow: /
User-agent: CCBot/1.0 (+http://www.commoncrawl.org/bot.html)  
Disallow: /
User-agent: CDR/1.7.1 Simulator/0.7(+http://timewe.net) Profile/MIDP-1.0 Configuration/CLDC-1.0  
Disallow: /
User-agent: CE-Preload  
Disallow: /
User-agent: CFNetwork/x.x  
Disallow: /
User-agent: CHttpClient by Open Text Corporation  
Disallow: /
User-agent: CJ Spider/  
Disallow: /
User-agent: CJB.NET Proxy  
Disallow: /
User-agent: COAST WebMaster Pro/4.x.x.xx (Windows NT)  
Disallow: /
User-agent: CSE HTML Validator Professional (http://www.htmlvalidator.com/)  
Disallow: /
User-agent: Cabot/Nutch-0.9 (Amfibi's web-crawling robot; http://www.amfibi.com/cabot/; agent@amfibi.com)  
Disallow: /
User-agent: Cabot/Nutch-1.0-dev (Amfibi's web-crawling robot; http://www.amfibi.com/cabot/; agent@amfibi.com)  
Disallow: /
User-agent: CamelHttpStream/1.0  
Disallow: /
User-agent: Cancer Information and Support International;  
Disallow: /
User-agent: Carnegie_Mellon_University_Research_WebBOT-->PLEASE READ-->http://www.andrew.cmu.edu/~brgordon/webbot/index.html http://www.andrew.cmu.edu/~brgordon/webbot/index.html  
Disallow: /
User-agent: Carnegie_Mellon_University_WebCrawler http://www.andrew.cmu.edu/~brgordon/webbot/index.html  
Disallow: /
User-agent: Catall Spider  
Disallow: /
User-agent: CazoodleBot/CazoodleBot-0.1 (CazoodleBot Crawler; http://www.cazoodle.com/cazoodlebot; cazoodlebot@cazoodle.com)  
Disallow: /
User-agent: CentiverseBot  
Disallow: /
User-agent: CentiverseBot - investigator  
Disallow: /
User-agent: CentiverseBot/3.0 (http://www.centiverse-project.net)  
Disallow: /
User-agent: Ceramic Tile Installation Guide (http://www.floorstransformed.com)  
Disallow: /
User-agent: Charon/1.x (Amiga)  
Disallow: /
User-agent: CheckLinks/1.x.x  
Disallow: /
User-agent: CheckUrl  
Disallow: /
User-agent: CheckWeb  
Disallow: /
User-agent: Checkbot/1.xx LWP/5.xx  
Disallow: /
User-agent: Chilkat/1.0.0 (+http://www.chilkatsoft.com/ChilkatHttpUA.asp)  
Disallow: /
User-agent: China Local Browse 2.6  
Disallow: /
User-agent: Chitika ContentHit 1.0  
Disallow: /
User-agent: ChristCRAWLER 2.0  
Disallow: /
User-agent: CipinetBot (http://www.cipinet.com/bot.html)  
Disallow: /
User-agent: Cityreview Robot (+http://www.cityreview.org/crawler/)  
Disallow: /
User-agent: ClariaBot/1.0  
Disallow: /
User-agent: Claymont.com  
Disallow: /
User-agent: CloakDetect/0.9 (+http://fulltext.seznam.cz/)  
Disallow: /
User-agent: Clushbot/2.x (+http://www.clush.com/bot.html)  
Disallow: /
User-agent: Clushbot/3.x-BinaryFury (+http://www.clush.com/bot.html)  
Disallow: /
User-agent: Clushbot/3.xx-Ajax (+http://www.clush.com/bot.html)  
Disallow: /
User-agent: Clushbot/3.xx-Hector (+http://www.clush.com/bot.html)  
Disallow: /
User-agent: Clushbot/3.xx-Peleus (+http://www.clush.com/bot.html)  
Disallow: /
User-agent: CoBITSProbe  
Disallow: /
User-agent: Cocoal.icio.us/1.0 (v36) (Mac OS X; http://www.scifihifi.com/cocoalicious)  
Disallow: /
User-agent: ColdFusion  
Disallow: /
User-agent: ColdFusion (BookmarkTracker.com)  
Disallow: /
User-agent: Combine/2.0 http://combine.it.lth.se/  
Disallow: /
User-agent: Combine/3 http://combine.it.lth.se/  
Disallow: /
User-agent: Combine/x.0  
Disallow: /
User-agent: Commerce Browser Center  
Disallow: /
User-agent: Computer_and_Automation_Research_Institute_Crawler crawler@ilab.sztaki.hu  
Disallow: /
User-agent: Comrite/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)  
Disallow: /
User-agent: Contact  
Disallow: /
User-agent: ContactBot/0.2  
Disallow: /
User-agent: ContentSmartz  
Disallow: /
User-agent: Convera Internet Spider V6.x  
Disallow: /
User-agent: ConveraCrawler/0.2  
Disallow: /
User-agent: ConveraCrawler/0.9d (+http://www.authoritativeweb.com/crawl)  
Disallow: /
User-agent: ConveraMultiMediaCrawler/0.1 (+http://www.authoritativeweb.com/crawl)  
Disallow: /
User-agent: CoolBot  
Disallow: /
User-agent: CoralWebPrx/0.1.1x (See http://coralcdn.org/)  
Disallow: /
User-agent: CoteoNutchCrawler/Nutch-0.9 (info [at] coteo [dot] com)  
Disallow: /
User-agent: CougarSearch/0.x (+http://www.cougarsearch.com/faq.shtml)  
Disallow: /
User-agent: Covac TexAs Arachbot  
Disallow: /
User-agent: Cowbot-0.1 (NHN Corp. / +82-2-3011-1954 / nhnbot@naver.com)  
Disallow: /
User-agent: Cowbot-0.1.x (NHN Corp. / +82-2-3011-1954 / nhnbot@naver.com)  
Disallow: /
User-agent: CrawlConvera0.1 (CrawlConvera@yahoo.com)  
Disallow: /
User-agent: Crawler  
Disallow: /
User-agent: Crawler (cometsearch@cometsystems.com)  
Disallow: /
User-agent: Crawler V 0.2.x admin@crawler.de  
Disallow: /
User-agent: Crawler admin@crawler.de  
Disallow: /
User-agent: CrawlerBoy Pinpoint.com  
Disallow: /
User-agent: Crawllybot/0.1 (Crawllybot; +http://www.crawlly.com; crawler@crawlly.com)  
Disallow: /
User-agent: CreativeCommons/0.06-dev (Nutch; http://www.nutch.org/docs/en/bot.html; nutch-agent@lists.sourceforge.net)  
Disallow: /
User-agent: CrocCrawler vx.3 [en] (http://www.croccrawler.com) (X11; I; Linux 2.0.44 i686)  
Disallow: /
User-agent: Cuam Ver0.050bx  
Disallow: /
User-agent: Cuasarbot/0.9b http://www.cuasar.com/spider_beta/   
Disallow: /
User-agent: CurryGuide SiteScan 1.1  
Disallow: /
User-agent: Custo x.x (www.netwu.com)  
Disallow: /
User-agent: Custom Spider www.bisnisseek.com /1.0  
Disallow: /
User-agent: CyberPatrol SiteCat Webbot (http://www.cyberpatrol.com/cyberpatrolcrawler.asp)  
Disallow: /
User-agent: CyberSpyder Link Test/2.1.12 (admin@mspennyworth.com)  
Disallow: /
User-agent: Cyberdog/2.0 (Macintosh; 68k)  
Disallow: /
User-agent: CydralSpider/1.x (Cydral Web Image Search; http://www.cydral.com)  
Disallow: /
User-agent: CydralSpider/3.0 (Cydral Image Search; http://www.cydral.com)  
Disallow: /
User-agent: DA 3.5 (www.lidan.com)  
Disallow: /
User-agent: DA 4.0  
Disallow: /
User-agent: DA 4.0 (www.downloadaccelerator.com)  
Disallow: /
User-agent: DA 5.0  
Disallow: /
User-agent: DA 7.0  
Disallow: /
User-agent: DBrowse 1.4b  
Disallow: /
User-agent: DBrowse 1.4d  
Disallow: /
User-agent: DC-Sakura/x.xx  
Disallow: /
User-agent: DDD  
Disallow: /
User-agent: DIIbot/1.2  
Disallow: /
User-agent: DISCo Pump x.x  
Disallow: /
User-agent: DNSRight.com WebBot Link Ckeck Tool. Report abuse to: dnsr@dnsright.com  
Disallow: /
User-agent: DSurf15a 01  
Disallow: /
User-agent: DSurf15a 71  
Disallow: /
User-agent: DSurf15a 81  
Disallow: /
User-agent: DSurf15a VA  
Disallow: /
User-agent: DTAAgent  
Disallow: /
User-agent: Dart Communications PowerTCP  
Disallow: /
User-agent: DataCha0s/2.0  
Disallow: /
User-agent: DataFountains/DMOZ Downloader  
Disallow: /
User-agent: DataFountains/DMOZ Feature Vector Corpus Creator (http://ivia.ucr.edu/useragents.shtml)  
Disallow: /
User-agent: DataFountains/Dmoz Downloader (http://ivia.ucr.edu/useragents.shtml)  
Disallow: /
User-agent: DataSpear/1.0 (Spider; http://www.dataspear.com/spider.html; spider@dataspear.com)  
Disallow: /
User-agent: DataSpearSpiderBot/0.2 (DataSpear Spider Bot; http://dssb.dataspear.com/bot.html; dssb@dataspear.com)  
Disallow: /
User-agent: DataparkSearch/4.47 (+http://dataparksearch.org/bot)  
Disallow: /
User-agent: DataparkSearch/4.xx (http://www.dataparksearch.org/)  
Disallow: /
User-agent: DatenBot( http://www.sicher-durchs-netz.de/bot.html)  
Disallow: /
User-agent: DaviesBot/1.7 (www.wholeweb.net)  
Disallow: /
User-agent: DeadLinkCheck/0.4.0 libwww-perl/5.xx  
Disallow: /
User-agent: Deep Link Calculator v1.0  
Disallow: /
User-agent: DeepIndex  
Disallow: /
User-agent: DeepIndex ( http://www.zetbot.com )  
Disallow: /
User-agent: DeepIndex (www.en.deepindex.com)  
Disallow: /
User-agent: DeepIndexer.ca  
Disallow: /
User-agent: DeleGate/9.0.5-fix1  
Disallow: /
User-agent: Demo Bot DOT 16b  
Disallow: /
User-agent: Demo Bot Z 16b  
Disallow: /
User-agent: Denmex websearch (http://search.denmex.com)  
Disallow: /
User-agent: Der große BilderSauger 2.00u  
Disallow: /
User-agent: DevComponents.com HtmlDocument Object  
Disallow: /
User-agent: DiaGem/1.1 (http://www.skyrocket.gr.jp/diagem.html)  
Disallow: /
User-agent: Diamond/x.0  
Disallow: /
User-agent: DiamondBot  
Disallow: /
User-agent: DigOut4U  
Disallow: /
User-agent: Digger/1.0 JDK/1.3.0rc3  
Disallow: /
User-agent: Dillo/0.8.5-i18n-misc  
Disallow: /
User-agent: Dillo/0.x.x  
Disallow: /
User-agent: DittoSpyder  
Disallow: /
User-agent: DoCoMo/1.0/Nxxxi/c10  
Disallow: /
User-agent: DoCoMo/1.0/Nxxxi/c10/TB  
Disallow: /
User-agent: DoCoMo/1.0/P502i/c10 (Google CHTML Proxy/1.0)  
Disallow: /
User-agent: DoCoMo/2.0 P900iV(c100;TB;W24H11)   
Disallow: /
User-agent: DoCoMo/2.0 SH901iS(c100;TB;W24H12),gzip(gfe) (via translate.google.com)  
Disallow: /
User-agent: DoCoMo/2.0 SH902i (compatible; Y!J-SRD/1.0; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-27.html)  
Disallow: /
User-agent: DoCoMo/2.0/SO502i (compatible; Y!J-SRD/1.0; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-27.html)  
Disallow: /
User-agent: DocZilla/1.0 (Windows; U; WinNT4.0; en-US; rv:1.0.0) Gecko/20020804  
Disallow: /
User-agent: DonutP; Windows98SE  
Disallow: /
User-agent: Doubanbot/1.0 (bot@douban.com http://www.douban.com)  
Disallow: /
User-agent: Download Demon/3.x.x.x  
Disallow: /
User-agent: Download Druid 2.x  
Disallow: /
User-agent: Download Express 1.0  
Disallow: /
User-agent: Download Master  
Disallow: /
User-agent: Download Ninja 3.0  
Disallow: /
User-agent: Download Wonder  
Disallow: /
User-agent: Download-Tipp Linkcheck (http://download-tipp.de/)  
Disallow: /
User-agent: Download.exe(1.1) (+http://www.sql-und-xml.de/freeware-tools/)  
Disallow: /
User-agent: DownloadDirect.1.0  
Disallow: /
User-agent: Dr.Web (R) online scanner: http://online.drweb.com/  
Disallow: /
User-agent: Dragonfly File Reader  
Disallow: /
User-agent: Drecombot/1.0 (http://career.drecom.jp/bot.html)  
Disallow: /
User-agent: Drupal (+http://drupal.org/)  
Disallow: /
User-agent: Dual Proxy  
Disallow: /
User-agent: DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)  
Disallow: /
User-agent: Dumbot(version 0.1 beta - dumbfind.com)  
Disallow: /
User-agent: Dumbot(version 0.1 beta - http://www.dumbfind.com/dumbot.html)  
Disallow: /
User-agent: Dumbot(version 0.1 beta)  
Disallow: /
User-agent: EARTHCOM.info/1.x [www.earthcom.info]  
Disallow: /
User-agent: EARTHCOM.info/1.xbeta [www.earthcom.info]  
Disallow: /
User-agent: EBrowse 1.4b  
Disallow: /
User-agent: ELI/20070402:2.0 (DAUM RSS Robot, Daum Communications Corp.; +http://ws.daum.net/aboutkr.html)  
Disallow: /
User-agent: ELinks (0.x.x; Linux 2.4.20 i586; 132x60)  
Disallow: /
User-agent: ELinks/0.x.x (textmode; NetBSD 1.6.2 sparc; 132x43)  
Disallow: /
User-agent: EMPAS_ROBOT  
Disallow: /
User-agent: ES.NET_Crawler/2.0 (http://search.innerprise.net/)  
Disallow: /
User-agent: ESISmartSpider  
Disallow: /
User-agent: ESurf15a 15  
Disallow: /
User-agent: EasyDL/3.xx  
Disallow: /
User-agent: EasyDL/3.xx http://keywen.com/Encyclopedia/Bot  
Disallow: /
User-agent: EchO!/2.0  
Disallow: /
User-agent: Educate Search VxB  
Disallow: /
User-agent: EgotoBot/4.8 (+http://www.egoto.com/about.htm)  
Disallow: /
User-agent: EldoS TimelyWeb/3.x  
Disallow: /
User-agent: EmailSiphon  
Disallow: /
User-agent: EmailSpider  
Disallow: /
User-agent: EmailWolf 1.00  
Disallow: /
User-agent: EmeraldShield.com WebBot  
Disallow: /
User-agent: EmeraldShield.com WebBot (http://www.emeraldshield.com/webbot.aspx)  
Disallow: /
User-agent: EnaBot/1.x (http://www.enaball.com/crawler.html)  
Disallow: /
User-agent: Enfish Tracker  
Disallow: /
User-agent: Enterprise_Search/1.0  
Disallow: /
User-agent: Enterprise_Search/1.0.xxx  
Disallow: /
User-agent: Enterprise_Search/1.00.xxx;MSSQL (http://www.innerprise.net/es-spider.asp)  
Disallow: /
User-agent: EroCrawler  
Disallow: /
User-agent: EuripBot/0.x (+http://www.eurip.com) GetFile  
Disallow: /
User-agent: EuripBot/0.x (+http://www.eurip.com) GetRobots  
Disallow: /
User-agent: EuripBot/0.x (+http://www.eurip.com) PreCheck  
Disallow: /
User-agent: Eurobot/1.0 (http://www.ayell.eu)  
Disallow: /
User-agent: EvaalSE - bot@evaal.com  
Disallow: /
User-agent: Everest-Vulcan Inc./0.1 (R&D project; host=e-1-24; http://everest.vulcan.com/crawlerhelp)  
Disallow: /
User-agent: Everest-Vulcan Inc./0.1 (R&D project; http://everest.vulcan.com/crawlerhelp)  
Disallow: /
User-agent: Exabot-Images/1.0  
Disallow: /
User-agent: Exabot-Test/1.0  
Disallow: /
User-agent: Exabot/2.0  
Disallow: /
User-agent: Exabot/3.0  
Disallow: /
User-agent: ExactSearch  
Disallow: /
User-agent: ExactSeek Crawler/0.1  
Disallow: /
User-agent: Exalead NG/MimeLive Client (convert/http/0.120)  
Disallow: /
User-agent: Excalibur Internet Spider V6.5.4  
Disallow: /
User-agent: Execrawl/1.0 (Execrawl; http://www.execrawl.com/; bot@execrawl.com)  
Disallow: /
User-agent: ExperimentalHenrytheMiragoRobot  
Disallow: /
User-agent: Expired Domain Sleuth  
Disallow: /
User-agent: Express WebPictures (www.express-soft.com)  
Disallow: /
User-agent: ExtractorPro  
Disallow: /
User-agent: Extreme Picture Finder  
Disallow: /
User-agent: EyeCatcher (Download-tipp.de)/1.0  
Disallow: /
User-agent: FANGCrawl/0.01  
Disallow: /
User-agent: FARK.com link verifier  
Disallow: /
User-agent: FAST Enterprise Crawler 6 (Experimental)  
Disallow: /
User-agent: FAST Enterprise Crawler 6 / Scirus scirus-crawler@fast.no; http://www.scirus.com/srsapp/contactus/  
Disallow: /
User-agent: FAST Enterprise Crawler 6 used by Cobra Development (admin@fastsearch.com)  
Disallow: /
User-agent: FAST Enterprise Crawler 6 used by Comperio AS (sts@comperio.no)  
Disallow: /
User-agent: FAST Enterprise Crawler 6 used by FAST (FAST)  
Disallow: /
User-agent: FAST Enterprise Crawler 6 used by Pages Jaunes (pvincent@pagesjaunes.fr)  
Disallow: /
User-agent: FAST Enterprise Crawler 6 used by Sensis.com.au Web Crawler (search_comments\at\sensis\dot\com\dot\au)  
Disallow: /
User-agent: FAST Enterprise Crawler 6 used by Singapore Press Holdings (crawler@sphsearch.sg)  
Disallow: /
User-agent: FAST Enterprise Crawler 6 used by WWU (wardi@uni-muenster.de)  
Disallow: /
User-agent: FAST Enterprise Crawler/6 (www.fastsearch.com)  
Disallow: /
User-agent: FAST Enterprise Crawler/6.4 (helpdesk at fast.no)  
Disallow: /
User-agent: FAST FirstPage retriever (compatible; MSIE 5.5; Mozilla/4.0)  
Disallow: /
User-agent: FAST MetaWeb Crawler (helpdesk at fastsearch dot com)  
Disallow: /
User-agent: FAST-WebCrawler/2.2.10 (Multimedia Search) (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)  
Disallow: /
User-agent: FAST-WebCrawler/2.2.6 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)  
Disallow: /
User-agent: FAST-WebCrawler/2.2.7 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)http://www.fast.no  
Disallow: /
User-agent: FAST-WebCrawler/2.2.8 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)http://www.fast.no  
Disallow: /
User-agent: FAST-WebCrawler/3.2 test  
Disallow: /
User-agent: FAST-WebCrawler/3.3 (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)  
Disallow: /
User-agent: FAST-WebCrawler/3.4/Nirvana (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)  
Disallow: /
User-agent: FAST-WebCrawler/3.4/PartnerSite (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)  
Disallow: /
User-agent: FAST-WebCrawler/3.5 (atw-crawler at fast dot no; http://fast.no/support.php?c=faqs/crawler)  
Disallow: /
User-agent: FAST-WebCrawler/3.6 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)  
Disallow: /
User-agent: FAST-WebCrawler/3.6/FirstPage (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)  
Disallow: /
User-agent: FAST-WebCrawler/3.7 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)  
Disallow: /
User-agent: FAST-WebCrawler/3.7/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)  
Disallow: /
User-agent: FAST-WebCrawler/3.8 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)  
Disallow: /
User-agent: FAST-WebCrawler/3.8/Fresh (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)  
Disallow: /
User-agent: FAST-WebCrawler/3.x Multimedia  
Disallow: /
User-agent: FAST-WebCrawler/3.x Multimedia (mm dash crawler at fast dot no)  
Disallow: /
User-agent: FDM 1.x  
Disallow: /
User-agent: FDM 2.x  
Disallow: /
User-agent: FFC Trap Door Spider  
Disallow: /
User-agent: FLATARTS_FAVICO  
Disallow: /
User-agent: FSurf15a 01  
Disallow: /
User-agent: FaEdit/2.0.x  
Disallow: /
User-agent: Factbot 1.09 (see http://www.factbites.com/webmasters.php)  
Disallow: /
User-agent: FairAd Client  
Disallow: /
User-agent: Fast Crawler Gold Edition  
Disallow: /
User-agent: Fast PartnerSite Crawler  
Disallow: /
User-agent: FastBug http://www.ay-up.com  
Disallow: /
User-agent: FastCrawler 3.0.1 (crawler@1klik.dk)  
Disallow: /
User-agent: FastSearch Web Crawler for Verizon SuperPages (kevin.watters@fastsearch.com)  
Disallow: /
User-agent: FavIconizer  
Disallow: /
User-agent: FavOrg  
Disallow: /
User-agent: Favcollector/2.0 (info@favcollector.com http://www.favcollector.com/)  
Disallow: /
User-agent: Favorites Checking (http://campulka.net)  
Disallow: /
User-agent: Favorites Sweeper v.2.03  
Disallow: /
User-agent: Faxobot/1.0  
Disallow: /
User-agent: Feed Seeker Bot (RSS Feed Seeker http://www.MyNewFavoriteThing.com/fsb.php)  
Disallow: /
User-agent: Feed24.com  
Disallow: /
User-agent: Feed::Find/0.0x  
Disallow: /
User-agent: FeedChecker/0.01  
Disallow: /
User-agent: FeedDemon/2.7 (http://www.newsgator.com/; Microsoft Windows XP)  
Disallow: /
User-agent: FeedForAll rss2html.php v2  
Disallow: /
User-agent: FeedHub FeedDiscovery/1.0 (http://www.feedhub.com)  
Disallow: /
User-agent: FeedHub MetaDataFetcher/1.0 (http://www.feedhub.com)  
Disallow: /
User-agent: FeedZcollector v1.x (Platinum) http://www.feeds4all.com/feedzcollector  
Disallow: /
User-agent: Feedable/0.1 (compatible; MSIE 6.0; Windows NT 5.1)  
Disallow: /
User-agent: Feedfetcher-Google-iGoogleGadgets; (+http://www.google.com/feedfetcher.html)  
Disallow: /
User-agent: Feedfetcher-Google; (+http://www.google.com/feedfetcher.html)  
Disallow: /
User-agent: Feedjit Favicon Crawler 1.0  
Disallow: /
User-agent: Feedreader 3.xx (Powered by Newsbrain)  
Disallow: /
User-agent: Feedshow/x.0 (http://www.feedshow.com; 1 subscriber)  
Disallow: /
User-agent: FeedshowOnline (http://www.feedshow.com)  
Disallow: /
User-agent: Feedster Crawler/3.0; Feedster, Inc.  
Disallow: /
User-agent: Felix - Mixcat Crawler (+http://mixcat.com)  
Disallow: /
User-agent: Filangy/0.01-beta (Filangy; http://www.nutch.org/docs/en/bot.html; filangy-agent@filangy.com)  
Disallow: /
User-agent: Filangy/1.0x (Filangy; http://www.filangy.com/filangyinfo.jsp?inc=robots.jsp; filangy-agent@filangy.com)  
Disallow: /
User-agent: Filangy/1.0x (Filangy; http://www.nutch.org/docs/en/bot.html; filangy-agent@filangy.com)  
Disallow: /
User-agent: FileHound x.x  
Disallow: /
User-agent: Filtrbox/1.0  
Disallow: /
User-agent: FindAnISP.com_ISP_Finder_v99a  
Disallow: /
User-agent: Findexa Crawler (http://www.findexa.no/gulesider/article26548.ece)  
Disallow: /
User-agent: FineBot  
Disallow: /
User-agent: Finjan-prefetch  
Disallow: /
User-agent: Firefly/1.0  
Disallow: /
User-agent: Firefly/1.0 (compatible; Mozilla 4.0; MSIE 5.5)  
Disallow: /
User-agent: Firefox (kastaneta03@hotmail.com)  
Disallow: /
User-agent: Firefox_1.0.6 (kasparek@naparek.cz)  
Disallow: /
User-agent: FirstGov.gov Search - POC:firstgov.webmasters@gsa.gov  
Disallow: /
User-agent: Flapbot/0.7.2 (Flaptor Crawler; http://www.flaptor.com; crawler at flaptor period com)  
Disallow: /
User-agent: FlashGet  
Disallow: /
User-agent: Flexum spider  
Disallow: /
User-agent: Flexum/2.0  
Disallow: /
User-agent: FlickBot 2.0 RPT-HTTPClient/0.3-3  
Disallow: /
User-agent: FocusedSampler/1.0  
Disallow: /
User-agent: Folkd.com Spider/0.1 beta 1 (www.folkd.com)  
Disallow: /
User-agent: Fooky.com/ScorpionBot/ScoutOut; http://www.fooky.com/scorpionbots  
Disallow: /
User-agent: Francis/1.0 (francis@neomo.de http://www.neomo.de/)  
Disallow: /
User-agent: Franklin Locator 1.8  
Disallow: /
User-agent: FreeFind.com-SiteSearchEngine/1.0 (http://freefind.com; spiderinfo@freefind.com)  
Disallow: /
User-agent: FreshDownload/x.xx  
Disallow: /
User-agent: FreshNotes crawler< report problems to crawler-at-freshnotes-dot-com  
Disallow: /
User-agent: Full Web Bot 0416B  
Disallow: /
User-agent: Full Web Bot 0516B  
Disallow: /
User-agent: Full Web Bot 2816B  
Disallow: /
User-agent: FuseBulb.Com  
Disallow: /
User-agent: FyberSpider (+http://www.fybersearch.com/fyberspider.php)  
Disallow: /
User-agent: aardvark-crawler  
Disallow: /
User-agent: abot/0.1 (abot; http://www.abot.com; abot@abot.com)  
Disallow: /
User-agent: accoona  
Disallow: /
User-agent: agadine/1.x.x (+http://www.agada.de)  
Disallow: /
User-agent: aipbot/1.0 (aipbot; http://www.aipbot.com; aipbot@aipbot.com)  
Disallow: /
User-agent: aipbot/2-beta (aipbot dev; http://aipbot.com; aipbot@aipbot.com)  
Disallow: /
User-agent: amibot  
Disallow: /
User-agent: amzn_assoc  
Disallow: /
User-agent: annotate_google; http://ponderer.org/download/annotate_google.user.js  
Disallow: /
User-agent: antibot-V1.1.5/i586-linux-2.2  
Disallow: /
User-agent: appie 1.1 (www.walhello.com)  
Disallow: /
User-agent: aranhabot  
Disallow: /
User-agent: archive.org_bot  
Disallow: /
User-agent: ask.24x.info  
Disallow: /
User-agent: asked/Nutch-0.8 (web crawler; http://asked.jp; epicurus at gmail dot com)  
Disallow: /
User-agent: asterias/2.0  
Disallow: /
User-agent: atSpider/1.0  
Disallow: /
User-agent: augurfind  
Disallow: /
User-agent: augurnfind V-1.x  
Disallow: /
User-agent: autoemailspider  
Disallow: /
User-agent: autohttp  
Disallow: /
User-agent: autowebdir 1.1 (www.autowebdir.com)  
Disallow: /
User-agent: axadine/ (Axadine Crawler; http://www.axada.de/; )  
Disallow: /
User-agent: bCentral Billing Post-Process  
Disallow: /
User-agent: bdcindexer_2.6.2 (research@bdc)  
Disallow: /
User-agent: beautybot/1.0 (+http://www.uchoose.de/crawler/beautybot/)  
Disallow: /
User-agent: billbot wjj@cs.cmu.edu  
Disallow: /
User-agent: blogWatcher_Spider/0.1 (http://www.lr.pi.titech.ac.jp/blogWatcher/)  
Disallow: /
User-agent: blogsearchbot-pumpkin-3  
Disallow: /
User-agent: bluefish 0.6 HTML editor  
Disallow: /
User-agent: boitho.com-dc/0.xx (http://www.boitho.com/dcbot.html)  
Disallow: /
User-agent: boitho.com-robot/1.x  
Disallow: /
User-agent: boitho.com-robot/1.x (http://www.boitho.com/bot.html)  
Disallow: /
User-agent: bumblebee@relevare.com  
Disallow: /
User-agent: bwh3_user_agent  
Disallow: /
User-agent: carleson/1.0  
Disallow: /
User-agent: ccubee/x.x  
Disallow: /
User-agent: cfetch/1.0  
Disallow: /
User-agent: cg-eye interactive  
Disallow: /
User-agent: collage.cgi/1.xx  
Disallow: /
User-agent: combine/0.0  
Disallow: /
User-agent: cometrics-bot, http://www.cometrics.de  
Disallow: /
User-agent: complex_network_group/Nutch-0.9-dev (discovering the structure of the world-wide-web; http://cantor.ee.ucla.edu/~networks/crawl; nimakhaj@gmail.com)  
Disallow: /
User-agent: contype  
Disallow: /
User-agent: cosmos/0.8_(robot@xyleme.com)  
Disallow: /
User-agent: cosmos/0.9_(robot@xyleme.com)  
Disallow: /
User-agent: crawler@alexa.com  
Disallow: /
User-agent: csci_b659/0.13  
Disallow: /
User-agent: curl/7.10.x (i386-redhat-linux-gnu) libcurl/7.10.x OpenSSL/0.9.7a ipv6 zlib/1.1.4  
Disallow: /
User-agent: curl/7.7.x (i386--freebsd4.3) libcurl 7.7.x (SSL 0.9.6) (ipv6 enabled)  
Disallow: /
User-agent: curl/7.8 (i686-pc-linux-gnu) libcurl 7.8 (OpenSSL 0.9.6)  
Disallow: /
User-agent: curl/7.9.x (win32) libcurl 7.9.x  
Disallow: /
User-agent: dCSbot/1.1  
Disallow: /
User-agent: daypopbot/0.x  
Disallow: /
User-agent: dbDig(http://www.prairielandconsulting.com)  
Disallow: /
User-agent: dds explorer v1.0 beta  
Disallow: /
User-agent: de.searchengine.comBot 1.2 (http://de.searchengine.com/spider)  
Disallow: /
User-agent: deepak-USC/ISI  
Disallow: /
User-agent: del.icio.us-thumbnails/1.0 Mozilla/5.0 (compatible; Konqueror/3.4; FreeBSD) KHTML/3.4.2 (like Gecko)  
Disallow: /
User-agent: dev-spider2.searchpsider.com/1.3b  
Disallow: /
User-agent: disco/Nutch-0.9 (experimental crawler; www.discoveryengine.com; disco-crawl@discoveryengine.com)  
Disallow: /
User-agent: disco/Nutch-1.0-dev (experimental crawler; www.discoveryengine.com; disco-crawl@discoveryengine.com)  
Disallow: /
User-agent: dlman  
Disallow: /
User-agent: dloader(NaverRobot)/1.0  
Disallow: /
User-agent: dodgebot/experimental  
Disallow: /
User-agent: dtSearchSpider  
Disallow: /
User-agent: e-SocietyRobot(http://www.yama.info.waseda.ac.jp/~yamana/es/)  
Disallow: /
User-agent: e-sense 1.0 ea(www.vigiltech.com/esensedisclaim.html)  
Disallow: /
User-agent: eApolloBot/2.0 (compatible; heritrix/2.0.0-SNAPSHOT-20071024.170148 +http://www.eapollo-opto.com)  
Disallow: /
User-agent: eCatch/3.0  
Disallow: /
User-agent: eStyleSearch 4 (compatible; MSIE 6.0; Windows NT 5.0)  
Disallow: /
User-agent: egothor/3.0a (+http://www.xdefine.org/robot.html)  
Disallow: /
User-agent: ejupiter.com  
Disallow: /
User-agent: elfbot/1.0 (+http://www.uchoose.de/crawler/elfbot/)  
Disallow: /
User-agent: endo/1.0 (Mac OS X; ppc i386; http://kula.jp/endo)  
Disallow: /
User-agent: envolk/1.7 (+http://www.envolk.com/envolkspiderinfo.php)  
Disallow: /
User-agent: envolk[ITS]spider/1.6(+http://www.envolk.com/envolkspider.html)  
Disallow: /
User-agent: eseek-larbin_2.6.2 (crawler@exactseek.com)  
Disallow: /
User-agent: eventax/1.3 (eventax; http://www.eventax.de/; info@eventax.de)  
Disallow: /
User-agent: exactseek-crawler-2.63 (crawler@exactseek.com)  
Disallow: /
User-agent: exactseek-pagereaper-2.63 (crawler@exactseek.com)  
Disallow: /
User-agent: exactseek.com  
Disallow: /
User-agent: exooba crawler/exooba crawler (crawler for exooba.com; http://www.exooba.com/; info at exooba dot com)  
Disallow: /
User-agent: exooba/exooba crawler (exooba; exooba)  
Disallow: /
User-agent: factbot : http://www.factbites.com/robots  
Disallow: /
User-agent: fastbot crawler beta 2.0 (+http://www.fastbot.de)  
Disallow: /
User-agent: favo.eu crawler/0.6 (http://www.favo.eu)  
Disallow: /
User-agent: fetch libfetch/2.0  
Disallow: /
User-agent: fileboost.net/1.0 (+http://www.fileboost.net)  
Disallow: /
User-agent: findlinks/x.xxx (+http://wortschatz.uni-leipzig.de/findlinks/)   
Disallow: /
User-agent: firstsbot  
Disallow: /
User-agent: flunky  
Disallow: /
User-agent: fly/6.01 libwww/4.0D  
Disallow: /
User-agent: flyindex.net 1.0/http://www.flyindex.net  
Disallow: /
User-agent: free-downloads.net download-link validator /0.1  
Disallow: /
User-agent: unknownght.com Web Server IIS vs Apache Survey. See Results at www.DNSRight.com  
Disallow: /
User-agent: *
Crawl-delay: 10.00
Disallow: /wp-content/uploads
Disallow: /*.gif$
Disallow: /*.Gif$
Disallow: /*.GIF$
Disallow: /*.jpg$
Disallow: /*.Jpg$
Disallow: /*.JPG$
Disallow: /*.jpeg$
Disallow: /*.Jpeg$
Disallow: /*.JPEG$
Disallow: /*.pdf$
Disallow: /*.Pdf$
Disallow: /*.PDF$
Disallow: /*.zip$
Disallow: /*.Zip$
Disallow: /*.ZIP$