# Clients using user agents entered below will be ignored while counting WWW 
# site statistics.
# Each user-agent name should be entered in separate line as a regular 
# expression pattern.
# Most of searchers' robots identify themselves using user-agents 
# strings containing words like bot or crawler. This general expression 
# allows to detect unknown robots.
#.*([Hh]arvester|[Bb]ot|[Cc]rawler).*
.*Zeus 32297 Webster Pro V2\.9 Win32.*
.*linklooker.*
.*INGRID/0\.1.*
.*Aqua_Products.*
.*Arachnophilia.*
.*ASpider/0\.09.*
.*asterias.*
.*AURESYS/1\.0.*
.*b2w/0\.1.*
.*BackRub/\..*
.*Baiduspider-video.*
.*Big Brother.*
.*Black Hole.*
.*Black\.Hole.*
.*BlowFish.*
.*BlowFish/1\.0.*
.*Bookmark search tool.*
.*BSpider/1\.0 libwww-perl/0\.40.*
.*Bullseye/1\.0.*
.*BunnySlippers.*
.*CACTVS Chemistry Spider.*
.*Cegbfeieh.*
.*ChangeDetection.*
.*CherryPicker.*
.*CherryPickerElite/1\.0.*
.*CherryPickerSE/1\.0.*
.*ChinaClaw.*
.*combine/0\.0.*
.*Copernic.*
.*CopyRightCheck.*
.*cosmos.*
.*Crescent.*
.*Crescent Internet ToolPak HTTP OLE Control v\.1\.0.*
.*Custo.*
.*CyberSpyder/2\.1.*
.*Daumoa.*
.*Deweb/1\.01.*
.*DISCo.*
.*DISCo Pump 3\.0.*
.*DISCo Pump 3\.2.*
.*DISCoFinder.*
.*DittoSpyder.*
.*Download Demon.*
.*Download Demon/3\.2\.0\.8.*
.*Download Demon/3\.5\.0\.11.*
.*eCatch.*
.*eCatch/3\.0.*
.*EirGrabber.*
.*EmailCollector.*
.*EmailSiphon.*
.*EmailWolf.*
.*explorersearch.*
.*Express WebPictures.*
.*Express WebPictures (www\.express-soft\.com).*
.*ExtractorPro.*
.*EyeNetIE.*
.*FairAd Client.*
.*FelixIDE/1\.0.*
.*fido/0\.9 Harvest/1\.4\.pl2.*
.*FlashGet.*
.*FlashGet WebWasher 3\.2.*
.*Freecrawl.*
.*FrontPage.*
.*gcreep/1\.0.*
.*GetRight.*
.*GetRight/2\.11.*
.*Mozilla/5\.0 \(X11; Linux x86_64\) AppleWebKit/537\.36 \(KHTML\, like Gecko; Google Web Preview\) Chrome/27\.0\.1453 Safari/537\.36\.*
.*360Spider.*
.*YisouSpider.*
.*webspider/1.*
.*Mozilla/5\.0 \(compatible; NetcraftSurveyAgent/1\.0; \+info@netcraft\.com\).*
.*Mozilla/5\.0 \(compatible; proximic; \+http://www\.proximic\.com/info/spider\.php\).*
.*Windows-RSS-Platform/[0-9]\.[0-9] \(MSIE 8\.0; Windows NT 6\.0\).*
.*FacebookExternalHit/[0-9]\.[0-9].*
.*FacebookExternalHit/[0-9]\.[0-9].*
.*facebookexternalhit/[0-9]\.[0-9] \(\+http://www\.facebook\.com/externalhit_uatext\.php\).*
.*facebookexternalhit/[0-9]\.[0-9] \(\+http://www\.facebook\.com/externalhit_uatext\.php\).*
.*AppleSyndication.*
.*Apple-PubSub/[0-9]{0,2}\.[0-9]?.*
.*Apple-PubSub/[0-9]{0,2}\.[0-9]{0,2}\.[0-9]?.*
.*GeoHasher/Nutch\-1\.0 \(GeoHasher Web Search Engine; geohasher\.gotdns\.org; geo_hasher at yahoo \* com\).*
.*CamontSpider/1\.0 \+http://epweb2\.ph\.bham\.ac\.uk/user/slater/camont/info\.html.*
.*Mozilla/5\.0 \(compatible; Baiduspider/[0-9]\.[0-9]; \+http://www.baidu\.com/search/spider\.html\).*
.*Mozilla/5\.0 \(Windows NT 6\.1; WOW64\) AppleWebKit/534\+ \(KHTML, like Gecko\) BingPreview/1\.0b.*
.*Mozilla/5\.0 \(compatible; Ezooms/[0-9]\.[0-9]; help@moz\.com\).*
.*Yahoo! Slurp.*
.*holmes/[0-9]\.[0-9].*
.*Accoona-AI-Agent.*
.*Szukacz/[0-9]\.[0-9].*
.*NetSprint.*
.*Slurp(\.so)*/[0-9]\.[0-9] \(slurp@inktomi\.com.*
.*nagios-plugins.*
.*ZyBorg/[0-9]\.[0-9].*
.*ScientificCommons\.org.*
.*Python-urllib/[0-9]+\.[0-9]+.*
.*Java/[0-9]+\.[0-9]+\.[0-9]+_[0-9]+.*
.*Jakarta Commons-HttpClient/[0-9]+\.[0-9]+.*
.*Speedy Spider \(http://www\.entireweb\.com/about/search_tech/speedy_spider/\).*
.*FDM 2\.x.*
.*GetRight/[0-9]+\.[0-9]+d.*
.*Xenu Link Sleuth /1\.3\.8.*
.*Xenu Link Sleuth 1\.2j.*
.*gnu-classpath/0\.92 \(CACAO/0\.97\).*
.*ia_archiver.*
.*Feedfetcher-Google.*
.*ichiro/[0-9]\.[0-9].*
.*libwww-perl/[0-9]\.[0-9]+.*
.*Wget/[0-9]\.[0-9]+\.[0-9].*
.*http://Anonymouse\.org/.*
.*Yandex/[0-9]+\.[0-9]+\.[0-9]+.*
.*StackRambler/*
.*WordPress.*
.*lms.*
.*Microsoft Office Protocol Discovery.*
.*CFSCHEDULE.*
.*OAIGet-[0-9]+\.[0-9]+.*
.*LinkScan/[0-9]+\.[0-9]+b Windows.*
.*GrubNG.*
.*LinkLint-checkonly/[0-9]\.[0-9]\.[0-9]+
.*DataCha0s/[0-9]+\.[0-9]+.*
.*DA [0-9]+\.[0-9]+.*
.*EZBLinkcheck/[0-9]+\.[0-9]+.*
.*alef/[0-9]+\.[0-9]+.*
.*Microsoft Data Access Internet Publishing Provider Cache Manager.*
.*GetRight/[0-9]+\.[0-9]+.*
.*JNLP/[0-9]+\.[0-9]+\.[0-9]+.*
.*LinkScan Enterprise/[0-9]+\.[0-9]+b Windows.*
.*Celestial/[0-9]\.[0-9]*.*
.*Download Master.*
.*<a href='http://www\.unchaos\.com/'> UnChaos </a> From Chaos To Order Hybrid Web Search Engine\.\(vadim_gonchar@unchaos\.com\).*
.*<http://www\.sygol\.com/> http://www\.sygol\.com.*
.*\*/Nutch-0\.9-dev.*
.*-DIE-KRAEHE- META-SEARCH-ENGINE/1\.1 http://www\.die-kraehe\.de.*
.*192\.comAgent.*
.*4anything\.com LinkChecker v2\.0.*
.*8484 Boston Project v 1\.0.*
.*A-Online Search.*
.*Aberja Checkomat.*
.*About/0\.1libwww-perl/5\.47.*
.*accoona.*
.*Ack \(http://www\.ackerm\.com/\).*
.*Acorn/Nutch-0\.9 \(Non-Profit Search Engine; acorn\.isara\.org; acorn at isara dot org\).*
.*AESOP_com_SpiderMan.*
.*agadine/1\.x\.x \(\+http://www\.agada\.de\).*
.*AgentName/0\.1 libwww-perl/5\.48.*
.*Aladin/3\.324.*
.*Aleksika Spider/1\.0 \(\+http://www\.aleksika\.com/\).*
.*Allesklar/0\.1 libwww-perl/5\.46.*
.*Allrati/1\.1 \(\+\).*
.*AltaVista Intranet V2\.0 AVS EVAL search@freeit\.com.*
.*AltaVista Intranet V2\.0 Compaq Altavista Eval sveand@altavista\.net.*
.*AnnoMille spider 0\.1 alpha - http://www\.annomille\.it.*
.*AnswerBus \(http://www\.answerbus\.com/\).*
.*AnzwersCrawl/2\.0 \(anzwerscrawl@anzwers\.com\.au;Engine\).*
.*Apexoo Spider 1\.x.*
.*Aport.*
.*appie 1\.1 \(www\.walhello\.com\).*
.*Arachnoidea \(arachnoidea@euroseek\.com\).*
.*ArchitextSpider.*
.*Arikus_Spider.*
.*ASAHA Search Engine Turkey V\.001 \(http://www\.asaha\.com/\).*
.*Asahina-Antenna/1\.x.*
.*Asahina-Antenna/1\.x \(libhina\.pl/x\.x ; libtime\.pl/x\.x\).*
.*ask\.24x\.info.*
.*ASPSeek/1\.2\.5.*
.*ASPseek/1\.2\.9d.*
.*ASPSeek/1\.2\.x.*
.*ASPSeek/1\.2\.xa.*
.*ASPseek/1\.2\.xx.*
.*ASPSeek/1\.2\.xxpre.*
.*ASSORT/0\.10.*
.*asterias/2\.0.*
.*Atomic_Email_Hunter/4\.0.*
.*Atomz/1\.0.*
.*atSpider/1\.0.*
.*augurfind.*
.*augurnfind V-1\.x.*
.*autoemailspider.*
.*autowebdir 1\.1 \(www\.autowebdir\.com\).*
.*AV Fetch 1\.0.*
.*AVSearch-1\.0\(peter\.turney@nrc\.ca\).*
.*AVSearch-3\.0\(AltaVista/AVC\).*
.*BabalooSpider/1\.3 \(BabalooSpider; http://www\.babaloo\.si; spider@babaloo\.si\).*
.*BaiduImagespider\+\(\+http://www\.baidu\.jp/search/s308\.html\).*
.*BaiDuSpider.*
.*Baiduspider\+\(\+http://help\.baidu\.jp/system/05\.html\).*
.*Baiduspider\+\(\+http://www\.baidu\.com/search/spider\.htm\).*
.*Baiduspider\+\(\+http://www\.baidu\.com/search/spider_jp\.html\).*
.*bdcindexer_2\.6\.2 \(research@bdc\).*
.*BDFetch.*
.*BIGLOTRON \(Beta 2;GNU/Linux\).*
.*Bigsearch\.ca/Nutch-x\.x-dev \(Bigsearch\.ca Internet Spider; http://www\.bigsearch\.ca/; info@enhancededge\.com\).*
.*BlackWidow.*
.*Blaiz-Bee/1\.0 \(\+http://www\.blaiz\.net\).*
.*Blaiz-Bee/2\.00\.8222 \(BE Internet Search Engine http://www\.rawgrunt\.com\).*
.*Blaiz-Bee/2\.00\.xxxx \(\+http://www\.blaiz\.net\).*
.*Bloglines Title Fetch/1\.0 \(http://www\.bloglines\.com\).*
.*Bloglines-Images/0\.1 \(http://www\.bloglines\.com\).*
.*Bloglines/3\.1 \(http://www\.bloglines\.com\).*
.*Blogpulse \(info@blogpulse\.com\).*
.*BlogPulseLive \(support@blogpulse\.com\).*
.*BlogSearch/1\.x \+http://www\.icerocket\.com/.*
.*blogWatcher_Spider/0\.1 \(http://www\.lr\.pi\.titech\.ac\.jp/blogWatcher/\).*
.*BlogzIce/1\.0 \(\+http://icerocket\.com; rhodes@icerocket\.com\).*
.*BlogzIce/1\.0 \+http://www\.icerocket\.com/.*
.*BPImageWalker/2\.0 \(www\.bdbrandprotect\.com\).*
.*BravoBrian SpiderEngine MarcoPolo.*
.*BSDSeek/1\.0.*
.*BullsEye.*
.*bumblebee@relevare\.com.*
.*Buscaplus Robi/1\.0 \(http://www\.buscaplus\.com/robi/\).*
.*bwh3_user_agent.*
.*carleson/1\.0.*
.*Catall Spider.*
.*ccubee/x\.x.*
.*Ceramic Tile Installation Guide \(http://www\.floorstransformed\.com\).*
.*cfetch/1\.0.*
.*China Local Browse 2\.6.*
.*Claymont\.com.*
.*CloakDetect/0\.9 \(\+http://fulltext\.seznam\.cz/\).*
.*combine/0\.0.*
.*Combine/2\.0 http://combine\.it\.lth\.se/.*
.*Combine/3 http://combine\.it\.lth\.se/.*
.*Combine/x\.0.*
.*ContentSmartz.*
.*Convera Internet Spider V6\.x.*
.*CougarSearch/0\.x \(\+http://www\.cougarsearch\.com/faq\.shtml\).*
.*CrawlConvera0\.1 \(CrawlConvera@yahoo\.com\).*
.*csci_b659/0\.13.*
.*CurryGuide SiteScan 1\.1.*
.*Custom Spider www\.bisnisseek\.com /1\.0.*
.*CydralSpider/1\.x \(Cydral Web Image Search; http://www\.cydral\.com\).*
.*CydralSpider/3\.0 \(Cydral Image Search; http://www\.cydral\.com\).*
.*DataCha0s/2\.0.*
.*DataCha0s/2\.0.*
.*DataFountains/DMOZ Downloader.*
.*DataFountains/Dmoz Downloader \(http://ivia\.ucr\.edu/useragents\.shtml\).*
.*DataFountains/DMOZ Feature Vector Corpus Creator \(http://ivia\.ucr\.edu/useragents\.shtml\).*
.*DataparkSearch/4\.xx \(http://www\.dataparksearch\.org/\).*
.*DataSpear/1\.0 \(Spider; http://www\.dataspear\.com/spider\.html; spider@dataspear\.com\).*
.*dbDig\(http://www\.prairielandconsulting\.com\).*
.*DBrowse 1\.4b.*
.*DBrowse 1\.4d.*
.*deepak-USC/ISI.*
.*DeepIndex.*
.*DeepIndex \(www\.en\.deepindex\.com\).*
.*DeepIndexer\.ca.*
.*Denmex websearch \(http://search\.denmex\.com\).*
.*dev-spider2\.searchpsider\.com/1\.3b.*
.*DiaGem/1\.1 \(http://www\.skyrocket\.gr\.jp/diagem\.html\).*
.*Diamond/x\.0.*
.*Digger/1\.0 JDK/1\.3\.0rc3.*
.*DigOut4U.*
.*DittoSpyder.*
.*DoCoMo/1\.0/Nxxxi/c10.*
.*DoCoMo/1\.0/Nxxxi/c10/TB.*
.*DoCoMo/2\.0 P900iV\(c100;TB;W24H11\) .*
.*DoCoMo/2\.0 SH902i \(compatible; Y!J-SRD/1\.0; http://help\.yahoo\.co\.jp/help/jp/search/indexing/indexing-27\.html\).*
.*DoCoMo/2\.0/SO502i \(compatible; Y!J-SRD/1\.0; http://help\.yahoo\.co\.jp/help/jp/search/indexing/indexing-27\.html\).*
.*Download-Tipp Linkcheck \(http://download-tipp\.de/\).*
.*DSurf15a 01.*
.*DSurf15a 71.*
.*DSurf15a 81.*
.*DSurf15a VA.*
.*dtSearchSpider.*
.*e-sense 1\.0 ea\(www\.vigiltech\.com/esensedisclaim\.html\).*
.*EARTHCOM\.info/1\.x \[www\.earthcom\.info\].*
.*EARTHCOM\.info/1\.xbeta \[www\.earthcom\.info\].*
.*EasyDL/3\.xx.*
.*EBrowse 1\.4b.*
.*EchO!/2\.0.*
.*Educate Search VxB.*
.*ejupiter\.com.*
.*EmailSiphon.*
.*EmailSpider.*
.*EmailWolf 1\.00.*
.*Enfish Tracker.*
.*Enterprise_Search/1\.0.*
.*Enterprise_Search/1\.0\.xxx.*
.*Enterprise_Search/1\.00\.xxx;MSSQL \(http://www\.innerprise\.net/es-spider\.asp\).*
.*envolk/1\.7 \(\+http://www\.envolk\.com/envolkspiderinfo\.php\).*
.*envolk\[ITS\]spider/1\.6\(\+http://www\.envolk\.com/envolkspider\.html\).*
.*ESISmartSpider.*
.*eStyleSearch 4 \(compatible; MSIE 6\.0; Windows NT 5\.0\).*
.*ESurf15a 15.*
.*eventax/1\.3 \(eventax; http://www\.eventax\.de/; info@eventax\.de\).*
.*exactseek\.com.*
.*Exalead NG/MimeLive Client \(convert/http/0\.120\).*
.*Excalibur Internet Spider V6\.5\.4.*
.*ExtractorPro.*
.*EyeCatcher \(Download-tipp\.de\)/1\.0.*
.*FAST FirstPage retriever \(compatible; MSIE 5\.5; Mozilla/4\.0\).*
.*FastBug http://www\.ay-up\.com.*
.*Favcollector/2\.0 \(info@favcollector\.com http://www\.favcollector\.com/\).*
.*Feed24\.com.*
.*FeedChecker/0\.01.*
.*Feedfetcher-Google; \(\+http://www\.google\.com/feedfetcher\.html\).*
.*FeedHub FeedDiscovery/1\.0 \(http://www\.feedhub\.com\).*
.*FeedHub MetaDataFetcher/1\.0 \(http://www\.feedhub\.com\).*
.*FFC Trap Door Spider.*
.*Filtrbox/1\.0.*
.*findlinks/x\.xxx \(\+http://wortschatz\.uni-leipzig\.de/findlinks/\) .*
.*Firefly/1\.0.*
.*Firefly/1\.0 \(compatible; Mozilla 4\.0; MSIE 5\.5\).*
.*Firefox \(kastaneta03@hotmail\.com\).*
.*Firefox_1\.0\.6 \(kasparek@naparek\.cz\).*
.*FirstGov\.gov Search - POC:firstgov\.webmasters@gsa\.gov.*
.*Flexum spider.*
.*Flexum/2\.0.*
.*flunky.*
.*FocusedSampler/1\.0.*
.*Folkd\.com Spider/0\.1 beta 1 \(www\.folkd\.com\).*
.*Francis/1\.0 \(francis@neomo\.de http://www\.neomo\.de/\).*
.*Franklin Locator 1\.8.*
.*FreeFind\.com-SiteSearchEngine/1\.0 \(http://freefind\.com; spiderinfo@freefind\.com\).*
.*FSurf15a 01.*
.*FuseBulb\.Com.*
.*FyberSpider \(\+http://www\.fybersearch\.com/fyberspider\.php\).*
.*GammaSpider/1\.0.*
.*gazz/x\.x \(gazz@nttrd\.com\).*
.*gigabaz/3\.1x \(baz@gigabaz\.com; http://gigabaz\.com/gigabaz/\).*
.*GNODSPIDER \(www\.gnod\.net\).*
.*Goblin/0\.9 \(http://www\.goguides\.org/\).*
.*Goblin/0\.9\.x \(http://www\.goguides\.org/goblin-info\.html\).*
.*GoForIt\.com.*
.*gonzo1\[P\] \+http://www\.suchen\.de/popups/faq\.jsp.*
.*gonzo2\[P\] \+http://www\.suchen\.de/faq\.html.*
.*Goofer/0\.2.*
.*GrapeFX/0\.3 libwww/5\.4\.0.*
.*Gromit/1\.0.*
.*grub-client.*
.*Guestbook Auto Submitter.*
.*Gulliver/1\.3.*
.*Harvest-NG/1\.0\.2.*
.*Hatena Pagetitle Agent/1\.0.*
.*Hatena RSS/0\.3 \(http://r\.hatena\.ne\.jp\).*
.*hbtronix\.spider\.2 -- http://hbtronix\.de/spider\.php.*
.*Helix/1\.x \( http://www\.sitesearch\.ca/helix/\).*
.*Hippias/0\.9 Beta.*
.*HitList.*
.*Hitwise Spider v1\.0 http://www\.hitwise\.com.*
.*holmes/3\.9 \(onet\.pl\).*
.*holmes/3\.xx \(OnetSzukaj/5\.0; \+http://szukaj\.onet\.pl\).*
.*holmes/x\.x.*
.*HomePageSearch\(hpsearch\.uni-trier\.de\).*
.*HPL/Nutch-0\.9 -.*
.*htdig/3\.1\.6 \(http://computerorgs\.com\).*
.*htdig/3\.1\.6 \(unconfigured@htdig\.searchengine\.maintainer\).*
.*htdig/3\.1\.x \(root@localhost\).*
.*http://Ask\.24x\.Info/ \(http://narres\.it/\).*
.*http://www\.istarthere\.com_spider@istarthere\.com.*
.*http://www\.monogol\.de.*
.*http://www\.trendtech\.dk/spider\.asp\).*
.*IAArchiver-1\.0.*
.*iaskspider2 \(iask@staff\.sina\.com\.cn\).*
.*ia_archiver.*
.*ia_archiver-web\.archive\.org.*
.*ia_archiver/1\.6.*
.*ichiro/x\.0 \(ichiro@nttr\.co\.jp\).*
.*ICRA_label_spider/x\.0.*
.*ideare - SignSite/1\.x.*
.*iFeed\.jp/2\.0 \(www\.psychedelix\.com/agents/agents\.rss; 0 subscribers\).*
.*igdeSpyder \(compatible; igde\.ru; \+http://igde\.ru/doc/tech\.html\).*
.*ilial/Nutch-0\.9-dev.*
.*IlTrovatore-Setaccio \( http://www\.iltrovatore\.it\).*
.*IlTrovatore-Setaccio/1\.2 \( http://www\.iltrovatore\.it/aiuto/faq\.html\).*
.*iltrovatore-setaccio/1\.2-dev \(spidering; http://www\.iltrovatore\.it/aiuto/\.\.\.\.\.\).*
.*ImageWalker/2\.0 \(www\.bdbrandprotect\.com\).*
.*IncyWincy data gatherer\(webmaster@loopimprovements\.com.*
.*IncyWincy\(http://www\.look\.com\).*
.*Industry Program 1\.0\.x.*
.*Inet library.*
.*info@pubblisito\.com- \(http://www\.pubblisito\.com\) il Sud dei Motori di Ricerca.*
.*InfoFly/1\.0 \(http://www\.versions-project\.org/\).*
.*INFOMINE/8\.0 Adders.*
.*INFOMINE/8\.0 RemoteServices.*
.*InfoSeek Sidewinder/0\.9.*
.*InfoSeek Sidewinder/1\.0A.*
.*InfoSeek Sidewinder/1\.1A.*
.*Infoseek SideWinder/1\.45 \(Compatible; MSIE 10\.0; UNIX\).*
.*Infoseek SideWinder/2\.0B \(Linux 2\.4 i686\).*
.*Inktomi Search.*
.*Insitor\.com search and find world wide!.*
.*Insitornaut.*
.*Internet Ninja x\.0.*
.*InternetSeer\.com.*
.*ISC Systems iRc Search 2\.1.*
.*IWAgent/ 1\.0 - www\.brandprotect\.com.*
.*Jack.*
.*JobSpider_BA/1\.1.*
.*k2spider.*
.*kalooga/kalooga-4\.0-dev-datahouse \(Kalooga; http://www\.kalooga\.com; info@kalooga\.com\).*
.*Kenjin Spider.*
.*Kevin http://dznet\.com/kevin/.*
.*Kevin http://websitealert\.net/kevin/.*
.*KE_1\.0/2\.0 libwww/5\.2\.8.*
.*KIT-Fireball/2\.0.*
.*KIT-Fireball/2\.0 \(compatible; Mozilla 4\.0; MSIE 5\.5\).*
.*KnowItAll\(knowitall@cs\.washington\.edu\).*
.*Knowledge\.com/0\.x.*
.*kulturarw3/0\.1.*
.*larbin \(samualt9@bigfoot\.com\).*
.*LARBIN-EXPERIMENTAL \(efp@gmx\.net\).*
.*larbin_2\.1\.1 larbin2\.1\.1@somewhere\.com.*
.*larbin_2\.2\.0 \(crawl@compete\.com\).*
.*larbin_2\.2\.1_de_Viennot \(Laurent\.Viennot@inria\.fr\).*
.*larbin_2\.2\.2 \(sugayama@lab7\.kuis\.kyoto-u\.ac\.jp\).*
.*larbin_2\.2\.2_guillaume \(guillaume@liafa\.jussieu\.fr\).*
.*larbin_2\.6\.0 \(larbin2\.6\.0@unspecified\.mail\).*
.*larbin_2\.6\.1 \(larbin2\.6\.1@unspecified\.mail\).*
.*larbin_2\.6\.2 \(hamasaki@grad\.nii\.ac\.jp\).*
.*larbin_2\.6\.2 \(larbin2\.6\.2@unspecified\.mail\).*
.*larbin_2\.6\.2 \(listonATccDOTgatechDOTedu\).*
.*larbin_2\.6\.2 \(pimenas@systems\.tuc\.gr\).*
.*larbin_2\.6\.2 \(tom@lemurconsulting\.com\).*
.*larbin_2\.6\.2 \(vitalbox1@hotmail\.com\).*
.*larbin_2\.6\.3 \(wgao@genieknows\.com\).*
.*larbin_2\.6\.3_for_\(http://cosco\.hiit\.fi/search/\) tsilande@hiit\.fi.*
.*larbin_2\.6_basileocaml \(basile\.starynkevitch@cea\.fr\).*
.*larbin_devel \(http://pauillac\.inria\.fr/~ailleret/prog/larbin/\).*
.*LECodeChecker/3\.0 libgetdoc/1\.0.*
.*LEIA/2\.90.*
.*LetsCrawl\.com/1\.0 \+http://letscrawl\.com/.*
.*Libby_1\.1/libwww-perl/5\.47.*
.*LibertyW \(\+http://www\.lw01\.com\).*
.*libWeb/clsHTTP -- hiongun@kt\.co\.kr.*
.*libwww-perl/5\.41.*
.*libwww-perl/5\.45.*
.*libwww-perl/5\.48.*
.*libwww-perl/5\.52 FP/2\.1.*
.*libwww-perl/5\.52 FP/4\.0.*
.*libwww-perl/5\.65.*
.*libwww-perl/5\.800.*
.*libwww/5\.3\.2.*
.*Lincoln State Web Browser.*
.*Links 2\.0 \(http://gossamer-threads\.com/scripts/links/\).*
.*Links SQL \(http://gossamer-threads\.com/scripts/links-sql/\).*
.*LinkScan/9\.0g Unix.*
.*LinkScan/x\.x Unix.*
.*LiveTrans/Nutch-0\.9 \(maintainer: cobain at iis dot sinica dot edu dot tw; http://wkd\.iis\.sinica\.edu\.tw/LiveTrans/\).*
.*lmspider \(lmspider@scansoft\.com\).*
.*LNSpiderguy.*
.*Lockstep Spider/1\.0.*
.*Look\.com.*
.*Lovel as 1\.0 \( \+http://www\.everatom\.com\).*
.*LTI/LemurProject Nutch Spider/Nutch-1\.0-dev \(Research spider using Nutch; http://www\.lemurproject\.org; mhoy@cs\.cmu\.edu\).*
.*lwp-trivial/1\.32.*
.*lwp-trivial/1\.34.*
.*lwp-trivial/1\.34.*
.*LWP::Simple/5\.22.*
.*LWP::Simple/5\.36.*
.*LWP::Simple/5\.48.*
.*LWP::Simple/5\.50.*
.*LWP::Simple/5\.51.*
.*LWP::Simple/5\.53.*
.*LWP::Simple/5\.63.*
.*LWP::Simple/5\.803.*
.*Lycos_Spider_\(modspider\).*
.*Lycos_Spider_\(T-Rex\).*
.*Lynx/2\.8\.4rel\.1 libwww-FM/2\.14 SSL-MM/1\.4\.1 OpenSSL/0\.9\.6c \(human-guided@lerly\.net\).*
.*Mac Finder 1\.0\.xx.*
.*Mackster\( http://www\.ukwizz\.com \).*
.*Mail\.Ru/1\.0.*
.*mailto:webcraft@bea\.com.*
.*mammoth/1\.0 \( http://www\.sli-systems\.com/\).*
.*MantraAgent.*
.*MapoftheInternet\.com \( http://MapoftheInternet\.com\).*
.*Marketwave Hit List.*
.*Martini.*
.*MARTINI.*
.*Marvin v0\.3.*
.*MaSagool/1\.0 \(MaSagool; http://sagool\.jp/; info@sagool\.jp\).*
.*MasterSeek.*
.*Mata Hari/2\.00 .*
.*MediaSearch/0\.1.*
.*MegaSheep v1\.0 \(www\.searchuk\.com internet sheep\).*
.*Megite2\.0 \(http://www\.megite\.com\).*
.*Mercator-1\.x.*
.*Mercator-2\.0.*
.*Mercator-Scrub-1\.1.*
.*MetaGer-LinkChecker.*
.*MetaGer_PreChecker0\.1.*
.*Metaspinner/0\.01 \(Metaspinner; http://www\.meta-spinner\.de/; support@meta-spinner\.de/\).*
.*metatagsdir/0\.7 \(\+http://metatagsdir\.com/directory/\).*
.*MFC Foundation Class Library 4\.0.*
.*MicroBaz.*
.*Microsoft Small Business Indexer.*
.*Microsoft URL Control - 6\.00\.8xxx.*
.*Missauga Locate 1\.0\.0.*
.*Missigua Locator 1\.9.*
.*Missouri College Browse.*
.*Miva \(AlgoFeedback@miva\.com\).*
.*Mizzu Labs 2\.2.*
.*MnogoSearch/3\.2\.xx.*
.*Mo College 1\.9.*
.*moget/x\.x \(moget@goo\.ne\.jp\).*
.*mogimogi/1\.0.*
.*Mouse-House/7\.4 \(spider_monkey spider info at www\.mobrien\.com/sm\.shtml\).*
.*Mozilla \(Mozilla@somewhere\.com\).*
.*Mozilla/2\.0 \(compatible; Ask Jeeves\).*
.*Mozilla/2\.0 \(compatible; Ask Jeeves/Teoma\).*
.*Mozilla/2\.0 \(compatible; Ask Jeeves/Teoma; http://about\.ask\.com/en/docs/about/webmasters\.shtml\) .*
.*Mozilla/2\.0 \(compatible; Ask Jeeves/Teoma; http://sp\.ask\.com/docs/about/tech_crawling\.html\).*
.*Mozilla/2\.0 \(compatible; EZResult -- Internet Search Engine\).*
.*Mozilla/2\.0 \(compatible; NEWT ActiveX; Win32\).*
.*Mozilla/2\.0 \(compatible; T-H-U-N-D-E-R-S-T-O-N-E\).*
.*Mozilla/3\.0 \(compatible; Fluffy the spider; http://www\.searchhippo\.com/; info@searchhippo\.com\).*
.*Mozilla/3\.0 \(compatible; Indy Library\).*
.*Mozilla/3\.0 \(compatible; MuscatFerret/1\.5\.4; claude@euroferret\.com\).*
.*Mozilla/3\.0 \(compatible; MuscatFerret/1\.5; olly@muscat\.co\.uk\).*
.*Mozilla/3\.0 \(compatible; MuscatFerret/1\.6\.x; claude@euroferret\.com\).*
.*Mozilla/3\.0 \(compatible; scan4mail \(advanced version\) http://www\.peterspages\.net/\?scan4mail\).*
.*Mozilla/3\.0 \(compatible; Webinator-DEV01\.home\.iprospect\.com/2\.56\).*
.*Mozilla/3\.0 \(compatible; Webinator-indexer\.cyberalert\.com/2\.56\).*
.*Mozilla/3\.0 \(Slurp\.so/Goo; slurp@inktomi\.com; http://www\.inktomi\.com/slurp\.html\).*
.*Mozilla/3\.0 \(Slurp/cat; slurp@inktomi\.com; http://www\.inktomi\.com/slurp\.html\).*
.*Mozilla/3\.0 \(Slurp/si; slurp@inktomi\.com; http://www\.inktomi\.com/slurp\.html\).*
.*Mozilla/3\.0 \(Vagabondo/1\.x MT; webagent@wise-guys\.nl; http://webagent\.wise-guys\.nl/\).*
.*Mozilla/3\.01 \(Compatible; Links2Go Similarity Engine\).*
.*Mozilla/4\.0 \(agadine3\.0\) www\.agada\.de.*
.*Mozilla/4\.0 \(compatible: AstraSpider V\.2\.1 : astrafind\.com\).*
.*Mozilla/4\.0 \(compatible; Advanced Email Extractor v2\.xx\).*
.*Mozilla/4\.0 \(compatible; B_L_I_T_Z_B_O_T\).*
.*Mozilla/4\.0 \(compatible; DAUMOA-video; \+http://ws\.daum\.net/aboutkr\.html\).*
.*Mozilla/4\.0 \(compatible; grub-client-0\.2\.x; Crawl your stuff with http://grub\.org\).*
.*Mozilla/4\.0 \(compatible; grub-client-0\.3\.x; Crawl your own stuff with http://grub\.org\).*
.*Mozilla/4\.0 \(compatible; grub-client-2\.x\).*
.*Mozilla/4\.0 \(compatible; Iplexx Spider/1\.0 http://www\.iplexx\.at\).*
.*Mozilla/4\.0 \(compatible; MSIE 4\.01; Vonna\.com b o t\).*
.*Mozilla/4\.0 \(compatible; MSIE 5\.01; Windows NT 5\.0\) \(samualt9@bigfoot\.com\).*
.*Mozilla/4\.0 \(compatible; MSIE 5\.0; Windows NT; DigExt; DTS Agent.*
.*Mozilla/4\.0 \(compatible; MSIE 5\.0; www\.galaxy\.com; www\.psychedelix\.com\).*
.*Mozilla/4\.0 \(compatible; MSIE 5\.0; YANDEX\).*
.*Mozilla/4\.0 \(compatible; MSIE 5\.5; Windows NT 4\.0; QXW03018\).*
.*Mozilla/4\.0 \(compatible; MSIE 6\.0; MSIE 5\.5; Windows NT 5\.1\) Skampy/0\.9\.x \[en\].*
.*Mozilla/4\.0 \(compatible; MSIE 6\.0; TargetSeek/1\.0; \+http://www\.targetgroups\.net/TargetSeek\.html\).*
.*Mozilla/4\.0 \(compatible; MSIE 6\.0; Windows NT 5\.0; ODP entries t_st; http://tuezilla\.de/t_st-odp-entries-agent\.html\).*
.*Mozilla/4\.0 \(compatible; MSIE 6\.0; Windows NT 5\.0; ODP links test; http://tuezilla\.de/test-odp-links-agent\.html\).*
.*Mozilla/4\.0 \(compatible; MSIE 6\.0; Windows NT 5\.1; heritrix/1\.3\.0 http://www\.cs\.washington\.edu/research/networking/websys/\).*
.*Mozilla/4\.0 \(compatible; SpeedySpider; www\.entireweb\.com\).*
.*Mozilla/4\.0 \(compatible; www\.galaxy\.com\).*
.*Mozilla/4\.0 \(JemmaTheTourist;http://www\.activtourist\.com\).*
.*Mozilla/4\.0 \(Sleek Spider/1\.2\).*
.*Mozilla/4\.0 efp@gmx\.net.*
.*Mozilla/4\.0 \[en\] \(Ask Jeeves Corporate Spider\).*
.*Mozilla/4\.08 \[en\] \(Win98; U ;Nav\).*
.*Mozilla/4\.6 \[en\] \(http://www\.cnet\.com/\).*
.*Mozilla/4\.7 \(compatible; http://eidetica\.com/spider\).*
.*Mozilla/4\.7 \(compatible; Intelliseek; http://www\.intelliseek\.com\).*
.*Mozilla/4\.7 \(compatible; Whizbang\).*
.*Mozilla/4\.72 \[en\] \(BACS http://www\.ba\.be\).*
.*Mozilla/5\.0 \(\+http://www\.eurekster\.com/mammoth\) Mammoth/0\.1.*
.*Mozilla/5\.0 \(\+http://www\.sli-systems\.com/\) Mammoth/0\.1.*
.*Mozilla/5\.0 \(compatible; \+http://www\.evri\.com/evrinid\).*
.*Mozilla/5\.0 \(compatible; 008/0\.83; http://www\.80legs\.com/spider\.html;\) Gecko/2008032620.*
.*Mozilla/5\.0 \(compatible; Abonti/0\.8 - http://www\.abonti\.com\).*
.*Mozilla/5\.0 \(compatible; Ask Jeeves/Teoma;  http://about\.ask\.com/en/docs/about/webmasters\.shtml\).*
.*Mozilla/5\.0 \(compatible; Charlotte/1\.0b; charlotte@betaspider\.com\).*
.*Mozilla/5\.0 \(compatible; Charlotte/1\.0b; http://www\.searchme\.com/support/\).*
.*Mozilla/5\.0 \(compatible; Crawling jpeg; http://www\.yama\.info\.waseda\.ac\.jp\).*
.*Mozilla/5\.0 \(compatible; de/1\.13\.2 \+http://www\.de\.com\).*
.*Mozilla/5\.0 \(compatible; DNS-Digger-Explorer/1\.0; \+http://www\.dnsdigger\.com\).*
.*Mozilla/5\.0 \(compatible; DNS-Digger/1\.0; \+http://www\.dnsdigger\.com\).*
.*Mozilla/5\.0 \(compatible; EARTHCOM\.info/2\.01; http://www\.earthcom\.info\).*
.*Mozilla/5\.0 \(compatible; EARTHCOM/2\.2; \+http://enter4u\.eu\).*
.*mozilla/5\.0 \(compatible; heritrix/1\.0\.4 http://innovationblog\.com\).*
.*Mozilla/5\.0 \(compatible; heritrix/1\.10\.2 \+http://i\.stanford\.edu/\).*
.*Mozilla/5\.0 \(compatible; heritrix/1\.12\.1 \+http://newstin\.com/\).*
.*Mozilla/5\.0 \(compatible; heritrix/1\.12\.1 \+http://www\.page-store\.com\).*
.*Mozilla/5\.0 \(compatible; heritrix/1\.12\.1 \+http://www\.page-store\.com\) \[email:paul@page-store\.com\].*
.*Mozilla/5\.0 \(compatible; heritrix/1\.4\.0 \+http://www\.chepi\.net\).*
.*Mozilla/5\.0 \(compatible; heritrix/1\.4t http://www\.truveo\.com/\).*
.*Mozilla/5\.0 \(compatible; heritrix/1\.5\.0 http://www\.l3s\.de/~kohlschuetter/projects/crawling/\).*
.*Mozilla/5\.0 \(compatible; heritrix/1\.5\.0-200506231921 http://pandora\.nla\.gov\.au/crawl\.html\).*
.*Mozilla/5\.0 \(compatible; heritrix/1\.6\.0 http://www\.worio\.com/\).*
.*Mozilla/5\.0 \(compatible; heritrix/1\.7\.0 \+http://www\.greaterera\.com/\).*
.*Mozilla/5\.0 \(compatible; heritrix/2\.0\.0-RC1 \+http://www\.aol\.com\).*
.*Mozilla/5\.0 \(compatible; Hermit Search\. Com; \+http://www\.hermitsearch\.com\).*
.*Mozilla/5\.0 \(compatible; HyperixScoop/1\.3; \+http://www\.hyperix\.com\).*
.*Mozilla/5\.0 \(compatible; InterseekWeb/3\.x\).*
.*Mozilla/5\.0 \(compatible; LemSpider 0\.1\).*
.*Mozilla/5\.0 \(compatible; OnetSzukaj/5\.0; http://szukaj\.onet\.pl\).*
.*Mozilla/5\.0 \(compatible; pogodak\.ba/3\.x\).*
.*Mozilla/5\.0 \(compatible; Pogodak\.hr/3\.1\).*
.*Mozilla/5\.0 \(compatible; ScoutJet; \+http://www\.scoutjet\.com/\).*
.*Mozilla/5\.0 \(compatible; Scrubby/2\.2;  http://www\.scrubtheweb\.com/\).*
.*Mozilla/5\.0 \(compatible; Theophrastus/x\.x; http://users\.cs\.cf\.ac\.uk/N\.A\.Smith/theophrastus\.php\).*
.*Mozilla/5\.0 \(compatible; TridentSpider/3\.1\).*
.*Mozilla/5\.0 \(compatible; WoW Lemmings Kathune/2\.0;http://www\.wowlemmings\.com/kathune\.html\).*
.*Mozilla/5\.0 \(compatible; Yahoo! DE Slurp; http://help\.yahoo\.com/help/us/ysearch/slurp\).*
.*Mozilla/5\.0 \(compatible; Yahoo! Slurp China; http://misc\.yahoo\.com\.cn/help\.html\).*
.*Mozilla/5\.0 \(compatible; Yahoo! Slurp; http://help\.yahoo\.com/help/us/ysearch/slurp\).*
.*Mozilla/5\.0 \(compatible; Yoono; http://www\.yoono\.com/\).*
.*Mozilla/5\.0 \(compatible; zermelo \+http://www\.powerset\.com\) \[email:paul@page-store\.com,crawl@powerset\.com\].*
.*Mozilla/5\.0 \(Slurp/cat; slurp@inktomi\.com; http://www\.inktomi\.com/slurp\.html\).*
.*Mozilla/5\.0 \(Slurp/si; slurp@inktomi\.com; http://www\.inktomi\.com/slurp\.html\).*
.*Mozilla/5\.0 \(Version: xxxx Type:xx\).*
.*Mozilla/5\.0 \(wgao@genieknows\.com\).*
.*Mozilla/5\.0 URL-Spider.*
.*Mozilla/5\.0 usww\.com-Spider-for-w8\.net.*
.*Mozilla/5\.0 wgao@genieknows\.com.*
.*MSNPTC/1\.0.*
.*MultiText/0\.1.*
.*MusicWalker2\.0 \( http://www\.somusical\.com\).*
.*MVAClient.*
.*NameOfAgent \(CMS Spider\).*
.*NASA Search 1\.0.*
.*NationalDirectory-WebSpider/1\.3.*
.*NationalDirectoryAddURL/1\.0.*
.*NCSA Beta 1 \(http://vias\.ncsa\.uiuc\.edu/viasarchivinginformation\.html\).*
.*NEC Research Agent -- compuman at research\.nj\.nec\.com.*
.*NetLookout/2\.24.*
.*Netluchs/0\.8-dev \( ; http://www\.netluchs\.de/; ___don't___spam_me_@netluchs\.de\).*
.*NetResearchServer\(http://www\.look\.com\).*
.*NetSprint -- 2\.0.*
.*NetZippy.*
^NG/1\.0$
.*NG/4\.0\.1229.*
.*NITLE Blog Spider/0\.01.*
.*Noago Spider.*
.*Nokia6610/1\.0 \(3\.09\) Profile/MIDP-1\.0 Configuration/CLDC-1\.0 \(compatible;YahooSeeker/M1A1-R2D2; http://help\.yahoo\.com/help/us/ysearch/crawling/crawling-01\.html\).*
.*Norbert the Spider\(Burf\.com\).*
.*noyona_0_1.*
.*Nsauditor/1\.x.*
.*nsyght\.com/Nutch-1\.0-dev \(nsyght\.com; Nsyght\.com\).*
.*nsyght\.com/Nutch-x\.x \(nsyght\.com; search\.nsyght\.com\).*
.*nuSearch Spider <a href='http://www\.nusearch\.com'>www\.nusearch\.com</a> \(compatible; MSIE 4\.01\).*
.*NuSearch Spider \(compatible; MSIE 6\.0\).*
.*NuSearch Spider www\.nusearch\.com.*
.*Nutch.*
.*nutchsearch/Nutch-0\.9 \(Nutch Search 1\.0; herceg_novi at yahoo dot com\).*
.*NutchVinegarCrawl/Nutch-0\.8\.1 \(Vinegar; http://www\.cs\.washington\.edu; eytanadar at gmail dot com\).*
.*Ocelli/1\.x \(http://www\.globalspec\.com/Ocelli\).*
.*Octora Beta - www\.octora\.com.*
.*Onet\.pl SA- http://szukaj\.onet\.pl.*
.*OntoSpider/1\.0 libwww-perl/5\.65.*
.*OpenAcoon v4\.0\.x \(www\.openacoon\.de\).*
.*OpenISearch/1\.x \(www\.openisearch\.com\).*
.*OpenWebSpider/0\.x\.x \(http://www\.openwebspider\.org\).*
.*OpenWebSpider/x.*
.*Oracle Ultra Search.*
.*OrangeSpider.*
.*ozelot/2\.7\.3 \(Search engine indexer; www\.flying-cat\.de/ozelot; ozelot@flying-cat\.de\).*
.*PADLibrary Spider.*
.*Pagebull http://www\.pagebull\.com/.*
.*page_verifier \(http://www\.securecomputing\.com/goto/pv\).*
.*ParaSite/1\.0b \(http://www\.ianett\.com/parasite/\).*
.*PBrowse 1\.4b.*
.*pd02_1\.0\.0 pd02_1\.0\.0@dzimi@post\.sk.*
.*PEval 1\.4b.*
.*PicoSearch/1\.0.*
.*Piffany_Web_Scraper_v0\.x.*
.*Piffany_Web_Spider_v0\.x.*
.*pipeLiner/0\.3a \(PipeLine Spider;http://www\.pipeline-search\.com/webmaster\.html; webmaster'at'pipeline-search\.com\).*
.*pipeLiner/0\.xx \(PipeLine Spider; http://www\.pipeline-search\.com/webmaster\.html\).*
.*Pita.*
.*PJspider/3\.0 \(pjspider@portaljuice\.com; http://www\.portaljuice\.com\).*
.*Poirot.*
.*Pompos/1\.x http://dir\.com/pompos\.html.*
.*Pompos/1\.x pompos@iliad\.fr.*
.*Popdexter/1\.0.*
.*Port Huron Labs.*
.*PortalBSpider/2\.0 \(spider@portalb\.com\).*
.*PrivacyFinder/1\.1.*
.*Program Shareware 1\.0\.2.*
.*Project XP5 \[2\.03\.07-111203\].*
.*ProWebGuide Link Checker \(http://www\.prowebguide\.com\).*
.*PSurf15a 11.*
.*PSurf15a 51.*
.*PSurf15a VA.*
.*psycheclone.*
.*PubCrawl \(pubcrawl\.stanford\.edu\).*
.*PycURL.*
.*Python-urllib/1\.1x.*
.*Python-urllib/2\.0a1.*
.*Qango\.com Web Directory \(http://www\.qango\.com/\).*
.*QEAVis Agent/Nutch-0\.9 \(Quantitative Evaluation of Academic Websites Visibility; http://nlp\.uned\.es/qeavis.*
.*QPCreep Test Rig \( We are not indexing- just testing \).*
.*QuepasaCreep v0\.9\.1x.*
.*QueryN Metasearch.*
.*rabaz \(rabaz at gigabaz dot com\).*
.*Rational SiteCheck \(Windows NT\).*
.*Reaper \[2\.03\.10-031204\] \(http://www\.sitesearch\.ca/reaper/\).*
.*Reaper/2\.0x \(\+http://www\.sitesearch\.ca/reaper\).*
.*RedKernel WWW-Spider 2/0 \(\+http://www-spider\.redkernel-softwares\.com/\).*
.*rico/0\.1.*
.*RoboCrawl \(http://www\.canadiancontent\.net\).*
.*RoboCrawl \(www\.canadiancontent\.net\).*
.*RoboPal \(http://www\.findpal\.com/\).*
.*Robozilla/1\.0.*
.*Rotondo/3\.1 libwww/5\.3\.1.*
.*RSurf15a 41.*
.*RSurf15a 51.*
.*RSurf15a 81.*
.*sait/Nutch-0\.9 \(SAIT Research; http://www\.samsung\.com\).*
.*SBIder/0\.7 \(SBIder; http://www\.sitesell\.com/sbider\.html; http://support\.sitesell\.com/contact-support\.html\).*
.*SBIder/0\.8-dev \(SBIder; http://www\.sitesell\.com/sbider\.html; http://support\.sitesell\.com/contact-support\.html\).*
.*ScanWeb.*
.*schwarzmann\.biz-Spider_for_paddel\.org\+\(http://www\.innerprise\.net/usp-spider\.asp\).*
.*Scooter-3\.0\.EU.*
.*Scooter-3\.0\.FS.*
.*Scooter-3\.0\.HD.*
.*Scooter-3\.0\.VNS.*
.*Scooter-3\.0QI.*
.*Scooter-3\.2.*
.*Scooter-3\.2\.BT.*
.*Scooter-3\.2\.DIL.*
.*Scooter-3\.2\.EX.*
.*Scooter-3\.2\.JT.*
.*Scooter-3\.2\.NIV.*
.*Scooter-3\.2\.SF0.*
.*Scooter-3\.2\.snippet.*
.*Scooter-3\.3dev.*
.*Scooter-ARS-1\.1.*
.*Scooter-ARS-1\.1-ih.*
.*scooter-venus-3\.0\.vns.*
.*Scooter-W3-1\.0.*
.*Scooter-W3\.1\.2.*
.*Scooter/1\.0.*
.*Scooter/1\.0 scooter@pa\.dec\.com.*
.*Scooter/1\.1 \(custom\).*
.*Scooter/2\.0 G\.R\.A\.B\. V1\.1\.0.*
.*Scooter/2\.0 G\.R\.A\.B\. X2\.0.*
.*Scooter/3\.3.*
.*Scooter/3\.3\.QA\.pczukor.*
.*Scooter/3\.3\.vscooter.*
.*Scooter/3\.3_SF.*
.*Scooter2_Mercator_x-x\.0.*
.*Scooter_bh0-3\.0\.3.*
.*Scooter_trk3-3\.0\.3.*
.*ScoutAbout.*
.*ScoutAnt/0\.1; \+http://www\.ant\.com/what_is_ant\.com/.*
.*scoutmaster.*
.*Scrubby/2\.x \(http://www\.scrubtheweb\.com/\).*
.*Scrubby/3\.0 \(\+http://www\.scrubtheweb\.com/help/technology\.html\).*
.*Search\+.*
.*Search-Engine-Studio.*
.*search\.ch V1\.4.*
.*search\.ch V1\.4\.2 \(spiderman@search\.ch; http://www\.search\.ch\).*
.*Search/1\.0 \(http://www\.innerprise\.net/es-spider\.asp\).*
.*SearchExpress Spider0\.99.*
.*SearchGuild/DMOZ/Experiment \(searchguild@gmail\.com\).*
.*SearchGuild_DMOZ_Experiment \(chris@searchguild\.com\).*
.*Searchmee! Spider v0\.98a.*
.*SearchSight/2\.0 \(http://SearchSight\.com/\).*
.*SearchSpider\.com/1\.1.*
.*Searchspider/1\.2 \(SearchSpider; http://www\.searchspider\.com; webmaster@searchspider\.com\).*
.*SearchTone2\.0 - IDEARE.*
.*Seeker\.lookseek\.com.*
.*Semager/1\.x \(http://www\.semager\.de\).*
.*ShopWiki/1\.0 \( \+http://www\.shopwiki\.com/\).*
.*Siigle Orumcex v\.001 Turkey \(http://www\.siigle\.com\).*
.*silk/1\.0.*
.*silk/1\.0 \(\+http://www\.slider\.com/silk\.htm\)/3\.7.*
.*SiteSpider \+\(http://www\.SiteSpider\.com/\).*
.*SiteTruth\.com site rating system.*
.*SiteXpert.*
.*Skampy/0\.9\.x \(http://www\.skaffe\.com/skampy-info\.html\).*
.*Skimpy/0\.x \(http://www\.skaffe\.com/skampy-info\.html\).*
.*Skywalker/0\.1 \(Skywalker; anonymous; anonymous\).*
.*Slarp/0\.1.*
.*Slider_Search_v1-de.*
.*Slurp/2\.0 \(slurp@inktomi\.com; http://www\.inktomi\.com/slurp\.html\).*
.*Slurp/2\.0-KiteWeekly \(slurp@inktomi\.com; http://www\.inktomi\.com/slurp\.html\).*
.*Slurp/si \(slurp@inktomi\.com; http://www\.inktomi\.com/slurp\.html\).*
.*Slurpy Verifier/1\.0.*
.*SlySearch \(slysearch@slysearch\.com\).*
.*SlySearch/1\.x http://www\.slysearch\.com.*
.*smartwit\.com.*
.*SmiffyDCMetaSpider/1\.0.*
.*sogou develop spider.*
.*Sogou Orion spider/3\.0\(\+http://www\.sogou\.com/docs/help/webmasters\.htm#07\).*
.*sogou spider.*
.*Sogou web spider/3\.0\(\+http://www\.sogou\.com/docs/help/webmasters\.htm#07\).*
.*Sogou web spider/4\.0\(\+http://www\.sogou\.com/docs/help/webmasters\.htm#07\).*
.*sohu agent.*
.*sohu-search.*
.*Sosospider\+\(\+http://help\.soso\.com/webspider\.htm\).*
.*Speedy Spider \(Beta/x\.x; speedy@entireweb\.com\).*
.*Speedy Spider \(Entireweb; Beta/1\.0; http://www\.entireweb\.com/about/search_tech/speedyspider/\).*
.*Speedy_Spider \(http://www\.entireweb\.com\).*
.*Sphere Scout&v4\.0 - scout at sphere dot com.*
.*Sphider.*
.*Spida/0\.1.*
.*Spider-Sleek/2\.0 \(\+http://search-info\.com/linktous\.html\).*
.*spider\.batsch\.com.*
.*spider\.yellopet\.com - www\.yellopet\.com.*
.*SpiderKU/0\.x.*
.*SpiderMan.*
.*SpiderMonkey/7\.0x \(SpiderMonkey\.ca info at http://spidermonkey\.ca/sm\.shtml\).*
.*Spinne/2\.0.*
.*Spinne/2\.0 med.*
.*Spinne/2\.0 med_AH.*
.*Sqworm/2\.9\.81-BETA \(beta_release; 20011102-760; i686-pc-linux-gnu\).*
.*Sqworm/2\.9\.85-BETA \(beta_release; 20011115-775; i686-pc-linux-gnu\).*
.*SSurf15a 11 .*
.*StackRambler/x\.x .*
.*Submission Spider at surfsafely\.com.*
.*suchbaer\.de.*
.*SurferF3 1/0.*
.*suzuran.*
.*Szukacz/1\.x.*
.*tags2dir\.com/0\.8 \(\+http://tags2dir\.com/directory/\).*
.*Tagword \(http://tagword\.com/dmoz_survey\.php\).*
.*Talkro Web-Shot/1\.0 \(E-mail: webshot@daumsoft\.com- Home: http://222\.122\.15\.190/webshot\).*
.*Teoma MP.*
.*teoma_agent1.*
.*Teradex Mapper; mapper@teradex\.com; http://www\.teradex\.com.*
.*Test spider.*
.*TheRarestParser/0\.2a \(http://therarestwords\.com/\).*
.*TJG/Spider.*
.*Tkensaku/x\.x\(http://www\.tkensaku\.com/q\.html\).*
.*Toutatis x-xx\.x \(hoppa\.com\).*
.*Toutatis x\.x \(hoppa\.com\).*
.*Toutatis x\.x-x.*
.*Trampelpfad-Spider.*
.*Trampelpfad-Spider-v0\.1.*
.*TSurf15a 11.*
.*Tumblr/1\.0 RSS syndication \(\+http://www\.tumblr\.com/\) \(support@tumblr\.com\).*
.*Turnpike Emporium LinkChecker/0\.1.*
.*Tycoon Agent/Nutch-1\.0-dev.*
.*TygoProwler.*
.*Ultraseek.*
.*Under the Rainbow 2\.2.*
.*UofTDB_experiment \(leehyun@cs\.toronto\.edu\).*
.*URL Spider Pro/x\.xx \(innerprise\.net\).*
.*URL_Spider_Pro/x\.x.*
.*URL_Spider_Pro/x\.x\+\(http://www\.innerprise\.net/usp-spider\.asp\).*
.*User-Agent: Mozilla/4\.0 \(compatible; MSIE 6\.0; Windows NT 5\.1\).*
.*User-Agent: Mozilla/4\.0 \(SKIZZLE! Distributed Internet Spider v1\.0 - www\.SKIZZLE\.com\).*
.*Vagabondo/1\.x MT \(webagent@wise-guys\.nl\).*
.*Vagabondo/2\.0 MT.*
.*Vagabondo/2\.0 MT \(webagent at wise-guys dot nl\).*
.*Vagabondo/2\.0 MT \(webagent@NOSPAMwise-guys\.nl\).*
.*Vagabondo/3\.0 \(webagent at wise-guys dot nl\).*
.*Vakes/0\.01 \(Vakes; http://www\.vakes\.com/; search@vakes\.com\).*
.*versus 0\.2 \(\+http://versus\.integis\.ch\).*
.*VeryGoodSearch\.com\.DaddyLongLegs.*
.*Verzamelgids/2\.2 \(http://www\.verzamelgids\.nl\).*
.*Vision Research Lab image spider at vision\.ece\.ucsb\.edu.*
.*voyager-hc/1\.0.*
.*voyager/1\.0.*
.*vspider.*
.*vspider/3\.x.*
.*Wavefire/0\.8-dev \(Wavefire; http://www\.wavefire\.com; info@wavefire\.com\).*
.*Waypath Scout v2\.x - info at waypath dot com.*
.*Web Snooper.*
.*web2express\.org/Nutch-0\.9-dev \(leveled playing field; http://web2express\.org/; info at web2express\.org\).*
.*webbandit/4\.xx\.0.*
.*Webclipping\.com.*
.*WebCompass 2\.0.*
.*WebCorp/1\.0.*
.*webcrawl\.net.*
.*Webglimpse 2\.xx\.x \(http://webglimpse\.net\).*
.*Weblog Attitude Diffusion 1\.0.*
.*WebSearch\.COM\.AU/3\.0\.1 \(The Australian Search Engine; http://WebSearch\.COM\.AU; Search@WebSearch\.COM\.AU\).*
.*WebsiteWorth v1\.0.*
.*Webspinne/1\.0 webmaster@webspinne\.de.*
.*WebStat/1\.0 \(Unix; beta; 20040314\).*
.*Webster v0\.3 \( http://webster\.healeys\.net/ \).*
.*WebVac \(webmaster@pita\.stanford\.edu\).*
.*Webverzeichnis\.de - Telefon: 01908 / 26005.*
.*WebVulnCrawl\.unknown/1\.0 libwww-perl/5\.803.*
.*Wells Search II.*
.*WEP Search 00.*
.*WFARC.*
.*whatUseek_winona/3\.0.*
.*WhizBang! Lab.*
.*WinHTTP Example/1\.0.*
.*WorldLight.*
.*WSB, http://websearchbench\.cs\.uni-dortmund\.de.*
.*Wwlib/Linux.*
.*www\.arianna\.it.*
.*wwwster/1\.x \(Beta- mailto:gue@cis\.uni-muenchen\.de\).*
.*xirq/0\.1-beta \(xirq; http://www\.xirq\.com; xirq@xirq\.com\).*
.*Y!J-BSC/1\.0 \(http://help\.yahoo\.co\.jp/help/jp/search/indexing/indexing-15\.html\).*
.*Y!J-SRD/1\.0.*
.*Y!J/1\.0 \(http://help\.yahoo\.co\.jp/help/jp/search/indexing/indexing-15\.html\).*
.*yacy \(www\.yacy\.net; v20040602; i386 Linux 2\.4\.26-gentoo-r13; java 1\.4\.2_06; MET/en\).*
.*Yahoo Pipes 1\.0.*
.*Yahoo! Mindset.*
.*Yahoo-Blogs/v3\.9 \(compatible; Mozilla 4\.0; MSIE 5\.5; http://help\.yahoo\.com/help/us/ysearch/crawling/crawling-02\.html \).*
.*Yahoo-Test/4\.0.*
.*YahooFeedSeeker/2\.0 \(compatible; Mozilla 4\.0; MSIE 5\.5; http://publisher\.yahoo\.com/rssguide\).*
.*YahooSeeker-Testing/v3\.9 \(compatible; Mozilla 4\.0; MSIE 5\.5; http://search\.yahoo\.com/\).*
.*YahooSeeker/1\.0 \(compatible; Mozilla 4\.0; MSIE 5\.5; http://help\.yahoo\.com/help/us/shop/merchant/\).*
.*YahooSeeker/1\.0 \(compatible; Mozilla 4\.0; MSIE 5\.5; http://search\.yahoo\.com/yahooseeker\.html\).*
.*YahooSeeker/1\.1 \(compatible; Mozilla 4\.0; MSIE 5\.5; http://help\.yahoo\.com/help/us/shop/merchant/\).*
.*YahooSeeker/bsv3\.9 \(compatible; Mozilla 4\.0; MSIE 5\.5; http://help\.yahoo\.com/help/us/ysearch/crawling/crawling-02\.html \).*
.*YahooSeeker/CafeKelsa-dev \(compatible; Konqueror/3\.2; FreeBSD ;cafekelsa-dev-webmaster@yahoo-inc\.com \).*
.*Yandex/1\.01\.001 \(compatible; Win16; I\).*
.*yarienavoir\.net/0\.2.*
.*Yeti.*
.*yggdrasil/Nutch-0\.9 \(yggdrasil biorelated search engine; www dot biotec dot tu minus dresden do de slash schroeder; heiko dot dietze at biotec dot tu minus dresden dot de\).*
.*yoogliFetchAgent/0\.1.*
.*AppleWebKit/600\.2\.5.*
.*Mozilla/[0-9]+\.[0-9]+ \(compatible; seoscanners\.net/1; \+spider@seoscanners\.net\).*
.*Mozilla/[0-9]+\.[0-9]+ \(compatible; SpiderLing \(a SPIDER for LINGustic research\); \+http://nlp\.fi\.muni\.cz/projects/biwec/\).*
.*JOC Web Spider.*
.*Typhoeus - https://github\.com/typhoeus/typhoeus.*
.*ltx71 - \(http://ltx71\.com/\).*
.*Wget/[0-9]+\.[0-9]+\+cvs-stable \(Red Hat modified\).*
.*Magic Browser.*
.*G-i-g-a-b-o-t.*
.*Go-http-client/[0-9]+\.[0-9]+.*
^-$
.*WebFetch.*
.*M$
.*Wget/[0-9]\.[0-9][0-9] \(linux-gnu\).*
.*12345.*
.*WeBoX/0\.xx.*
.*HttpComponents/[0-9]+\.[0-9]+.*
.*Mindjet MindManager.*
.*Dragonfly File Reader.*
.*Jersey/[0-9]\.[0-9] \(HttpUrlConnection [0-9]\.[0-9]\.[0-9]_[0-9][0-9][0-9]\).*
.*FeedBurner/[0-9]+\.[0-9]+.*
.*AHC/[0-9]+\.[0-9]+.*
.*Bot.*
.*Crawler.*
.*Harvester.*
.*Zabbix.*
.*Turnitin.*
.*Indexer.*
.*bot.*