hrbrmstr/robotstxt.csv

Created August 1, 2017 02:37

Star () You must be signed in to star a gist
Fork () You must be signed in to fork a gist

Learn more about clone URLs
Clone this repository at <script src="https://gist.github.com/hrbrmstr/7bfbc4a3b4cadb3a32d2e1b72440bf6a.js"></script>
Save hrbrmstr/7bfbc4a3b4cadb3a32d2e1b72440bf6a to your computer and use it in GitHub Desktop.

Download ZIP

robots.txt user agent strings from June 2017 robots.txt Common Crawl

Raw

robotstxt.csv

	useragent	n
	*	494648
	IRLbot	378368
	bingbot	100948
	MJ12bot	94377
	msnbot	66142
	AhrefsBot	51999
	Yandex	50260
	Slurp	49160
	SemrushBot	45041
	sogouspider	42866
	Sogouwebspider	41080
	MJ12bot/v1.4.3	39158
	MSNBot	24619
	Y!J-ASR/0.1crawler	19286
	MSNBOT	19285
	Googlebot	16963
	spbot	15224
	Baiduspider	12954
	psbot	12081
	Bingbot	8647
	Mail.Ru	8213
	BLEXBot	8124
	CCBot	7384
	Mail.RU_Bot	7156
	Mediapartners-Google	6199
	WebAlta	6031
	WebAltaCrawler	6027
	Rambler	6021
	YandexBot	5308
	008	5154
	Googlebot-Image	4795
	ia_archiver	4503
	dotbot	4444
	TurnitinBot	4292
	Gigabot	4102
	Exabot	4091
	mantam	4085
	Sosospider	3836
	asterias	3739
	YahooPipes1.0	3620
	WebCopier	3557
	SearchmetricsBot	3475
	YandexImages	3256
	WebStripper	3229
	moget	3223
	Teleport	3142
	SiteSnagger	3064
	Applebot	3063
	twiceler	3061
	OfflineExplorer	3020
	Mediapartners-Google*	2983
	LinkWalker	2945
	rogerbot	2915
	XoviBot	2915
	EmailSiphon	2888
	Telesoft	2847
	CherryPicker	2845
	EmailCollector	2802
	WebsterPro	2802
	EmailWolf	2764
	Wget	2760
	WebSauger	2758
	WebAuto	2746
	NetAnts	2746
	Zeus	2743
	WWW-Collector-E	2736
	InfoNaviRobot	2736
	TheNomad	2736
	BuiltBotTough	2735
	lwp-trivial	2735
	BunnySlippers	2735
	LinkextractorPro	2734
	NICErsPRO	2733
	MIIxpc	2733
	BotALot	2732
	LexiBot	2732
	SpankBot	2732
	EroCrawler	2731
	WebBandit	2729
	ProWebWalker	2729
	CheeseBot	2729
	DittoSpyder	2727
	True_Robot	2727
	VCI	2727
	JennyBot	2726
	WebEnhancer	2726
	RepoMonkey	2726
	Openfind	2726
	spanner	2724
	humanlinks	2722
	Crescent	2722
	suzuran	2722
	hloader	2722
	cosmos	2718
	CopyRightCheck	2718
	RMA	2718
	httplib	2716
	WebsiteQuester	2715
	turingos	2713
	ExtractorPro	2712
	WebZip	2700
	MisterPiX	2700
	MataHari	2700
	URLyWarning	2699
	BlowFish/1.0	2698
	toCrawl/UrlDispatcher	2698
	Bullseye/1.0	2697
	libWeb/clsHTTP	2697
	KenjinSpider	2697
	WebBandit/3.50	2695
	WebImageCollector	2694
	Szukacz/1.4	2694
	TheIntraformant	2693
	SemrushBot-SA	2693
	CherryPickerElite/1.0	2692
	CherryPickerSE/1.0	2691
	True_Robot/1.0	2690
	moget/2.1	2689
	BackDoorBot/1.0	2689
	LinkScan/8.1aUnix	2688
	Harvest/1.5	2688
	lwp-trivial/1.34	2688
	MIIxpc/4.2	2686
	CrescentInternetToolPakHTTPOLEControlv.1.0	2686
	QueryNMetasearch	2685
	ProPowerBot/2.14	2681
	WebZip/4.0	2677
	Zeus32297WebsterProV2.9Win32	2673
	VCIWebViewerVCIWebViewerWin32	2671
	Wget/1.5.3	2652
	Foobot	2650
	RepoMonkeyBait&Tackle/v1.01	2650
	Wget/1.6	2643
	Xenu's	2634
	Yahoo!Slurp	2619
	HTTrack	2606
	Xenu'sLinkSleuth1.1c	2600
	baiduspider	2594
	MicrosoftURLControl-5.01.4511	2537
	MicrosoftURLControl-6.00.8169	2532
	Baiduspider-image	2527
	TightTwatBot	2458
	Baiduspider-video	2435
	Openfinddatagathere	2427
	Yeti	2414
	MegaIndex.ru/2.0	2384
	MegaIndex.ru	2369
	WebmasterWorldForumBot	2307
	Titan	2245
	SeznamBot	2232
	Cegbfeieh	2211
	BlackHole	2184
	Alexibot	2161
	WebReaper	2127
	NetMechanic	2105
	googlebot	1964
	DotBot	1946
	proximic	1919
	archive.org_bot	1892
	Teoma	1890
	HTTrack3.0	1868
	msnbot-media	1855
	ConveraCrawler	1847
	QuepasaCreep	1766
	findlinks	1765
	sistrix	1763
	BacklinkCrawler	1755
	YoudaoBot	1753
	Jetbot	1746
	discobot	1717
	SiteExplorer	1710
	VoilaBot	1704
	SuperBot	1688
	HuaweiSymantecSpider	1662
	Twitterbot	1653
	WinHTTrack	1638
	DeuSu	1634
	SEOkicks-Robot	1627
	Google	1627
	NetAttache	1626
	webcopy	1619
	webmirror	1617
	websiteextractor	1613
	DISCoPump3.1	1613
	WebStripper/2.02	1612
	SuperBot/2.6	1611
	NetAttacheLight1.1	1595
	semalt.com	1574
	Gigabot/3.0	1572
	linkdexbot	1571
	Nutch	1561
	sitebot	1552
	eDintornicrawler	1551
	MLBot	1518
	grapeshot	1485
	Findxbot	1476
	DomainCrawler	1473
	TeleportPro	1456
	Qwantify	1437
	MSIECrawler	1398
	Ezooms	1388
	Aboundex	1381
	Yisouspider	1374
	360Spider	1359
	Googlebot-Mobile	1324
	ShopWiki	1262
	NaverBot	1261
	LNSpiderguy	1179
	CazoodleBot	1176
	Twiceler	1173
	Riddler	1147
	Speedy	1137
	MJ12Bot	1130
	Facebot	1124
	ahrefs	1118
	panscient.com	1085
	KeywordDensity/0.9	1078
	YandexImageResizer	1070
	Adsbot-Google	1051
	NextGenSearchBot	1042
	TweetmemeBot	1036
	larbin	1014
	ichiro	1012
	BecomeBot	996
	googlebot-image	925
	yahoo-mmcrawler	915
	Claritybot	908
	Linguee	887
	Mozilla/4.0(compatible;BullsEye;Windows95)	878
	WeSEE	863
	Wotbox	854
	yeti	846
	naverbot	835
	noxtrumbot	833
	Scrubby	814
	Abonti	814
	yacybot	802
	Python-urllib	794
	baidu	775
	coccoc	769
	Robozilla	760
	gigabot	756
	Gaisbot	756
	Yahoo	729
	Spinn3r	728
	AdsBot-Google	725
	SafeDNSBot	722
	grub-client	715
	SputnikBot	714
	libwww	710
	TwengaBot	708
	yahoo-blogs/v3.9	703
	BingPreview	694
	SMTBot	693
	MSNbot	685
	istellabot	682
	Aboundexbot	673
	WBSearchBot	670
	EtaoSpider	665
	Microsoft.URL.Control	663
	Flamingo_SearchEngine	654
	Aqua_Products	651
	teoma	636
	Openbot	634
	bing	631
	YandexMobileBot	630
	Cliqzbot	623
	URL_Spider_Pro	616
	MojeekBot	608
	slurp	606
	googlebot-mobile	598
	Laserlikebot	598
	Kraken	597
	adbeat_bot	597
	b2w/0.1	597
	Blekkobot	596
	PaperLiBot	594
	Yahoo-Newscrawler	593
	Bookmarksearchtool	592
	URLControl	591
	Uptimebot	590
	ZeusLinkScout	588
	Iron33/1.0.2	588
	AdIdxBot	587
	searchpreview	583
	Copernic	583
	QuerySeekerSpider	582
	twitterbot	581
	naver	581
	RadiationRetriever1.1	579
	TweetedTimesBot	579
	Mozilla/4.0(compatible;MSIE4.0;WindowsNT)	577
	MicrosoftURLControl	577
	GetRight/4.2	577
	FlamingAttackBot	576
	JikeSpider	576
	Mozilla/4.0(compatible;MSIE4.0;Windows95)	576
	ScoutJet	576
	PerMan	573
	FairAdClient	573
	Mozilla/4.0(compatible;MSIE4.0;Windows98)	572
	CFNetwork	570
	uMBot-LN	569
	TinEye-bot	569
	Plukkie	565
	UptimeRobot	565
	g2reader-bot	560
	Msnbot	558
	PiplBot	558
	Girafabot	556
	OracleUltraSearch	556
	Vegi	555
	heritrix	555
	Sogou	555
	Pinterestbot	554
	wonderbot	554
	VigLink	554
	Leikibot	553
	coccocbot	553
	GetIntent	552
	uipbot	552
	Superfeedr	551
	trendiction	551
	Dragonbot	550
	avkzarabotok	550
	_zbot	550
	feedbot	550
	metabot.ru	550
	Clickagy	550
	calculon	550
	contxbot	550
	YaK	550
	mcsbot	550
	Tapatalk	550
	moatbot	550
	Seoterritory	550
	Taboolabot	550
	RSSMicro	550
	Ocarinabot	550
	Slack-ImgProxy	550
	HybridBot	550
	Feedspotbot	550
	Zombiebot	550
	Discordbot	550
	GnowitNewsbot	543
	Mediatoolkitbot	543
	oBot	542
	Multiviewbot	541
	Treato-Bot	541
	CharlieBot	540
	wget	540
	zarabotok--doma	540
	Sottopop	540
	focusbot	540
	Pvblcbot	540
	ZoominfoBot	540
	bot-pge.chlooe.com	540
	NPBot	530
	Ezooms/1.0	528
	yandex	526
	magpie-crawler	525
	Purebot	525
	Goodzer	514
	MSNBot-Media	508
	NerdByNature.Bot	505
	YodaoBot	504
	BaiduImagespider	501
	aipbot	497
	Baiduspider+	492
	Baiduspider/2.0	491
	gsa-crawler	489
	Baidu	481
	LinkpadBot	480
	mozilla/4	476
	mozilla/5	469
	007ac9	462
	80legs	459
	TwengaBot-Discover	457
	GoogleBot	455
	grub	454
	daumoa	449
	Mozilla/4.0(compatible;MSIE4.0;WindowsXP)	449
	Mozilla/4.0(compatible;MSIE4.0;Windows2000)	449
	RavenCrawler	447
	Ocelli	443
	BLP_bbot/0.1	442
	MSSearch4.0Robot	440
	BingBot	439
	msn	437
	Snapbot	431
	trovitBot	430
	Mozilla/4.0(compatible;MSIE4.0;WindowsME)	427
	YahooSeeker	423
	turnitinbot	419
	SemrushBot-SI	417
	NerdyBot	410
	EasouSpider	409
	sitecheck.internetseer.com	408
	WebZIP	408
	Mozilla/4.0(compatible;MSIE4.01;WindowsNT;MSSearch4.0Robot)Microsoft	407
	SurveyBot	405
	ScanAlert	405
	ips-agent	404
	Steeler	403
	ZoomSpider	403
	YahooYSMcm	389
	Scooter	388
	OmniExplorer_Bot	385
	SuperPagesUrlVerifyBot	383
	Fasterfox	381
	nutch	380
	YandexBot/3.0	379
	AdvBot	378
	Genieo/1.0	377
	LCC	377
	Xenu	376
	MSNBot/Bingbot	373
	Dolphin	368
	Voyager/1.0	365
	Riddlerbot	359
	SISTRIXCrawler	358
	mj12bot	358
	Fetch	356
	InternetSeer.com	355
	ccbot	350
	TwengaBot-2.0	343
	AppleBot	341
	linko	341
	SuperPagesBot	340
	YandexFavicons	340
	YPBot	340
	Yahoo-MMCrawler	338
	DownloadNinja	338
	wotbox	337
	e-SocietyRobot	336
	ContextAd	335
	Zealbot	335
	AskJeeves	335
	linkdex.com	334
	ezooms	333
	Y!J-BRW	332
	AugustBot	332
	SearQuBot	330
	Seznamscreenshot-generator	330
	ZyBORG	323
	Java/1.6.0_04	322
	Jyxobot	322
	SISTRIX	315
	linkdex	312
	IconSurf	309
	duggmirror	308
	Zao	307
	ICC-Crawler	305
	Yahoo-slurp	298
	BotRightHere	298
	UbiCrawler	296
	R6_CommentReader	296
	* #anyrobot	294
	megaIndex.ru	290
	netEstateNECrawler	289
	Charlotte	289
	DotBot/1.1	289
	betaBot	287
	aiHitBot	285
	BaiduMobaider	285
	R6_FeedFetcher	284
	BUbiNG	282
	WebCopierv3.2a	277
	checks.panopta.com	277
	linkdex.com/v2.0	276
	TurnitinBot/1.5	275
	Healthbot	275
	Healthbot/Health_and_Longevity_Project_(HealthHaven.com)	275
	//www.baidu.com/search/spider.html)	274
	AdvBot/2.0	274
	Gimme60bot	273
	FindTheBest.com	273
	SpeedySpider	272
	BLP_bbot	271
	Synapse	270
	WebZIP/4.21	268
	DOC	267
	WebZIP/5.0	267
	Openfinddatagatherer	266
	envolk	265
	MegaIndex	265
	WebCapture2.0	264
	//www.baidu.com/search/spider.html	264
	HaosouSpider	262
	WebCopierv.2.2	261
	lmspider	258
	Bing	254
	Adidxbot	253
	Twitterbot/1.0	252
	CamelStampede	251
	Rakutenbot	250
	IstellaBot	249
	msnbot-products	247
	YesupBot	246
	voyager	245
	MSRBOT	245
	mxbot	243
	meanpathbot	240
	msnbot-news	239
	KSCrawler	238
	MSR-ISRCCrawler	238
	yahoo-slurp	237
	AhrefsBot/5.1	236
	Fbot	233
	nu_tch-princeton	232
	//ahrefs.com/robot/	232
	abby	231
	MSUbot	231
	SheenBot	231
	Lycos_Spider	231
	Kwaclebot	231
	LargeSmallCrawler	229
	UnwindFetchor/1.0	229
	FASTEnterpriseCrawler6	229
	PostRank	229
	DotSpotsBot	229
	WillowInternetCrawlerbyTwotrees	228
	inagist.comurlcrawler	228
	Jaxified	228
	agbot	228
	Domaincrawler1.0	228
	Hailoobot	228
	//labs.topsy.com/butterfly/)Gecko/2009032608Firefox/3.0.8	228
	diribot	228
	YahooMobile/1.0	228
	Influencebot/0.9	228
	CyberPatrolSiteCatWebbot	228
	YisouSpider	224
	k2spider	219
	//megaindex.com/crawler)	217
	robozilla	213
	//yandex.com/bots	212
	CompSpyBot	212
	Trident	211
	MJ12bot/v1.4.5	211
	Mozilla/4.0+(compatible;+T-H-U-N-D-E-R-S-T-O-N-E)	211
	ZyBorg	210
	ahrefsbot	208
	//www.sogou.com/docs/help/webmasters.htm#07)	205
	scrubby	205
	Vagabondo	203
	Wotbox/2.01	202
	exabot	202
	Mail.RU_Bot/2.0	201
	applebot	200
	//www.wotbox.com/bot/	200
	ia_archiver/1.6	200
	StackRambler	199
	SBIder	196
	OrangeBot	196
	//yandex.com/bots)	192
	AhrefBot	192
	seznambot	192
	becomebot	191
	MaxPointCrawler	189
	SolomonoBot	186
	Exabot-Thumbnails	185
	VegeBot	184
	//go.mail.ru/help/robots	182
	megaindex.com	182
	BPImageWalker	181
	Java/1.4.1_04	181
	SurveyBot_IgnoreIP	181
	htdig	178
	discoverybot	178
	Nutch-1.4	177
	adidxbot	176
	//www.majestic12.co.uk/bot.php?+)	175
	//deusu.de/robot.html	175
	linkdexbot/2.2	175
	//www.profound.net/domainappender)	175
	MSSearch6.0Robot	174
	//ltx71.com/)	173
	spbot/5.0.2	173
	Sogou+web+spider/4.0	172
	//go.mail.ru/help/robots)	172
	//ltx71.com/	172
	//ahrefs.com/robot/)	172
	//www.wotbox.com/bot/)	171
	yoozBot-2.2	171
	MegaIndex.ru/	170
	DeuSu/5.0.2	170
	SMTBot/1.0	170
	DataparkSearch	170
	Mail.RU_Bot/Img/2.0	170
	GrapeshotCrawler/2.0	169
	DuckDuckGo	169
	//www.genieo.com/webfilter.html)	169
	Applebot/0.1	169
	//www.linkdex.com/bots/)	169
	Mail.RU_Bot/Robots/2.0	169
	spiderman	169
	maxpoint.crawler	168
	//api.slack.com/robots	168
	Siteimprove.com	168
	Cliqzbot/1.0	168
	Googlebot-image	168
	Java/1.6.0_21	168
	maxpoint.crawler+at+maxpointinteractive+dot+com	167
	//www.proximic.com/info/spider.php)	167
	//www.apple.com/go/applebot)	167
	//www.similartech.com/smtbot)	167
	Spider/5.1	167
	//www.opensiteexplorer.org/dotbot,[email protected])	167
	//www.grapeshot.co.uk/crawler.php	167
	majestic12.co.uk/bot.php	167
	//cliqz.com/company/cliqzbot)	167
	//www.archive.org/details/archive.org_bot)	167
	//sur.ly/bot.htm	167
	safedns.com/searchbot	167
	//www.haosou.com	167
	//www.profound.net/domainappender	167
	Nutch-1.10	167
	//yooz.ir;[email protected]	167
	//megaindex.com/crawler	167
	go.mail.ru/	167
	//duckduckgo.com	167
	curl/7.19.7+(x86_64-unknown-linux-gnu)+libcurl/7.19.7+NSS/3.12.7.0+zlib/1.2.3+libidn/1.18+libssh2/1.2.2	167
	MojeekBot/0.6	167
	www.proximic.com/info/spider.php	167
	//www.similartech.com/smtbot	167
	www.apple.com/go/applebot	167
	//safesearch.avira.com,[email protected])	167
	orange.com	167
	www.baidu.com/search/spider.html	167
	awooo	167
	www.dataparksearch.org/bot	167
	Slack-ImgProxy+1.136	167
	//www.dataprovider.com/)	167
	libcurl/7.19.7	167
	//cliqz.com	167
	SurdotlyBot/1.0	167
	Xenu+Link	167
	LinkCheck+by+Siteimprove.com	167
	//yooz.ir;[email protected])	167
	js-crawler	167
	//inet-sochi.ru/gday	167
	//www.mojeek.com/bot.html)	167
	Simple/6.13+libwww-perl/6.13	167
	www.genieo.com/webfilter.html	167
	ZoomInformation+Bot	167
	Nutch-1.9	167
	Sleuth/1.3.8	167
	SafeSearch+microdata+crawler	167
	NRLCorpusBuilder/Nutch-1.9	167
	//archive.org/details/archive.org_bot	167
	//api.slack.com/robots)	167
	HETZNER-RZ-NBG-NET	167
	//www.linkdex.com/bots/	167
	NextGenSearchBot.aspx	167
	//www.zoominfo.com/About/misc/NextGenSearchBot.aspx)	167
	//archive.org/details/archive.org_bot)	167
	crawler/js-crawler	167
	zlib/1.2.3	167
	//www.zoominfo.com	167
	//www.haosou.com/help/help_3_2.html)	167
	//www.opensiteexplorer.org/dotbot	167
	www.sogou.com	167
	libssh2/1.2.2	167
	safesearch.avira.com	167
	nutch-1.4/Nutch-1.4	167
	[email protected]	167
	//www.safedns.com/searchbot)	167
	//www.mojeek.com/bot.html	167
	//www.megaindex.ru/	167
	yooz.ir	167
	curl/7.29.0	167
	[email protected]	167
	DomainAppender+/1.0	167
	Dataprovider/6.92	167
	OrangeBot/2.0	167
	ContentVerification/5.1	167
	Xenu+Link+Sleuth/1.3.8	167
	Screaming+Frog+SEO	167
	Mozilla/5.0+(compatible;+OrangeBot/2.0;[email protected])	167
	www.baidu.com	167
	UXCrawlerBot	167
	Mozilla/5.0+(compatible;+MSIE+10.0;+Windows+NT+6.1;+Trident/6.0)+LinkCheck+by+Siteimprove.com	167
	www.proximic.com	167
	//em7.sciencelogic.com	167
	//duckduckgo.com)	167
	//sur.ly/bot.html)	167
	xSocks+v0.1	167
	libidn/1.18	167
	Semalt.com	167
	//OpenLinkProfiler.org/bot	167
	Screaming+Frog+SEO+Spider/5.1	167
	//www.dataprovider.com/	167
	tutorgigbot	167
	[email protected]	167
	//www.grapeshot.co.uk/crawler.php)	167
	//deusu.de/robot.html)	167
	curl/7.43.0	167
	//OpenLinkProfiler.org/bot+)	167
	seekbot	166
	NA	164
	looksmart	164
	SWEBot	164
	GrapeshotCrawler	157
	Jetbot/1.0	156
	//www.almaden.ibm.com/cs/crawler	155
	HatenaAntenna	154
	dumbot	154
	curl	153
	Enterprise_Search	152
	facebookexternalhit	152
	sootle	152
	WellsSearchII0.0	151
	EdisterBot	151
	heritrix/1.10.0	151
	lemurwebcrawler	150
	WebVac	150
	memorybot	149
	google-hoteladsverifier	149
	FreeFind	149
	Fatbot	149
	netseer	148
	Java/1.6.0_11	148
	Genieo	147
	PagePeeker	147
	CatchBot	147
	Yahoo-Blogs	146
	es	146
	Enterprise_Search/1.0	145
	sogou	144
	plukkie	144
	Updownerbot	144
	Eluta.ca	143
	Java/1.6.0_20	141
	sentibot	140
	Java/1.6.0_16	139
	Java/1.6.0_15	139
	Java/1.6.0_18	139
	Java/1.6.0_24	139
	FunWebProducts	139
	Java/1.6.0_25	139
	Java/1.6.0_17	139
	Java/1.6.0_22	139
	Stanford	139
	Java/1.6.0	138
	blp_bbot/0.1	138
	Ultraseek	138
	StanfordCompSci	137
	GoldfireServer	137
	Googlebot-Docs	137
	gsa-crawler+(Enterprise;+T3-HR8MK3S756ETJ;[email protected])	137
	DowJonesSearchbot	136
	lipperhey	135
	RPT-HTTPClient	134
	dotbot/1.0	133
	Sogouwebspider/4.0	133
	szukacz	132
	blexbot	132
	Shim-Crawler	132
	HubSpot	131
	FAST	130
	Insieve+Bot	130
	mail.ru	130
	xovibot	129
	SiteBot	128
	OpenindexSpider	128
	ScreamingFrogSEOSpider	128
	SEOkicks	127
	smtbot	126
	wbsearchbot	125
	publiclibraryarchive.org	125
	msnbot-NewsBlogs	125
	seokicks-robot	125
	Heritrix	124
	linkdexbot/2.0	124
	AddThis.com	123
	megaindex.ru	123
	libwww-perl	121
	SlySearch	121
	MotoMinerBot	120
	MSN	119
	semrushbot	119
	siteexplorer	118
	Googlebot-News	118
	linkdexbot/2.1	117
	DISCoPump	117
	Yanga	117
	advbot	116
	netestatenecrawler	116
	exblanguagecrawler	116
	aboundexbot	116
	feedbooster	116
	mixbot	116
	bubing	116
	searchmetricsbot	116
	easouspider	116
	bpimagewalker/2.0	116
	bingbot/2.0	116
	abonti	116
	unisterbot	116
	megaindex.ru/2.0	116
	linkpadbot	116
	screenerbot	116
	Webinator	115
	ZumBot	114
	Yetibot	114
	EC2LinkFinder	114
	CrescentInternetToolPak	113
	voltron	112
	vspider	112
	SindiceBot	112
	XoviBot/2.0	112
	YandexNews	112
	worldwebheritage.org	111
	seoscanners.net	111
	Exabot*	109
	PerManSurfer	109
	aihitbot	108
	MFCFoundationClassLibrary	108
	WebEMailExtractor	108
	Netprospector	108
	LNSpiderGuy	108
	AdamaticSolutions	107
	Control	107
	Microsoft	107
	google	106
	worldwebheritage.org/1.0	106
	JamesBOT	106
	oodlebot	105
	Baiduspider-news	105
	Echo	105
	Yahoo!-AdCrawler	104
	Mozilla	104
	Mozilla/4.0+(compatible;MSIE4.0;Windows98)	103
	BaiDuSpider*	103
	happyfunbot	103
	NewsTroveBot	103
	gigabot*	103
	GenieBot	103
	ipselonbot	103
	UnisterBot	102
	cuil	102
	Jeeves	102
	Orthogaffe	100
	voilabot	99
	Bender	97
	Java	97
	suggybot	94
	findfiles.net	93
	Huaweisymantecspider	92
	GetRight	92
	BaiDuSpider	92
	ip-web-crawler.com	91
	omgilibot	91
	ECCP	91
	Baiduspider-favo	91
	Baiduspider-cpro	91
	TinEye	90
	worldwebheritage.org/1.1	90
	YakazBot	90
	BackDoorBot	90
	uniplace-bot	89
	Mail.ru	89
	FyberSpider	88
	HubSpotWebcrawler	88
	LexxeBot	87
	Harvest	87
	Bullseye	87
	BlackWidow	86
	UnwindFetchor	86
	HubSpotLinksCrawler1.0	86
	Pixray-Seeker	85
	Sogouspider	85
	LSSRocketCrawler	85
	AhrefsBot/4.0	85
	Baiduspider-ads	85
	BlowFish	84
	AhrefsBotisnotallowedanymore	84
	uniplace-index	82
	Custo	82
	FlashGet	81
	Webalta	81
	ltx71	81
	Googlebot-Video	81
	eCatch	80
	AppCodesCrawler	80
	WebFetch	80
	vebidoobot	79
	Go-Ahead-Got-It	79
	DuckDuckBot	79
	Voluniabot	79
	WWWOFFLE	79
	Surfbot	79
	MsnBot	79
	CrazyWebCrawler-Spider	79
	ReGet	78
	VoidEYE	78
	HypeStat	78
	JetCar	78
	HMView	78
	Rogerbot	78
	NearSite	78
	RealDownload	78
	NetSpider	78
	SmartDownload	78
	SuperHTTP	78
	ImageWalker	78
	pcBrowser	78
	//www.hubspot.com/	77
	YandexMobileBot/3.0	77
	Teemer	76
	LinkpadBot/1.06	76
	NetZIP	75
	DomainRe-AnimatorBot	75
	www.integromedb.org/Crawler	75
	pavuk	75
	*	75
	LeechFTP	74
	TruliaBot	74
	Grafula	74
	EirGrabber	74
	LWNutch	74
	costam	74
	gonzo	73
	Semrush	73
	Widow	73
	SemrushBot/1.1~bl	73
	Dotbot	73
	EyeNetIE	73
	Navroad	72
	ChinaClaw	72
	picmole	72
	InterGET	72
	Mp3Bot	72
	WebLeacher	72
	GrabNet	72
	Octopus	72
	PageGrabber	72
	tAkeOut	72
	WebWhacker	72
	Meridian-crawler	71
	hap-crawler	71
	OpenHoseBot	71
	fast	71
	searchmetrics-bot	71
	KaloogaBot	70
	PropsmartCrawler	70
	BDFetch	70
	yandexbot	70
	Go!Zilla	70
	SimilarPages	70
	MediaPartners-Google	70
	Mozilla*	69
	Zing-BottaBot	69
	jobs.de-Robot	69
	Presto	69
	Yahoo!slurp	69
	SimilarPages/Nutch-1.0-dev	69
	contype	69
	YandexMedia	68
	FriendFeedBot	68
	FlipboardProxy	68
	bot/1.0	67
	//www.baidu.com/search/spider.htm)	66
	spider	66
	msnbot-MM	66
	CrystalSemanticsBot	66
	TosCrawler	66
	egothor	66
	SEOkicksRobot	66
	woriobot	66
	Intelliseek	65
	ExBLanguageCrawler	65
	SogouOrionspider	64
	Sogouinstspider	64
	iCjobs	63
	BoardReader	63
	Psbot	63
	RookeeBot	63
	SogouNewsSpider	63
	AhrefsBot/5.0	62
	lb-spider	62
	Sogouspider2	62
	WebmasterCoffee	62
	FrontPage	61
	Aport	61
	rookeebot	61
	Szukacz	60
	W3C-checklink	60
	Sogoublog	60
	YandexPagechecker	60
	yetibot	59
	AppleNewsBot	59
	MaxPointCrawler/Nutch-1.1	59
	ProPowerBot	58
	Mozilla/4.0(compatible;MSIE6.0;WindowsNT;MSSearch4.0Robot)	58
	yisouspider	58
	ia_archiver-web.archive.org	58
	LinkScan	57
	+Baiduspider/2.0	57
	zibber-v0.1(www.zibb.com/crawler/)	57
	AhrefsBot/3.1	56
	search17	56
	DISCo	56
	*"	56
	/	56
	+Baiduspider	56
	InfoPath	55
	StorygizeBot	55
	YandexDirect	55
	LinkisBot	55
	HoaxyBot	55
	bot	55
	UptimeRobot/2.0	54
	WikioFeedBot	54
	xovi	54
	bdbrandprotect	54
	Comodo-Certificates-Spider	54
	GSLFbot	54
	NetResearchServer	53
	Yeti-Mobile	53
	schrein	53
	ContentCrawler	53
	AfiliasWebMiningTool	53
	ReverseGet	53
	DCPbot	53
	lex	53
	OpidooBOT	53
	COMODOSSLChecker	53
	kalooga	52
	ImageBot	52
	Exabot/3.0	52
	CuriousGeorge	52
	ShowyouBot	52
	FatBot	52
	crawl.yahoo.net	52
	IDBot	52
	Eurobot	52
	JenkersBot	52
	cityrevier	51
	Snoopy	51
	Diffbot	51
	WijuBot	51
	niki-bot	51
	bingbot.	51
	Kemvibot	51
	spbot/4.4.2	50
	Xenu’sLinkSleuth1.1c	50
	Xenu’s	50
	Swiftbot	50
	appie	50
	Pinterest	50
	scooter	50
	antibot	49
	BLEXBot/1.0	49
	MajesticSEO	49
	Naverbot	49
	ChangeDetection	49
	Unister	49
	thunderstone	49
	Spiderlytics	49
	zibber-v0.1	49
	Go-http-client/1.1	48
	SogouPicSpider	48
	semantic-visions.comcrawler	48
	Sqworm	48
	GingerCrawler	48
	ImageWalker/2.0	48
	Flash+Processor	48
	SiteExplorer/1.0b	48
	DoCoMo	47
	//www.cuil.com/twiceler/robot.html)	47
	GermCrawler	47
	WebmasterWorldExtractor	47
	YahooSeeker/M1A1-R2D2	47
	Powermarks	47
	[email protected]	47
	MJ12bot/v1.4.0	47
	it2media-domain-crawler	46
	BaiduSpider	46
	BSpider	46
	gsa-crawler-www	46
	cityreview	46
	Scrapy	45
	GetWeb!	45
	URLy.Warning	45
	NimbleCrawler	45
	adsbot-google	45
	trendictionbot	45
	OOZBOT	45
	DomainAppender	44
	boitho.com-dc	44
	YandexMetrika	44
	OfflineExplorer/1.9	44
	FASTEnterpriseCrawler	44
	NPbot	44
	SimplePie	44
	LocalcomBot	44
	ChinasoSpider	44
	CherryPicker/1.0	44
	Toweyabot	44
	Flipboard	43
	ObjectsSearch	43
	Baiduspider/5.0	43
	newsbot	43
	Radian6	43
	Baiduspider/3.0	43
	SiteLockSpider	43
	Baiduspider/4.0	43
	yetbot	43
	baiduspider+	43
	SapphireWebCrawler	42
	OpenHoseBot/2.1	42
	omgilibot/0.4	42
	pompos	42
	careerbot	42
	msnbot-UDiscovery/2.0b	42
	Xaldon\WebSpider	42
	fr-crawler/1.1	42
	ca-crawler/1.0	42
	SiteSucker	42
	MemoryBot	41
	Offline\Navigator	41
	//www.majestic12.co.uk/bot.php?+	41
	Express\WebPictures	41
	Sosospider+	41
	MIDown\tool	41
	Web\Image\Collector	41
	Offline\Explorer	41
	Kemvibot/1.0	41
	JOC\Web\Spider	41
	Mass\Downloader	41
	Image\Sucker	41
	Papa\Foto	41
	Website\Quester	41
	RediffNewsBot	41
	WebSucker	41
	WebGo\IS	41
	Spiderbot	41
	Web\Sucker	41
	Internet\Ninja	41
	sosospider	41
	Image\Stripper	41
	Mister\PiX	41
	Download\Demon	41
	Net\Vampire	41
	Indy\Library	41
	SentiBot	41
	GwdangSpider	40
	Lachesis	40
	HuihuiSpider	40
	LinkedInBot	40
	ASPSeek	40
	autoemailspider	40
	Daumoa	40
	WochachaSpider	40
	IsraBot	40
	Gigabot/2.0	39
	Yahoo!	39
	YandexBlogs	39
	vscooter	39
	AhrefsBot/2.0	39
	AboutUsBot	39
	yolinkBot	39
	stress-agent	39
	Wget/1.8.2	39
	Infohelfer	39
	ArchitextSpider	39
	IndyLibrary	38
	Xenu'sLinkSleuth	38
	Generic	38
	x28-job-bot	38
	kulturarw3	38
	YangaWorldSearchBot	38
	penthesilea	38
	NinjaBot	38
	newsregistry	38
	msnbot-UDiscovery	38
	um-LN	38
	Grabber	38
	DoCoMo/2.0	37
	Reaper	37
	DigExt	37
	All	37
	[email protected]	37
	YYSpider	37
	Unister*	36
	Yahoo!SlurpChina	36
	ahrefs.com	36
	AbachoBOT	36
	Qualidator*	36
	MicrosoftURLControl–6.00.8169	36
	4SeoHuntBot	36
	Simple	36
	JobboerseBot	36
	MicrosoftURLControl–5.01.4511	36
	MIA	36
	ipsAgent	36
	WebofantBot	36
	Roverbot	36
	BPImageWalker*	36
	thunderstone	36
	WASALive-Bot	36
	YodaoBot/1.0	36
	MassDownloader	35
	//www.spidersoft.com)	35
	NPBot-1/2.0	35
	Webster.Pro	35
	WebStripper/2.16	35
	LingueeBot	35
	FAST-WebCrawler	35
	Yandexbot	35
	RogerBot	35
	Yahoo-Slurp	35
	OpenWebIndex	35
	YaDirectFetcher	35
	RealDownload/4.0.0.41	34
	Magnet	34
	RealDownload/4.0.0.40	34
	NetAnts/1.25	34
	Website.Quester	34
	RealDownload/4.0.0.42	34
	Wget/1.7	34
	LinkScan/8.1a.Unix	34
	GetRight/4.5	34
	WebStripper/2.19	34
	GetRight/4.5b	34
	GetRight/4.5b6	34
	Yandex.com	34
	Baiduspider-mobile	34
	EasyDL	34
	ImageSucker	34
	DISCoFinder	34
	GetRight/5.0beta1	34
	JOCWebSpider	34
	NetVampire	34
	eCatch/3.0	34
	MIDowntool	34
	Keyword.Density	34
	semanticdiscovery	34
	GetRight/4.3	34
	WebGoIS	34
	SuperHTTP/1.0	34
	Webster	34
	GetRight/4.2c	34
	GetRight/4.1.0	34
	yacy	34
	Drip	34
	Wget/1.5.2	34
	OfflineExplorer/1.7	34
	Twiceler-0.9	34
	TeleportPro/1.29	34
	Siphon	34
	GetRight/4.5b3	34
	GetRight/3.1	34
	GetRight/3.3	34
	ImageStripper	34
	WebStripper/2.13	34
	GetRight/4.5a	34
	OfflineExplorer/1.6	34
	GetRight/5.0beta2	34
	GetRight/4.5d	34
	GetRight/4.5e	34
	OfflineExplorer/1.4	34
	GetRight/4.5c	34
	Mag-Net	34
	GetRight/4.5b7	34
	OfflineExplorer/1.2	34
	BatchFTP	33
	Pockey	33
	//www.sogou.com/docs/help/webmasters.htm	33
	BackWeb	33
	//www.archive.org/details/archive.org_bot	33
	msrbot	33
	Mail.RU	33
	AskPeterBot	33
	Buddy	33
	bingbot-media	33
	//www.exabot.com/go/robot	33
	stalker	33
	Pump	33
	webcrawler	33
	Zade	33
	SpaceBison	33
	Vacuum	33
	//www.linkpad.ru	33
	//www.proximic.com/info/spider.php	33
	Ninja	33
	Whacker	33
	Galbot	33
	Collector	33
	Recorder	33
	Sucker	33
	Bandit	33
	JustView	33
	MFC_Tear_Sample	33
	Stripper	33
	SEOENGWorldBot	33
	Snake	33
	Mozilla/5.0(compatible;Ezooms/1.0;ezooms.bot[at]gmail[dot]com)	33
	Iria	33
	Xaldon_WebSpider	33
	Copier	33
	Zeusbot	33
	fyberspider	32
	PGImageCrawler	32
	HTTrack[NC,OR]	32
	JobdiggerSpider	32
	netEstateRSScrawler	32
	search-one-scgov	32
	Alexabot	32
	Alexa	32
	Accelobot	32
	XenuLinkSleuth/1.3.8	32
	Browsershots	32
	yahoo	32
	Plukkie/1.5	32
	NG/2.0	32
	MisterPixII2.02a	31
	Spinne	31
	MassDownloader/2.2	31
	DISCoPump3.2	31
	Go!Zilla(www.gozilla.com)	31
	Zeus11652WebsterProV2.9Win32	31
	NetVampire/3.0	31
	TeleportPro/1.29.1847	31
	SmartDownload/1.2.77(Win32;Feb12000)	31
	crawler4j	31
	WebCopierv3.0.1	31
	OfflineExplorer/2.0	31
	DomainsDB.netMetaCrawler	31
	WebSauger1.20j	31
	Zeus97371WebsterProV2.9Win32	31
	BoardTracker	31
	psbot/0.1	31
	WebCopierv2.8	31
	WebCopierv2.7a	31
	DownloadDemon/3.2.0.8	31
	WebAuto/3.40(Win98;I)	31
	OfflineExplorer/2.4	31
	Zeus82016WebsterProV2.9Win32	31
	WebSauger1.20b	31
	WebSauger1.20k	31
	FrontPage[NC,OR]	31
	Bilbo	31
	Mister.PiX	31
	MisterPixII2.01	31
	Zeus30747WebsterProV2.9Win32	31
	NetZipDownloader1.0Win32(Nov121998)	31
	OfflineExplorer/2.5	31
	MisterPiXversion.dll	31
	TeleportPro/1.29.1634	31
	NetZip-Downloader/1.0.62(Win32;Dec71998)	31
	InternetNinja5.0	31
	GetRight/4.2b(Portuguxeas)	31
	Zeus63567WebsterProV2.9Win32	31
	FlashGetWebWasher3.2	31
	InternetNinja6.0	31
	InternetNinja4.0	31
	Zeus90872WebsterProV2.9Win32	31
	Zeus44238WebsterProV2.9Win32	31
	Tagoobot	31
	Zeus11389WebsterProV2.9Win32	31
	WebCopierv2.5	31
	Mata\Hari	30
	DTS\Agent	30
	thesubot	30
	httrack	30
	ScreenerBot	30
	Keyword\Density	30
	Kenjin\Spider	30
	Black.Hole	30
	likse	30
	Uptimebot/1.0	30
	InterfaxBot	30
	TMCrawler	30
	gotit	30
	genieBot	30
	Black\Hole	30
	The\Intraformant	30
	URLy\Warning	30
	Gulliver	30
	mozilla/3	30
	occ-crawler	30
	Microsoft\URL\Control	30
	Download\Wonder	30
	libWeb	30
	attach	30
	lftp	30
	PycURL	30
	JOC	30
	//www.bing.com)	30
	AhrefsBot/3.0	30
	Spiderbot/Nutch-1.7	29
	Pompos	29
	The.Intraformant	29
	OrangeBot-Collector	29
	TeleportPro/1.29.1590	29
	www.deadlinkchecker.com	29
	YandexCalendar	29
	MSNBOT/0.1	29
	holmes	29
	//www.innerprise.net/usp-spider.asp)	29
	ExpressWebPictures	29
	ApocalXExplorerBot	29
	DownloadDemon	29
	Wget/1.8	28
	XaldonWebSpider	28
	Wget/1.9-beta	28
	NetAnts/1.23	28
	GetRight/3.3.3	28
	libcrawl	28
	ScreamingFrogSEOSpider/4.1	28
	Website\eXtractor	28
	GetRight/4.0.0	28
	Mata.Hari	28
	GetRight/2.11	28
	msnbot<BR>	28
	Offline.Explorer	28
	WebStripper/2.15	28
	WebStripper/2.10	28
	Wget/1.8.1	28
	GetRight/4.5b2	28
	repparser	28
	GalaxyBot	28
	SurdotlyBot	28
	PhpDig	28
	OfflineNavigator	28
	GetRight/3.2	28
	JakartaCommons-HttpClient	28
	007ac9Crawler	28
	YandexMarket	28
	bitlybot	28
	sindice-site-manager	28
	Scanmine	28
	NetAnts/1.24	28
	NetAnts/1.10	28
	PapaFoto	28
	CybEye.com	28
	Genio	28
	SimplePie/1.3.1	28
	Teleport\Pro	28
	GetRight/3.3.4	28
	GetRight/4.5b1	28
	sp_auditbot	28
	InternetNinja	28
	OdklBot	28
	WebStripper/2.03	28
	WocBot	28
	WebStripper/2.12	28
	GetRight/4.1.1	28
	webfetch/2.1.0	28
	GetRight/4.1.2	28
	metajobbot	27
	KeywordDensity	27
	robot	27
	sukibot_heritrix	27
	HTTrackWebsiteCopier	27
	facebookexternalhit/1.1	27
	YamanaLab-Robot	27
	VocusBot	27
	ZmEu	27
	Euripbot	27
	DBLBot	27
	SEMrushBot	27
	OpenLinkProfiler	27
	RepoMonkeyBait&Tackle/v1.01	27
	accelobot	27
	SiteBot/0.1	27
	//help.soso.com/webspider.htm)	27
	ScSpider	27
	NutchCVS/0.7.1	27
	smrjbot/0.0.20	26
	deepcrawl	26
	smrjbot	26
	scrapybot	26
	QueryN.Metasearch	26
	nabot	26
	Altavista	26
	psycheclone	26
	Slackbot-LinkExpanding	26
	expo9	26
	DIIbot	26
	CydralSpider	26
	Dumbot	26
	SEOdiver	26
	swish-e	26
	TeleportPro/1.29.1718	26
	Web.Image.Collector	26
	Unknownrobot	26
	seoscanners	26
	CoolBot	26
	XenuLinkSleuth	26
	OfflineExplorer/2.1	25
	Zeus84842WebsterProV2.9Win32	25
	Wget/1.8.1+cvs	25
	Zeus95245WebsterProV2.9Win32	25
	XaldonWebSpider2.5.b3	25
	WebCapture	25
	WebsiteeXtractor	25
	SuperBot/3.1(Win32)	25
	Kenjin.Spider	25
	MSNBot-media	25
	Zeus82900WebsterProV2.9Win32	25
	Zeus94934WebsterProV2.9Win32	25
	cfetch	25
	Zeus39206WebsterProV2.9Win32	25
	crystalsemanticsbot	25
	WebCopierv3.2	25
	Zeus51837WebsterProV2.9Win32	25
	LexxeBot/1.0	25
	Zeus51070WebsterProV2.9Win32	25
	WebCopierv2.6	25
	TeleportPro/1.29.1820	25
	Zeus6694WebsterProV2.9Win32	25
	SmartDownload/1.2.77(Win32;Aug171999)	25
	BruinBot	25
	SensisWebCrawler	25
	WebCopierv3.0	25
	DownloadDemon/3.5.0.11	25
	Go!Zilla3.3(www.gozilla.com)	25
	Zeus18018WebsterProV2.9Win32	25
	Zeus51674WebsterProV2.9Win32	25
	SogouSpider	25
	IndyLibrary[NC,OR]	25
	gonzo*	25
	SuperBot/3.0(Win32)	25
	RufusBot	25
	ExpressWebPictures(www.express-soft.com)	25
	HooWWWer	25
	Mozilla/4.0(compatible;MSIE5.0;WindowsNT;DigExt;DTSAgent	25
	Microsoft.URL	25
	Zeus95351WebsterProV2.9Win32	25
	OfflineExplorer/2.3	25
	ScoutAbout	25
	SmartDownload/1.2.77(Win32;Jun192001)	25
	Go!Zilla3.5(www.gozilla.com)	25
	SightupBot	25
	Zeus41641WebsterProV2.9Win32	25
	Zeus71129WebsterProV2.9Win32	25
	DISCoPump3.0	25
	SmartDownload/1.2.76(Win32;Apr11999)	25
	Zeus26378WebsterProV2.9Win32	25
	WebReaper[[email protected]]	24
	Yahoo-MMAudVid	24
	zitebot	24
	InterfaxScanBot	24
	GozaikBot	24
	ICCrawler-iCjobs	24
	MSSearch5.0Robot	24
	alexa	24
	Googlebot*	24
	thumbshots-de-bot	24
	[email protected]	24
	CamontSpider	24
	SearchDaimon.com-dc	24
	NetSeercrawler	24
	Amfibibot	23
	WebViewer	23
	WebReaper[[email protected]]	23
	TutorGig	23
	majestic12	23
	Bitacle	23
	//www.picmole.com)	23
	FindLinks	23
	lwp-request	23
	BizInformation	23
	iCCrawler	23
	larbin_2.6.2([email protected])	23
	Larbin	23
	WebEMailExtrac.*	23
	Zeusbot/Nutch-1.0-dev	23
	larbin_2.6.2(listonATccDOTgatechDOTedu)	23
	larbin_2.6.2([email protected])	23
	//www.seoprofiler.com/bot/)	23
	[email protected]	23
	mlbot	23
	larbin([email protected])	23
	Rankivabot	23
	alexabot	23
	larbin_2.6.2([email protected])	23
	larbin_2.6.2listonATccDOTgatechDOTedu	23
	nicebot	23
	Feedfetcher-Google	23
	LiteFinder	23
	crawl	23
	LivelapBot	23
	roadrunner	23
	[email protected]	23
	mozilla	23
	[email protected]	23
	//www.accelobot.com)	23
	btbot	23
	//garlik.com/,[email protected])	23
	URLAppendBot	22
	Purebot/1.1	22
	purebot	22
	Ahrefs	22
	Soso	22
	Touche	22
	NetStat.RuAgent	22
	GameSpyHTTP/1.0	22
	ConveraCrawler/0.9e	22
	HTTP	22
	Mercator	22
	AdsBot-Google-Mobile-Apps	22
	Yahoo!-MMCrawler/3.x	22
	gigablast	22
	WebReapervWebReaperv7.3-www,otway.com/webreaper	22
	HenryTheMiragoRobot	22
	YandexDirectDyn	22
	UrlDispatcher	22
	WebReaperv9.1-www.otway.com/webreaper	22
	WebsiteQuester-www.asona.org	22
	Snappy	22
	WebReaperv9.7-www.webreaper.net	22
	CherryPickerSE	22
	MMCrawler/3.x	22
	radian	22
	twiceler.	22
	mirago	22
	WebsiteQuester-www.esalesbiz.com/extra/	22
	Tasapspider	22
	Yahoo-NewsCrawler	21
	NutchCVS	21
	Mail.Ru/1.0	21
	Birubot	21
	Caliperbot/1.0	21
	MyEngines-Bot	21
	toCrawl	21
	Typhoeus	21
	CherryPickerElite	21
	//crawler.sistrix.net/)	21
	OutfoxBot	21
	pimonster	21
	bot*	21
	YadirectBot	21
	Veooz	21
	//www.thefind.com/crawler)	21
	EzoomsRobot	21
	SEOlyticsCrawler	21
	Lycos_Spider_(T-Rex)	21
	DomainSigmaCrawler	21
	Pixray	21
	Cowbot	21
	BLEXbot	21
	CloudFlare-AlwaysOnline	21
	Lycos	21
	ApptusBot	21
	Igentia	21
	Moni	20
	Haste	20
	dcbspider	20
	GoForIt	20
	YandexImages/3.0	20
	JemmaTheTourist	20
	"*"	20
	IlseBot	20
	Searchspider	20
	MMCrawler	20
	Butterfly	20
	Spider_Monkey	20
	Georgios	20
	Pi-Monster	20
	gridBOT	20
	obot	20
	flatlandbot	20
	HaoSouSpider	20
	aranhabot	20
	Inktomi	20
	speedy	20
	MSFrontPage	20
	Molbsy	20
	ADmantX	20
	Iron33	20
	KnowItAll	20
	Adidxbox	20
	DealOzBot	20
	USyd-NLP-Spider	20
	woobot	20
	sna	20
	VegeBot#allVegeBotservices	20
	TAMU_CS_IRL_CRAWLER	20
	YRSPider	20
	Nikto	20
	BNSBot	20
	foobar	20
	iCcrawler	20
	InfoPath.2	20
	[email protected]	20
	sohu-search	20
	Vegibot	20
	LinkdexBot	20
	NPT	20
	VSE/1.0	20
	Pimonster	20
	WWW	20
	IXECrawler	20
	SuperGet	20
	rj302004.crawl.yahoo.net	19
	cXensebot	19
	WotBox	19
	YandexVideoParser	19
	MicrosoftURL	19
	abayonne-256-1-89-49.w90-45.abo.wanadoo.fr	19
	spider23.picsearch.com	19
	search.live.com	19
	solomono	19
	Netcraft	19
	um-FC	19
	MuscatFerret	19
	RankActiveLinkBot	19
	WebTrends	19
	rj302006.crawl.yahoo.net	19
	Y!J-ASR	19
	spider25.picsearch.com	19
	AlexaBot	19
	rj302005.crawl.yahoo.net	19
	ACONTBOT	19
	crawler.bloglines.com	19
	METASpider	19
	rj302007.crawl.yahoo.net	19
	endecawebcrawler	19
	ImageScapeRobot	19
	atoulouse-257-1-118-62.w82-125.abo.wanadoo.fr	19
	VeriCiteCrawler	19
	msnbot/2.0b	19
	WinHttp	19
	MnoGoSearch	19
	rj302008.crawl.yahoo.net	19
	onlinehome-server.info	19
	ImageScapeRobot([email protected])	19
	RadiationRetriever	19
	CrazyWebCrawler	19
	//www.asona.org)	19
	MetaURI	18
	Picsearch	18
	JoBo	18
	GetFoundBot	18
	aboundex/0.3	18
	BotOnParade	18
	FlickBot	18
	webcopier	18
	SpiderJack	18
	w2gbot	18
	//www.SearchEngineWorld.combot	18
	FatBot2.0	18
	dloader	18
	Memo	18
	MixBot	18
	Intelix	18
	Mirror	18
	Slurp.so/1.0	18
	SkimBot	18
	Slurp/3.0-AU	18
	//www.WebmasterWorld.combot	18
	Detectify	18
	//www.seokicks.de/robot.html)	18
	PHP	18
	Yasni	18
	yahoobot	18
	Slurp/2.0-OwlWeekly	18
	linguatools	18
	webvac	18
	Shelob	18
	Isidorus	18
	bumblebee	18
	Slurp/2.0j	18
	W3C_Validator	17
	Website	17
	DownloadWonder	17
	Perl	17
	Extreme\Picture\Finder	17
	our\agent	17
	Heretrix	17
	Vintage	17
	httpscraper	17
	LinqiaMetadataDownloaderBot	17
	SocialRankIOBot	17
	Openfind\data\gatherer	17
	nexuscache	17
	fireball	17
	GetSmart	17
	Xara	17
	Webhook	17
	Webdownloader	17
	Siteimprove	17
	Alligator	17
	ColdFusion	17
	FileHound	17
	[email protected]	17
	anarchie	17
	Link	17
	Y!TunnelPro	17
	LinqiaScrapeBot	17
	Webster\Pro	17
	Qualidator	17
	DA	17
	Xaldon	17
	onestop	17
	FAST\WebCrawler	17
	WellsSearchII	17
	Net\Probe	17
	Konqueror	17
	80bot	17
	WebCopy	17
	ZBot	17
	PHPot	17
	Sqworm/2.9.85-BETA(beta_release;20011115-775;i686-pc-linux	17
	slysearch	17
	Missigua	17
	HTTPviewer	17
	Fetch\API\Request	17
	Bot\mailto	17
	QueryN	17
	laycat	17
	NetZip	17
	Copyscape	17
	PingALink\Monitoring\Services	17
	Voyager	17
	semager	17
	httpfetcher	17
	Mozilla/4.0(compatible;NetcraftWebServerSurvey)	17
	Web\Downloader	17
	PHP\version	17
	Rico	17
	Stealer	17
	Websucker	17
	B2w	17
	Libby_	17
	clsHTTP	17
	Websites	17
	ClariaBot	17
	Webminer	17
	TwengaBot/2.0	17
	Google-bot	17
	googlebot-news	17
	BackStreet	17
	FreeFind.com	17
	YandexBlog	17
	AISearchBot	17
	WebReaperv9.8-www.webreaper.net	17
	fr-crawler	17
	lwp\request	17
	GigablastOpenSource	17
	HitboxDoctor	17
	Downloader	17
	NationalDirectory\WebSpider	17
	Zookabot	17
	GosoSpider	17
	*	17
	pingdom	17
	libwwwperl	17
	WinHttpRequest	17
	COAST\WebMaster	17
	Ping	17
	Mewsoft\Search\Engine	17
	HTTPapp	17
	Diamond	17
	Python\urllib	17
	Seekbot	17
	PerlLWP	17
	sogouwebspider	17
	DISCo\Pump	17
	Vayala	17
	Metacarta	17
	Webmirror	17
	Wells	17
	PageAnalyzer	17
	RAMPyBot	17
	Wildsoft\Surfer	17
	Jonzilla	17
	HTTPTrack	17
	Nmap	16
	Metauri	16
	berlin-fu-cow	16
	008/0.85	16
	w3mir	16
	SEOstats	16
	AllSubmitter	16
	MarkMonitor	16
	Voil	16
	Anarchie	16
	MarkWatch	16
	Sqlworm	16
	HTMLparser	16
	WebsiteExtractor	16
	Meltwater	16
	Joomla	16
	WebEmailExtrac	16
	Brandprotect	16
	Extractor	16
	Devil	16
	YioopBot	16
	InfoTekies	16
	Turingos	16
	grub-client-1.2.1	16
	vobsub	16
	Havij	16
	Demon	16
	AskTbORJ	16
	Badass	16
	BuiltWith	16
	WebMirror	16
	T0PHackTeam	16
	Evil	16
	Telesphorep	16
	T8Abot	16
	WhitevectorCrawler	16
	8LEGS	16
	Yandex*	16
	Twice	16
	InfoSpiders	16
	ECCP/1.0	16
	Nameprotect	16
	Mechanize	16
	AIBOT	16
	Bigfoot	16
	Googlebot-Image/1.o	16
	LeechGet	16
	bingbot-mobile	16
	Clewwa-Bot	16
	BBBike	16
	Trendictionbot	16
	msnbot-media/1.1	16
	Apexoo	16
	LinqiaRSSBot	16
	Openvas	16
	Ripper	16
	WEBDAV	16
	WebFuck	16
	Blow	16
	GetWeb	16
	teleport	16
	PyCurl	16
	Brandwatch	16
	WebFetcher	16
	WebCatcher	16
	Picscout	16
	Cosmos	16
	POE-Component-Client-HTTP	16
	WebPix	16
	Whack	16
	VidibleScraper	16
	musobot	16
	WISENutbot	16
	OrangeSpider	16
	voyager/1.0	16
	Telesphoreo	16
	Disco	16
	Covario-IDS/1.0	16
	UrlPouls	16
	Sucuri	16
	YahooSlurp	16
	SpiderBot	16
	GridBot	16
	RepoMonkey*	16
	CSHttp	16
	Cogentbot	16
	CPython	16
	abot	15
	FurlBot	15
	Appie	15
	Ingrid	15
	MRSPUTNIK	15
	Pipl	15
	iskanie	15
	Accoona	15
	DSurf	15
	YandexCatalog	15
	MailSweeper	15
	FairShare	15
	yoozBot	15
	python-requests	15
	A6-Indexer	15
	123People	15
	xGet	15
	PangusoSpider	15
	AntenneHatena	15
	facebot	15
	nutch-solr-integration	15
	Mail	15
	Yahoo-Blogs/v3.9	15
	EliteSysEntry	15
	AlvinetSpider	15
	Templeton	15
	WebWalk	15
	Mediapartners	15
	munky	15
	seoengbot	15
	//www.flamingosearch.com/bot)	15
	taptubot	15
	SnapPreviewBot	15
	Seznambot	15
	GurujiBot	15
	atSpider	15
	VIREL	14
	Discobot	14
	MyFamilyBot	14
	grapeFX	14
	Pingdom.com_bot	14
	yanga	14
	WorldBrewBot	14
	Y!J-BSC	14
	trovit	14
	facebookexternalhit/1.0	14
	Webwhacker	14
	//www.opensiteexplorer.org/dotbot,[email protected])	14
	GetBot	14
	Zyborg	14
	jobrapido	14
	TurnitinBot/2.1	14
	vilainrobot	14
	//www.cuill.com/twiceler/robot.html	14
	Ruby	14
	gonzo1	14
	twengabot	14
	moget*	14
	PhpDig*	14
	woozweb-monitoring	14
	seoscanners.net/1	14
	IntegromeDB	14
	Wget/1.10.2	14
	YahooPipes2.0	14
	yodaobot	14
	Webzip	14
	HappyFunBot	14
	MentorMateSpider	14
	deusu	14
	sprobot	14
	MyOnID	14
	IssueCrawler	14
	Java/1.6.0_10	14
	AdsBot-Google-Mobile	14
	KFSW-Bot	14
	polybot	14
	LightningDownload	14
	penthesilea*	14
	DiamondBot	14
	intelium_bot	14
	MSIE	14
	inktomi	14
	knowaboutBot	14
	EMail\Wolf	13
	Summify	13
	datagnionbot	13
	Web\Fuck	13
	needle	13
	boardreader	13
	ebingbong	13
	mojeek	13
	Go-http-client	13
	Shai'Hulud	13
	GetUrl	13
	java	13
	lanshanbot	13
	nettrack	13
	WebDownloader	13
	ScooperBot	13
	sqlmap	13
	UltraSeek	13
	KDDExploror	13
	Web\Reaper	13
	MSSearch	13
	iblog	13
	postrank	13
	Page\Analyzer	13
	searchestate	13
	CyberPatrol	13
	LapozzBot	13
	QueryN\Metasearch	13
	Page\Grabber	13
	EMail\Collector	13
	Web\Whacker	13
	Battleztar\Bazinga	13
	jbrofuzz	13
	gonzo1P	13
	webcollage	13
	MSIE\6.0	13
	calculon\spider	13
	Yandex	13
	imagefetch	13
	Web\Fetch	13
	MS\Web\Services\Client\Protocol	13
	Website\Extractor	13
	HavIndex	13
	seomoz	13
	msrabot	13
	whatweb	13
	gonzo2	13
	Web\Bandit	13
	nessus	13
	spammen	13
	nikto	13
	sitebeam	13
	nibbler	13
	Web\Pix	13
	VB\Project	13
	freeuploader	13
	Microsoft\Data\Access	13
	Sogou\web\spider	13
	webshag	13
	//pubget.com/help/bot)	13
	80legs.com	13
	gonzo2P	13
	Turnitin\Bot	13
	Site\Sucker	13
	MaxPointCrawler/Nutch-1.10	13
	EMail\Siphon	13
	Holmes	13
	Web\Stripper	13
	zgrab	13
	Web\Enhancer	13
	Collage	13
	libwhisker	13
	ssearcher100	13
	WEBDAV\Client	13
	getintent	13
	PBWF	13
	Lipperhey	13
	PagesInventory	13
	WiseGuys\Robot	13
	Web\Sauger	13
	squirrobot	13
	internetVista\monitor	13
	Landau-Media-Spider	13
	MSNBOT_Mobile	13
	Screaming\Frog\SEO\Spider	13
	dirbuster	13
	convera	13
	Name\Intelligence	13
	Image\Fetch	13
	ESIRover	13
	memoryBot	13
	Web\Copier	13
	InternetExplore	13
	Turnitin\Robot	13
	Download\Devil	13
	sitevigil	13
	Acoon	13
	MarketwireBot	13
	dragonfly	13
	fimap	13
	FuelMyRoute.com	13
	netEstate\NE\Crawler	13
	SISTRIX\Crawler	13
	Sitebot	13
	memoryBot*	13
	kraken	13
	EMail\Extractor	13
	flunky	13
	Spaidu	13
	Y!J-MBS/1.0	12
	LipperheySEOService	12
	//www.sogou.com/docs/help/webmasters.htm#07	12
	Covario	12
	SemanticScholarBot	12
	SafeSearchmicrodatacrawler	12
	Yahoo-MMCrawler/3.x	12
	Y!J-SRD/1.0	12
	citeseerxbot	12
	cb/nutch	12
	Sogou+web+spider	12
	Libwww-perl	12
	crawly	12
	Webbot	12
	^Pixray	12
	swebot	12
	DepSpid	12
	googlespider	12
	linksmanager_bot	12
	uipbot/1.0([email protected])	12
	WebCopierv3.3	12
	Vortex	12
	SogouPicSpider/3.0	12
	Moreoverbot	12
	Exabot/2.0	12
	linksmanager	12
	*#directedtoallspiders	12
	ZipppBo	12
	Slurp/2.0-Kite-Hourly	12
	Googlebot/2.1	12
	Pingdom	12
	Trendiction-Bot	12
	AhrefsBot/1.0	12
	NetSprint	12
	phpcrawl	12
	MantraAgent	12
	VoilaBot*	12
	lssrocketcrawler	12
	^echobot	12
	Bot	12
	GrubNG	12
	CyberAlert	12
	ICCrawler-ICjobs	12
	httpunit	12
	iisbot	12
	FatBot/2.0	12
	QihooBot	12
	MicrosoftURLControl*	12
	Nutch*	12
	OmnitureTestAndTargetCrawl	12
	BlogPulseLive	12
	TerrawizBot	12
	Yandex/1.01.001	12
	DTAAgent	12
	^Lydia	12
	MVAClient	12
	myurlcrawler	12
	Teoma#ASK.com	12
	//www.google.com/bot.html)	12
	BusinessWireBot	12
	WebAltaCrawler/2.0	12
	Socialradarbot/2.0	12
	InfoSeekSidewinder/9.0	12
	OpenXSpider	11
	JaxifiedBot	11
	PageBitesHyperBot	11
	sbider	11
	Ruky-Bot	11
	AppleWebKit	11
	Cityreview	11
	SogouOrionspider/3.0	11
	infometrics-bot	11
	Yasaklibot	11
	askpeter_jeanie	11
	JobRoboter	11
	PostPost	11
	Sputnik	11
	uMBot-FC	11
	vkShare	11
	sistrixcrawler	11
	iearthworm/1.0	11
	echobot	11
	Sidewinder	11
	jyxobot	11
	mogimogi	11
	ContextAdBot	11
	Nebullabot	11
	searchlink	11
	Yandex/1.01.001(compatible;Win16;P)	11
	newslookup-bot	11
	KoepaBot	11
	EsperanzaBot	11
	LinkChecker	11
	ExaBot	11
	Peew	11
	AnyApexBot	11
	website-datenbank.de	11
	YandexBot3.0	11
	iearthworm	11
	Mail.RU_Bot/Fast/2.0	11
	askpeter_bot	11
	SistrixCrawler	11
	TwengaBot1.x	11
	SimpleCrawler	11
	MJ12bot#onlythenewsservice	11
	VYU2	11
	Scoutjet	11
	Shoularobot	11
	BOT	11
	bingspider	11
	FAST-WebCrawler3.6	11
	NewsGator	11
	AddSugarSpiderBot	11
	wf84	11
	Sogou-Test-Spider/4.0	11
	envolk[ITS]spider	11
	RexyoBot1.11	11
	Orbiter	11
	Eurobot/1.0	11
	FAST-WebCrawler3.7	11
	AddThis	11
	Yahoo*	11
	NetSeerCrawler	11
	blekkobot	11
	NutchOrg	11
	//wortschatz.uni-leipzig.de/findlinks/)	11
	ArchiveTeamArchiveBot	11
	Netluchs	11
	Acunetix	11
	lwp*	11
	SearchSight	11
	LDSpider	11
	WomlpeFactory	11
	heise-IT-Markt-Crawler	11
	Twitter	11
	SetLinks	11
	updated	11
	Qseero	11
	Nymesis	11
	SocialShare	11
	MooseBot	11
	newslebot	11
	FAST-WebCrawler3.8	11
	Urlfilebot	11
	Sogouheadspider/3.0	11
	SetLinksbot	11
	EmailSearch	11
	NG-Search	11
	truwoGPS	11
	dloader(NaverRobot)	11
	yoogliFetchAgent	11
	FAST-WebCrawler3.x	11
	pixraybot	11
	Grapeshot	11
	FDSErobot	11
	Maxthon	11
	silk	11
	Mozilla/5.0(compatible;Ezooms/1.0;[email protected])	11
	g2crawler	11
	TheSuBot	11
	ConveraCrawler/0.9d	11
	mabontland	11
	PrivacyFinder	11
	oegp	11
	SetLinksbot2.0	11
	STIBcrawler	11
	Y!J	11
	Francis	11
	gyffu_bot	10
	mediajam-bot	10
	Sosoimagespider	10
	Linknzbot	10
	LDRbot	10
	Gigablast	10
	Sistrix	10
	aipbot*	10
	PHPCrawl	10
	NetinfoBot	10
	WikioImagesBot	10
	WiseGuysRobot	10
	MicrosoftURLControl-6.01.9782	10
	"Mozilla/5.0(Java)outbrain"	10
	//search.msn.com/msnbot.htm)	10
	MarketBrewBot	10
	Yeti/1.0	10
	Yet	10
	megaindex	10
	BPImageWalker/2.0	10
	focuseekbot	10
	HubSpotPageFetchingBot	10
	aipbot/1.0	10
	Newsbot	10
	Websquash.com	10
	SwayyAPI	10
	//ahrefs.com/robot	10
	duckduckbot	10
	BufferBot	10
	PHP/	10
	Linknzbot*	10
	Friendlyrobot	10
	Sogouwebspider/3.0	10
	EveryoneSocialBot	10
	findxbot	10
	Moozilla	10
	MissiguaLocator	10
	LWP	10
	JakartaCommons	10
	SeznamBot/3.0	10
	changedetection	10
	BadBot	10
	LWP*	10
	AcademicBot	10
	omniexplorer_bot	10
	L.webis	10
	//help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)	10
	admantx	10
	*#allowallbots	10
	lssbot	10
	ia_archiver*	10
	PictureBot	10
	//www.baidu.com/search/spider_jp.html)	10
	WebCrawler	10
	y!j-bsc	10
	HTTrack3.0x	10
	SinaWeiboBot	10
	RightIntelBot	10
	y!j/1.0	10
	EMailExractor	10
	DynamicTemplates/2.0	10
	NutchCVS/0.06-dev	10
	MorningPaper	10
	Y!J-BSC/1.0	10
	sReleaseBot	10
	htdig/3.1.5	10
	ZoomBot	10
	Ahrefsbot	10
	Esribot	10
	siteexplorer.info	10
	facebookexternalhit/*	10
	8484BostonProjectv1.0	10
	NG\1.x(Exalead)	10
	montastic-monitor	10
	dtSearchSpider	10
	Linknzbot2004	10
	EnaBot	10
	fr_crawler	10
	//www.cuill.com/twiceler/robot.html)	10
	WhoWhere	10
	xintellibot	10
	Lipperhey-Kaus-Australis	10
	WX_mail/2.000	10
	YahooYSMcm/2.0.0	10
	robotleapit.com	10
	wombatbot	10
	Yandex.GazetaBot	10
	Ask	9
	//www.google.com/adsbot.html)	9
	Web.Imge.Collector	9
	netresearchserver	9
	libwww	9
	OpenISearch	9
	Majestic12	9
	drupact	9
	Lite	9
	crawler	9
	fluffy	9
	Yahoo!Slurp/3.0	9
	BeslistBot	9
	NewsNow	9
	Gooblebot	9
	Yandex/1.01.001(compatible;Win16;H)	9
	RealSpider	9
	Yandex/1.01.001(compatible;Win16;I)	9
	WWW-Mechanize	9
	siteimprove	9
	toplistbot	9
	binlar	9
	domaincrawler	9
	Sysomos	9
	EmeraldShield.com	9
	RetrevoPageAnalyzer	9
	//github.com/typhoeus/typhoeus	9
	urlappendbot	9
	//www.picsearch.com/bot.html)	9
	MojeekBot\/	9
	MetaJobBot	9
	Search	9
	eknip	9
	deadlinkchecker	9
	webalta	9
	Commons-HttpClient	9
	backlinkcrawler	9
	Mozilla/4.0(compatible;Zealbot1.0)	9
	Megalodon	9
	g00g1e.net	9
	openindexspider	9
	ec2linkfinder	9
	DeuSu\/	9
	Fetch	9
	MSNPTC	9
	KomodiaBot	9
	biglotron	9
	webcompanycrawler	9
	Cliqzbot\/	9
	wGet	9
	gnamgnamspider	9
	Twicelerwww.cuill.com/twiceler/robot.html	9
	CapsuleChecker	9
	WebEMailExtrac	9
	panscient	9
	europarchive.org	9
	web-archive-net.com.bot	9
	//tineye.com/crawler.html)	9
	yandeximages	9
	scribdbot	9
	SlurpChina	9
	Nusearch	9
	//www.dotnetdotcom.org/,[email protected])	9
	gslfbot	9
	//research.microsoft.com/research/sv/msrbot/)	9
	Trove	9
	OfflineExplorer/1.5	9
	findlink	9
	SemrushBotSemrushBot-SA	9
	UsineNouvelleCrawler	9
	BorderManager*	9
	WebduniaBot	9
	*\	9
	buzzbot	9
	yahoo-blogs	9
	brainobot	9
	edisterbot	9
	bender	9
	Mozilla/4.0	9
	//feedback.redkolibri.com/	9
	Sonic	9
	d3d9446802a44259755d38e6d163e820	9
	b2w	9
	ngbot	9
	NTENTbot	9
	Livelapbot	9
	[email protected]	9
	//www.kototoi.org/zao/)	9
	//www.searchtechnologies.com)	9
	integromedb	9
	webzip	9
	IBM_Planetwide	9
	wocbot	9
	nerdybot	9
	webmon	9
	BDCbot	9
	//fulltext.sblog.cz/screenshot/)	9
	IOI	9
	bibnum.bnf	9
	GlutenFreeCrawler\/	9
	AddThis.comrobot	9
	Mozilla/4.0(compatible;MSIE4.0;Windows9	9
	IRLbot/3.0	9
	CCMetadataScaper	9
	MixrankBot	9
	elisabot	9
	contentcrawlerspider	9
	y!j-asr	9
	AcoonBot	9
	acoonbot	9
	bnf.fr_bot	9
	Googlebot-Image/1.0	9
	arabot	9
	//www.semrush.com/bot/	9
	//www.WISEnutbot.com)	9
	//webmeup-crawler.com/	9
	SeznamBot/3.2	9
	OmniExplorer	9
	sogoudevelopspider	9
	simplepie	9
	//help.naver.com/robots/)	9
	findthatfile	9
	tagoobot	9
	HeinrichderMiragoRobot	9
	VoilaBotBETA1.2	9
	Java*	9
	seokicks	9
	google-proxy	9
	cometrics-bot	9
	* #*forallagents	9
	spock	9
	Psbot-Imagesearch	9
	SemrushBot/0.92	9
	page2rss	9
	grub.org	9
	summify	9
	rabaz	8
	istellabot/t.1	8
	ezooms.bot	8
	LemurWebCrawler	8
	BecomeBot/3.0	8
	Exabot-Images/1.0	8
	iCjobs/3.2.3	8
	Artabus	8
	um-IC	8
	WebReaperv9.7Ã¢â‚¬â€œwww.webreaper.net	8
	WebSnake	8
	OfflineCommander	8
	iCcrawler-iCjobsStellenangeboteJobs	8
	ichiro/2.0	8
	Owlin	8
	YandexImages*	8
	Buzzbot	8
	SapphireWebCrawler/1.0	8
	PeterBot	8
	Mozilla/4.0(compatible;MSIE4.01;WindowsNT;MSSearch5.0Robot)	8
	coc_coc	8
	MicrosoftURLControlÃ¢â‚¬â€œ5.01.4511	8
	Arachnoidea	8
	MissiguaLocator1.9	8
	Mozilla/4.0(compatible;MSIE6.0;WindowsNT;MSSearch5.0Robot)	8
	sproose	8
	Zing-BottaBot/2.0	8
	MSRBot	8
	NGBot/4.5	8
	adnbot	8
	Twicelerwww.cuill.com/robots.html	8
	ISCSystemsiRcSearch2.1	8
	Java/1.6.0_13	8
	Mozilla/4.0(compatible;MSIE4.01;WindowsNT;MSSearch4.0Robot)	8
	GSiteCrawler	8
	PingdomGIGRIB	8
	[email protected]	8
	ZipppBot	8
	//www.majestic12.co.uk/projects/dsearch/mj12bot.php	8
	dloader(SpeedySpider)	8
	Bitaclebot/1.1	8
	//help.goo.ne.jp/door/crawler.html)	8
	CCBot/1.0	8
	WebsiteQuesterÃ¢â‚¬â€œwww.esalesbiz.com/extra/	8
	//misc.yahoo.com.cn/help.html)	8
	TutorialCrawler	8
	boitho	8
	SEMrushBot-Desktop	8
	oodlebot/1.0	8
	Leech	8
	ImplisenseBot	8
	webmeasurement-bot	8
	ptd-crawler	8
	//moz.com/researchtools/ose/dotbot	8
	Pythonurllib	8
	WebReaperv9.8Ã¢â‚¬â€œwww.webreaper.net	8
	Arachmo	8
	//siteexplorer.info/Backlink-Checker-Spider/	8
	LinkWalker/2.0	8
	zibber	8
	Jetslide	8
	PingdomGIGRIBv1.1	8
	OfflineExplorerPro	8
	RU_Bot	8
	HMSE_Robot	8
	magpie-crawler*	8
	webmeup-crawler	8
	WebReaperv9.1Ã¢â‚¬â€œwww.otway.com/webreaper	8
	AMZNKAssocBot	8
	Snapbot/1.0	8
	SurveyBot/2.3	8
	Zend_Http_Client	8
	WebsiteQuesterÃ¢â‚¬â€œwww.asona.org	8
	//www.qwant.com/	8
	KINGSpider	8
	MicrosoftURLControlÃ¢â‚¬â€œ6.00.8169	8
	*	8
	*#matchallbots	8
	WebReapervWebReaperv7.3Ã¢â‚¬â€œwww,otway.com/webreaper	8
	Infovell	8
	Netvibes	8
	Bitaclebot	8
	HDnutchagent/Nutch-1.1(Think)	8
	AutoSpider1.0	8
	Googlebot#robotalquevadirigido	8
	LoadTimeBot	8
	//www.exalead.com/search/webmasterguide	8
	sg-Orbiter	8
	linkwalker	8
	COMODOSpider/Nutch-1.2	8
	mindUpBot	8
	OSCE	8
	Sosospider/2.0	8
	FollowSiteBot	8
	WebDataCentreBot	8
	SixtrixCrawler	8
	SurveyBot/2.3(WhoisSource)	8
	HTTPWeazel	8
	//urlm.nl/	7
	Exalead	7
	Semager	7
	StratagemsKumo	7
	//moz.com/help/guides/moz-procedures/what-is-rogerbot	7
	FeedBooster	7
	yandex*	7
	//napoveda.seznam.cz/en/seznamcz-web-search/crawling-control/	7
	//www.twenga.com/bot.html	7
	peekyou	7
	wikiwix-bot	7
	Bad-Robot	7
	ichiro/2.0([email protected])	7
	Searcharoo	7
	webcollage/1.77	7
	naughtyrobot	7
	Sogouweb	7
	linkdexbot-mobile/2.1	7
	Wget/1.9.1	7
	Jobrapido	7
	BigBrother	7
	InktomiSlurp	7
	Mozilla/4.0(compatible;B-l-i-t-z-B-O-T)	7
	Bitacle*	7
	snitch	7
	netEstate	7
	yodao	7
	Java/1.6.0_07	7
	TurnitinRobot	7
	Exabot-Images	7
	megaindex.com/crawler	7
	mediapartners-google	7
	NetSeer	7
	LipperheySpider	7
	HttpTool*	7
	BecomeJPBot	7
	FetchAPIRequest	7
	FHscan	7
	wink	7
	Apple-PubSub	7
	*#directedtoallrobots	7
	Sproose	7
	discovery	7
	Java/1.5.0_06	7
	CMSCrawler	7
	SEOKicks	7
	SEOKicks-Robot	7
	//www.wise-guys.nl/webcrawler.php	7
	legs	7
	//majestic12.co.uk/bot.php?+)	7
	zoominfo	7
	bingbot #topreventbingbogfromcrawlingtoomuch	7
	SearchmetericsBot	7
	baidu*	7
	businessdbbot	7
	//yandex.com/support/webmaster/controlling-robot/robots-txt.xml	7
	StackRambler/2.0	7
	Pingdom.com	7
	tweepz	7
	Charlotte/1.0b	7
	Owlinbot	7
	ExaleadCloudView	7
	Bilbo/2.3b-UNIX	7
	[email protected])"	7
	bluemasterbot	7
	ICCrawler	7
	larbin_2.6.1	7
	WebDownloader/5.7	7
	Curl	7
	Pcore-HTTP	7
	opera	7
	seostats	7
	rapleaf	7
	DataSpear	7
	rogerBot	7
	5	7
	aihit	7
	bspider	7
	WebsiteWiki	7
	/images/	7
	Java/1.5.0_05	7
	ReadCube	7
	NexaBot	7
	yoname	7
	wiederfreibot/1.0	7
	OWR_Crawler	7
	//help.yahoo.com/kb/SLN22600.html	7
	YandexSomething/1.0	7
	slurp@inktomi	7
	YandexAccessibilityBot	7
	YahooSeeker/CafeKelsa	7
	fairshare.cc	7
	newsexpress	7
	Zing-BottaBot/1.0	7
	Gigabot*	7
	SEOprofiler	7
	PricecloudBot	7
	InfoPath.1	7
	uw_cse_xwc	7
	Veoozbot	7
	Ichiro	7
	LMQueueBot/0.2	7
	LinksManager	7
	scoutjet	7
	allowfromall	7
	iaskspider	7
	BBot	7
	yourtraces	7
	Y!J-BRJ/YATScrawler	7
	DoCoMoSpider	7
	coccocbot-web	7
	funnelback	7
	ZyBorg/1.0	7
	SiteSeekerCrawler/1.0	7
	zyborg	7
	MSNBot-NewsBlogs	7
	findestars	7
	PGBot	7
	msnbot #topreventmsnbotfromcrawlingtoomuch	7
	FAST-WebCrawler/3.6/FirstPage	7
	360spider	7
	WillyBot	7
	OpenX	7
	spinn3r	7
	zermelo	7
	bingbot*	7
	crawl6.exabot.com	6
	genieBot([email protected])	6
	HTTP/1.1zibber-v0.1(www.zibb.com/crawler/)	6
	Slurp/2.0-KiteHourly	6
	WebCopierv3.3.0	6
	YandexBot/3.0-MirrorDetector	6
	nexen	6
	Snappybot	6
	orderallow,deny	6
	classroombot	6
	Wada.vnVietnameseSearch/2.1	6
	Mozilla/3.0(compatible;IndyLibrary)	6
	webbandit	6
	Java/1.5.0_10	6
	//majestic12.co.uk/bot.php?)	6
	XenuLinkSleuth1.2i	6
	PSIBots	6
	moget/2.1([email protected])	6
	Webspider	6
	www.websuche.de-spider	6
	ApexooSpider1.0	6
	voyager/2.0	6
	SBIder/SBIder-0.8.2-dev	6
	Cabot	6
	SlySearch/1.0	6
	//www.page-store.com)	6
	YahooFeedSeeker	6
	/sbconf/	6
	//search.yahoo.com/)	6
	T-OnlineBrowser	6
	page_prefetcher	6
	/enc/	6
	ru_com_viewer	6
	Aghaven	6
	therarestparser	6
	GOOGLEBOT	6
	zibberv0.1	6
	CiZilla	6
	del.icio.us	6
	YoudaoBot/1.0	6
	Exabot-Images/3.0	6
	aiHitBot/1.1	6
	AttributorCorporationDMCABot	6
	JavaBot	6
	LaBot	6
	OmniExplorer_Bot/1.09	6
	//irl.cs.tamu.edu/crawler)	6
	/storeadmin/	6
	QEAVis	6
	whoisde.de	6
	uberbot	6
	SuperBot/4.4.0.60(WindowsXP)	6
	fast-Webcrawler	6
	percbotspider	6
	compatible;	6
	seolytics	6
	JavaBot2.0	6
	worio	6
	//www.become.com/site_owners.html)	6
	/www.innerprise.net/usp-spider.asp)	6
	librabot	6
	MagpieRSS	6
	Scooter/2.0	6
	Bork-edition	6
	StackRambler/2.0+(MSIE+incompatible)	6
	findlinks/1.1.3-beta9	6
	Axonize-bot	6
	RiceComputerArchitecture	6
	Wada.vnVietnameseSearch	6
	Caliperbot	6
	msnbot/1.1	6
	WebDownloader/6.9	6
	IBMEVV	6
	ASPSimply	6
	AdIdxbot	6
	Findexa	6
	httpclient	6
	//www.website-datenbank.de/)	6
	Mozilla/4.0(compatible;MSIE5.05;WindowsNT4.0)	6
	libcurl-agent	6
	aiHitBot/1.0	6
	cyberpatrol	6
	Qwantify/2.3w	6
	Webcollage	6
	fatbot	6
	OutfoxBot/0.5	6
	fastmetawebcrawler	6
	LexiBotWebImageCollector	6
	YandexOntoDBAPI	6
	ZapBot	6
	LinkWalker/2.0-www.seventwentyfour.com	6
	crawl13.exabot.com	6
	sitecat	6
	Win32	6
	Googlebot-mobile	6
	GoForIt.com	6
	Cazoodle-Bot	6
	sphsearch	6
	whois.de	6
	MouseBOT	6
	libwww-perl/5.65	6
	noxtrumbot/1.0	6
	VeBot	6
	Gasbot	6
	wise-guys	6
	AmazonCloudServices	6
	Acoon-Robot	6
	SeznamBot/1.0	6
	python	6
	Everest-Vulcan	6
	denyfrom82.99.30.	6
	nambu	6
	speedyspider	6
	Yandex/1.01.001(compatible;Win16;m)	6
	OnTownsBot	6
	TeleportProMIIxpc	6
	LocalBot	6
	SearchBot	6
	R6_FeedFetcher(www.radian6.com/crawler)	6
	Blexbot	6
	NationalDirectory-SuperSpider	6
	YandexSomething	6
	RepoMonkeyBait&Tackle	6
	[email protected]	6
	botmobi	6
	IntegraTelecom	6
	+BecomeBot/3.0	6
	InfluenceBot	6
	//www.turnitin.com/robot/crawlerinfo.html	6
	searchme	6
	paraisobot	6
	Mozilla/4.0+compatible+ZyBorg/1.0+	6
	SensisWebCrawler(search_comments\\at\\sensis\\dot\\com\\dot\\au)	6
	Wget/1.9+cvs-stable(RedHatmodified)	6
	GWPImages	6
	ViolaBot	6
	bsalsa	6
	OpenWebIndex/Nutch-1.6	6
	Ichiro3.0	6
	AdsBot-google	6
	phpversion	6
	PRTGCloudBot	6
	VCIVCI	6
	filterdb.iss.net/crawler	6
	eventax	6
	Websense	6
	Fess	6
	Piffany_Web_Cacher_v0.91	6
	BobCrawl	6
	HornySexSearch	6
	ExDomain	6
	aihitdata	6
	larbin_2.6.1+([email protected])	6
	WEPSearch	6
	ServageRobot	6
	Mozilla/4.0(compatible;MSIE6.0;WindowsNT5.0)	6
	Sensis+Web+Crawler+	6
	OmniExplorer_Bot/1.07	6
	LinkAider	6
	ActiveTouristBot	6
	internetseer	6
	Vocus	6
	/svgallery/	6
	User-Agent	6
	//;[email protected])	6
	ExaLeadCrawler	6
	WebarooBot	6
	Java/1.5.0_11	6
	MaxPointBot	6
	^Jakarta	6
	SmabblerBot	6
	YRSpider	6
	AdsBot	6
	setooz	6
	DTSAgent	6
	/template/	6
	//www.jaxcarpentry.com/sitemap.xml	6
	EUDORA	6
	Crowsnest	6
	Plista	6
	SeznamBot/1.1	6
	WebDownloader/4.5	6
	ThumbSniper	6
	MJ12bot*	6
	golabbot	6
	JobCrawlerBot	6
	fastEnterprisecrawler	6
	findlinks/2.6	6
	sogou+spider	6
	SuperPages	6
	YandexSomething/1.	6
	GarlikCrawler	6
	FunWebProductsf39-2350	6
	ZupeeCrawler	6
	Advista	6
	perl	6
	TencentTraveler	6
	/www.spidersoft.com)	6
	Mozilla/2.0(compatible;AskJeeves/Teoma)	6
	kyluka	6
	^Java	6
	RootleCrawler	6
	Wada.vn	6
	Avant+Browser+	6
	ModifiedGMMBotRightHere	6
	Slicehost	6
	//discoveryengine.com/discobot.html)	6
	eagle	6
	majestic	6
	shopwiki	6
	WGet	6
	ia_archive	6
	tarspider	6
	Servage	6
	boitho.com	6
	/secure/	6
	wegobot	6
	IBM	6
	complex_network_group	6
	AITCSRobot	6
	arks	6
	genieBot+([email protected])	6
	Linkdex	6
	FacebookExternalHit	6
	CovarioIDS	6
	/www.asona.org)	6
	LinkStats	6
	True_Robot/1.0turingos	6
	sohuagent	6
	RedCarpet	6
	exooba	6
	VWBot	6
	urllib	6
	centrumbot	6
	[email protected]	6
	OmniExplorer_Bot/1.10	6
	MJ12bot/	6
	Accoona-AI-Agent	6
	datacha0s/2.0	6
	T-Rex	6
	/go/	6
	DjangoTraineeBot	6
	Sika	6
	SmabblerBot/1.0	6
	Cutbot	6
	InnovantageBot	6
	//help.yahoo.com/help/us/ysearch/slurp)	6
	Mozilla/4.0(compatible;MSIE5.0;Windows95)VoilaBotBETA1.2	6
	Mozilla/4.0(compatible;MSIE4.01;WindowsNT;MSSearch6.0Robot)	6
	AideRSS2.0	6
	robotgenius	6
	MJ12bot/v1.2.0	6
	Directcrawler	6
	SlySearch/1.x	6
	WebCopierv3.3.2	6
	del.icio.us-thumbnails/1.0	6
	OpenAcoon	6
	[A-Z][a-z]{3,}[a-z]{4,}[a-z]{4,}	6
	findlinks/1.1.1-a5	6
	dcbot	6
	cyberpatrolcrawler	6
	yoogliFetchAgent/0.1	6
	meds-online24.com	6
	Mozilla/4.0(compatible;MSIE5.05;WindowsNT5.0)	6
	REAP-Crawler	6
	backlink-check.de	6
	Mozilla/4.0(compatible;MSIE5.05;WindowsNT3.51)	6
	//www.jobrapido.com)	5
	MJ12bot/v1.0.7	5
	DeepnetExplorer	5
	AdMediabot	5
	Mozilla/5.0	5
	SBider	5
	msnbot*	5
	msn	5
	Adidum	5
	Tarantula	5
	//www.dataprovider.com/spider/	5
	dubaiindex	5
	spider_monkey	5
	AportWorm	5
	cfetch/1.0	5
	Voila	5
	baiduspider-video	5
	Google-Sitemaps	5
	Siteliner	5
	DataproviderSiteExplorer	5
	//openlinkprofiler.org/bot	5
	WebVulnScan	5
	CloudServerMarketSpider	5
	Spanner	5
	googlebot-video	5
	DuckDuckBot/1.1	5
	StanfordCompClub	5
	MJ12bot/v1.0.8	5
	Java/	5
	//www.xovibot.net/	5
	InternetSeer	5
	EmailWolf1.00	5
	MJ12bot/v1.0.5	5
	istellabot-nutch/Nutch-1.10	5
	Yankex	5
	Nutraspace	5
	jcrawler	5
	Jooblebot	5
	Bing	5
	PathDefender	5
	MJ12bot/v1.2.3	5
	WordPress*	5
	MJ12bot/v1.2.4	5
	gooblogsearch	5
	oBot/2.3.1	5
	//www.linkdex.com/en-us/about/bots/	5
	MJ12bot/v1.0.6	5
	BuzzSumo	5
	Moreover	5
	Mozilla/5.0(JobRapidoWebPump)	5
	Proximic	5
	Diffbot/0.1	5
	Node/simplecrawler	5
	OwlinBot	5
	LoggerManager	5
	OpenWebSpider	5
	RedBot	5
	WebWasher	5
	OffByOne	5
	ActiveAgent	5
	orangebot	5
	bixolabs	5
	waybackarchive.org	5
	turnitin	5
	LinkAlarm	5
	NING/1.0	5
	alltheweb	5
	portalU	5
	Bitvorebot	5
	WebCopier*	5
	ConveraMultiMediaCrawler	5
	StanfordCompSciClub	5
	Majestic-12	5
	baiduimagespider	5
	netEstateFOAFcrawler	5
	FeedBurner	5
	192.comAgent	5
	wakame	5
	Linkdexbot	5
	FlamingoSearch	5
	lycos	5
	Morfeus	5
	WIJobRoboterSpider	5
	WebVulnCrawl	5
	FreeWebMonitoringSiteChecker/0.1	5
	BizInformasjon	5
	D115Crawler	5
	Yet-Another-Spider	5
	almaden	5
	gazz	5
	Slurp\	5
	InnyBot	5
	Yahoo!DESlurp	5
	URLspy	5
	larbin_2.6.3	5
	StanfordSpiderboys	5
	BlekkoBot	5
	msnbot/1.0	5
	MindCrawler	5
	iCorpusBot	5
	scrapy	5
	RB2B-bot	5
	wikiwix-bot-3.0	5
	FunnelBack	5
	AhrefsBot/5.2	5
	NaverBot-1.0	5
	WebIndex	5
	minibot(NaverRobot)/1.0	5
	innosense	5
	CityreviewRobot	5
	//www.jobrapido.com)"	5
	Sphider	5
	baiduspider-image	5
	newscan-online	5
	msnbot-media/	5
	aspider	5
	fido	5
	Jyxobot/1	5
	Googlebot	5
	PercolateCrawler	5
	Sensis.com.auWebCrawler	5
	HouxouCrawler	5
	SemrushBot*	5
	likebot	4
	jobo	4
	Java/1.7.0_51	4
	DuckDuckBot/1.0	4
	Scooter-3.2	4
	BuddhaBot	4
	NetcraftWebServerSurvey	4
	Y!J-BRJ/YATS	4
	Arachnophilia	4
	weborama-fetcher	4
	AgentLinkSpammer	4
	Scooter/[email protected]	4
	Jambot	4
	echo!	4
	Iron	4
	Scooter/3.3	4
	//URLFAN	4
	Seobility	4
	webreaper	4
	simplecrawler	4
	DemoBotDOT16b	4
	Scooter/2.0G.R.A.B.X2.0	4
	EBrowse1.4b	4
	wapspider	4
	Mozilla/4.0+(compatible;+MSIE+4.01;+Windows+NT;+MS+Search+6.0+Robot)	4
	warebay	4
	seznam	4
	Scooter2_Mercator_x-x.0	4
	Mozilla/4.5(compatible;HTTrack3.0x;Windows98)	4
	wbot	4
	Marketwirebot	4
	Onet.pl	4
	EmailExtractor	4
	Mozilla/2.0(compatible;AskJeeves)	4
	URLmetrics	4
	whowhere	4
	verifybot	4
	attentio	4
	Freecrawl	4
	webstripper	4
	excite	4
	WIRE/0.	4
	EchO!/2.0	4
	Psycheclone	4
	solbot	4
	W3C-checklink/	4
	/account/	4
	AVFetch1.0	4
	FullWebBot0516B	4
	Bloodhound	4
	Scooter-3.2.JT	4
	SynthesioCrawlerreleaseMonaLisa	4
	research-spider	4
	/*	4
	SiteCheck-sitecrawlbySiteimprove.com	4
	baiduspider-mobile	4
	DataCha0s/2.0	4
	Mozilla/5.0(compatible;Pogodak.co.yu/3.1)	4
	combine	4
	SogouPicAgent	4
	wowrack	4
	sitesnagger	4
	teleportpro	4
	[email protected]	4
	NetShelterContentScan	4
	emBot	4
	//www.openhose.org/bot.html)	4
	MetagerBot	4
	baiduspider-news	4
	BrandProtect	4
	Scooter_trk3-3.0.3	4
	spiderbot	4
	FDM3.x	4
	WebZIP/3.65	4
	Baiduspide*	4
	Dulancebot	4
	SiteSnagger*	4
	ebiness	4
	OkHttp	4
	msnbot-Products	4
	myfamilybot	4
	marvin	4
	baidumobaider	4
	Scooter/1.1(custom)	4
	DSurf15a81	4
	MicrosoftURLControl�6.00.8169	4
	Scooter/3.3.QA.pczukor	4
	Mozilla/4.0(compatible;Y!J;forrobotstudy;keyoshid)	4
	panoptaStudyBot	4
	Scooter/	4
	Scooter-3.0.FS	4
	Xenu�sLinkSleuth1.1c	4
	Linkpad	4
	webs	4
	webcollage*	4
	SpiderKU/0.9	4
	ContentSmartz	4
	FranklinLocator1.8	4
	sygol	4
	Synthesio	4
	Mozilla/5.0(compatible;Najdi.si/3.1)	4
	WebSauger*	4
	Fairshare	4
	fastcrawler	4
	Seekbot/1.0	4
	LincolnStateWebBrowser	4
	BrightEdgeCrawler/1.0([email protected])	4
	FullWebBot0416B	4
	radian6commentreader	4
	Wise-Guys	4
	Bitvore	4
	TsolCrawler	4
	twenga*	4
	ESurf15a15	4
	Butterfly/1.0	4
	default.ida	4
	waybackarchive	4
	butterfly	4
	Teleport*	4
	gosospider	4
	YaDirectBot	4
	*1	4
	Toweya	4
	AWSCloudBased	4
	ConveraMultiMediaCrawler/0.1	4
	SpiderLing	4
	hul-wax	4
	WebnewsArianna	4
	jikespider	4
	inspectorwww	4
	omgili	4
	Scooter_bh0-3.0.3	4
	ariadne	4
	DSurf15a01	4
	Getleft	4
	WebDownloader/5.8	4
	FamilyBot	4
	Scooter-3.0.HD	4
	DomainStatsBot	4
	NetZip-Downloader	4
	Tweetmeme	4
	GomezAgent	4
	ecollector	4
	griffon	4
	SEOsearch/	4
	perlcrawler	4
	Ultraseek-www	4
	anthill	4
	VadixBot	4
	openfind	4
	WeblexBot	4
	DBrowse1.4d	4
	Willow	4
	y!j-bsc/1.0	4
	GlobalSpecLinkChecker	4
	Scooter/3.3.vscooter	4
	Boomtrain-Content-Bot*	4
	BattleztarBazinga	4
	MicrosoftURLControl�5.01.4511	4
	[email protected]	4
	BIGLOTRON(BETA2;GNU/Linux)	4
	Scooter-3.2.DIL	4
	SputnikBot/2.3	4
	WebGather3.0	4
	WinInetTest	4
	NPBot/3	4
	shopstylebot	4
	YandexBot*	4
	Watching/UnitCrawler	4
	[email protected]	4
	Y!J/1.0	4
	Gaisbot/3.0	4
	FSurf15a01	4
	Mozilla/2.0(compatible;NEWTActiveX;Win32)	4
	Voinicsbot	4
	Scooter/1.0	4
	MSIE6.0	4
	DBrowse1.4b	4
	msnboot	4
	seckicks	4
	dotnetdotcom	4
	Scooter-ARS-1.1-ih	4
	WWW-Collector	4
	Google-HTTP-Java-Client	4
	Scooter-W3.1.2	4
	QuickFinderCrawler	4
	exabot.	4
	DSurf15aVA	4
	e-collector	4
	YahooSlurp!	4
	YahooSeeker/CafeKelsa-dev	4
	Argus/1.1	4
	msnbot\	4
	elfinbot	4
	Scooter/3.3_SF	4
	ScreamingFrogSEOSpider/3.3	4
	Xenu�s	4
	suke	4
	GoogleAdSense	4
	robofox	4
	Scooter/2.0G.R.A.B.V1.1.0	4
	Hackertarget.com	4
	DinoPing	4
	LjSEEK	4
	core	4
	Baiduspider+(	4
	Sogouheadspider	4
	Atomz	4
	//www.pingdom.com)	4
	EmailSmartz	4
	mattie	4
	Scooter-W3-1.0	4
	dotBot	4
	special_archiver	4
	WebIndexer	4
	waybackarchive.org/1.0	4
	Omgili	4
	spider72.yandex.ru	4
	y!j	4
	Site24x7	4
	FlaxCrawler	4
	wwwster	4
	XenusLinkSleuth1.1c	4
	Embedly	4
	revivebot	4
	Scooter-3.0.VNS	4
	dumbBot	4
	Scooter-3.3dev	4
	Walhelloappie	4
	Facebot/1.0	4
	whitevectorcrawler	4
	HiScan	4
	ECCP/1.0([email protected])	4
	wprocketbot	4
	Tibot	4
	Scooter-3.2.EX	4
	DBLBot/1.0	4
	//www.alexa.com/site/help/webmasters;[email protected])	4
	Scooter-3.0QI	4
	LBot	4
	WebFilter	4
	favorstarbot	4
	IstellaBot/1.23.15	4
	curl/	4
	Mail.rubot	4
	Scooter-ARS-1.1	4
	searchprocess	4
	ShopWiki/1.0	4
	Deep-Crawl	4
	[email protected]	4
	Whitevector+Crawler	4
	MSWebServicesClientProtocol	4
	webcrawl.net	4
	radian6Feedfetcher	4
	atSpider/1.0	4
	ConveraCrawler/0.2	4
	Mozilla/4.0(compatible;y!j;forrobotstudy;keyoshid)	4
	* #Theserulesapplytoallcrawlers	4
	SEOsearchCrawler/	4
	Firefly	4
	AlexaBitlybot	4
	Mozilla/4.0(compatible;Synapse)	4
	Zeus2.6	4
	TridentSpider	4
	InternetSupervision	4
	ContactBot/0.2	4
	kraken-crawler/*	4
	WebStripper/2.56	4
	WebReaper*	4
	BingBot/MSNBot	4
	OfflineExplorer/1.3	4
	ASpider/0.09	4
	Scooter-3.2.BT	4
	oneriot	4
	Scooter-3.0.EU	4
	esculapio	4
	Voilabot	4
	Alexa(IAArchiver)	4
	DemoBotZ16b	4
	MJ12bot/*	4
	GuzzleHttp	4
	microsoftbot	4
	askjeeves	4
	AVSearch-3.0(AltaVista/AVC)	4
	NetCaptor	4
	Jobo	4
	[email protected]	4
	pagefreezer	4
	ConveraInternetSpiderV6.x	4
	MetamojiCrawler	4
	Kraken/0.1	4
	Scooter-3.2.NIV	4
	LinkedIn	4
	ImageCollector	4
	tbot-nutch	4
	*bot	4
	pjspider	4
	packrat	4
	TechnoratiBot/8.1	4
	Aghaven/Nutch-1.2	4
	templeton	4
	baiduspider.$	4
	//www.google.com/feedfetcher.html;1subscribers;feed-id=14975829261283866692)	4
	emBot-GalaBuzz	4
	urlck	4
	W3SiteSearchCrawler	4
	Smetrics	4
	Yahoo-MMCrawler*	4
	NetcraftSurveyAgent/1.0	4
	pegasus	4
	polite-bot	4
	LinksManager.com_bot	4
	SeaMonkey	4
	[email protected]	4
	Scooter-3.2.SF0	4
	mediafox	4
	ZumBot*	4
	bl.uk_lddc_bot	4
	SoftlayerServer	4
	cydralspider	4
	scooter-venus-3.0.vns	4
	Spbot	4
	W3C_*Validator	4
	Xenus	4
	WebDownloader/4.9	4
	PChomebot	4
	spbot/4.0.9	4
	msnbot-NewsBlogs/	4
	Gigabot/	4
	DuckDuck	4
	msnbot-newsblogs	4
	BoardPulse	4
	MegaIndex.ru/*	4
	NetLyzer	4
	DSurf15a71	4
	Echo!	4
	WatchDog/3.0	4
	Anonymous	4
	Scooter-3.2.snippet	4
	myweb	4
	sato-crawler	4
	FullWebBot2816B	4
	DotBot*	4
	[email protected]	4
	BlueRobot	3
	Semalt	3
	confuzzledbot	3
	msnbot-207-46-13-97.search.msn.com	3
	Voltron	3
	Alkaline	3
	ca-crawler	3
	KIT-Fireball	3
	webquest	3
	Raven	3
	botify	3
	NameIntelligence	3
	Shopwiki	3
	visionutils	3
	SputnikImageBot	3
	borg-bot	3
	MJ12bot	3
	CrawlDaddy	3
	RedesScrapy	3
	Zitebot	3
	dataprovider.com	3
	OneRiot	3
	snooper	3
	WorldBrewBot/2.1	3
	InAGistURLResolver	3
	pagesinventory	3
	Valkyrielibwww-perl	3
	crescent	3
	LadenZeile	3
	mmu-gsa-crawler	3
	Webshag	3
	OfflineExplorer/1.1	3
	strucr	3
	Ahrefs-Bot/1.0	3
	teomaagent	3
	Calculon	3
	panopta.com	3
	*"lines!	3
	//www.bing.com/bingbot.htm)	3
	Biglotron	3
	Searchmetrics	3
	escan	3
	BackRub	3
	qwant	3
	quipu	3
	website	3
	arachnophilia	3
	ko_yappo_robot	3
	AlexaRobot	3
	//support.paper.li/entries/20023257-what-is-paper-li)	3
	scooter #Robotd'Astalavista	3
	SemrushBot/1.1	3
	webcollage/1.93	3
	MLBot	3
	PMAFind	3
	tarantula	3
	NaverRobot	3
	Netsparker	3
	Combine	3
	VSynCrawler	3
	BLP_bbot	3
	Atomic_Email_Hunter/4.0	3
	Libwhisker	3
	qihoo	3
	Aipbot	3
	Asterias	3
	Discoverybot	3
	Daum	3
	HTTrack\	3
	Monogosearch	3
	GlutenFreeCrawler	3
	EMailCollector	3
	ltbot	3
	arale	3
	GrapeFX	3
	Gootkitauto-rooterscanner	3
	Musobot	3
	jooblebot	3
	WebAltaCrawler/1.3.25	3
	Dragonfly	3
	Nibbler	3
	nzexplorer	3
	//www.seokicks.de/robot.html	3
	Lftp	3
	WIJobRoboterSpiderVersion3	3
	linkapediabot	3
	patric	3
	robi	3
	webwalk	3
	weblayers	3
	Sitesucker	3
	hambot	3
	Needle	3
	htmlgobble	3
	NuSearchSpider	3
	WebLinkValidator	3
	EMailSiphon	3
	Lanshanbot	3
	titin	3
	Niki-bot	3
	Wprecon	3
	Birubot	3
	directhit	3
	Seeker.lookseek.com	3
	Suzuran	3
	SiteBot*	3
	boxseabot	3
	verticrawl	3
	CareerBot	3
	CovarioCSE	3
	WebCorp	3
	msnbot-157-55-17-146.search.msn.com	3
	Sougouspider	3
	msnbot-65-52-104-29.search.msn.com	3
	Y!J-BRZ/YATSHAcrawler	3
	safesearch	3
	grabber	3
	*ISTHEFIRSTLINE------------------------------------------	3
	[email protected])	3
	PaperLiBot	3
	IntrafindBot	3
	Woobot	3
	zoominfobot	3
	QIHOO	3
	churl	3
	deweb	3
	yacybot	3
	cusco	3
	charlotte	3
	Ecxi	3
	Spammen	3
	skymob	3
	hbot	3
	Yandex/*	3
	irobot	3
	Covario-IDS	3
	Likse	3
	//www.puritysearch.net/)	3
	MetaURIAPI	3
	Yandex/1.03.003(compatible;Win16;D)	3
	Toata	3
	opsh	3
	openwebspider	3
	,.;/\-)	3
	Sitebeam	3
	CCBot/2.0	3
	trendictionbot0	3
	ActiveCacheRequest	3
	Re-Animator	3
	Y!J-BRY/YATSHcrawler	3
	PageScorer	3
	Ahrefs-Bot	3
	AppEngine	3
	Bolt	3
	TelegramBot	3
	Ebingbong	3
	yahoo!slurp	3
	pageboy	3
	Nettrack	3
	sindice-fetcher	3
	MorfeusFuckingScanner	3
	Yeti/0.01	3
	ChinaLocalBrowse2.6	3
	Unknownrobot(identifiedby'bot*')	3
	Funnelback	3
	NEC-MeshExplorer	3
	LinksCrawler	3
	bbot	3
	Sensis	3
	msnbot/	3
	Quantcastbot/1.0	3
	Litemage_walker	3
	GravityStream	3
	uMBot	3
	Scanbot	3
	Gromit	3
	lifestyle_contributor	3
	AhrefsBot*	3
	youdao	3
	msnbot-157-55-18-23.search.msn.com	3
	URLMetriken	3
	DomainTools	3
	GoogleBot-Mobile	3
	Humanlinks	3
	LinkCheckbySiteimprove.com	3
	EgotoBot	3
	Drecombot	3
	newsexpress	3
	BetaBot	3
	Flexumspider	3
	Flamingo_SearchEngine	3
	Collective	3
	Rankbot	3
	Twikle	3
	Fimap	3
	esther	3
	willybot	3
	sift	3
	FreeWebMonitoringSiteChecker	3
	duckduckgo	3
	msnbot-157-55-18-24.search.msn.com	3
	msnbot-65-52-110-34.search.msn.com	3
	muninn	3
	ShopAlikeBot	3
	ccubee	3
	xirq	3
	Nederland.zoek	3
	WISEnutbot	3
	Masscan	3
	PortalBSpider	3
	jumpstation	3
	CCbot	3
	crawl31	3
	Iblog	3
	Zgrab	3
	iajabot	3
	msnbot-65-52-104-115.search.msn.com	3
	twenga.com	3
	Webclipping.com	3
	Jbrofuzz	3
	udmsearch	3
	QueryCAT	3
	bot*.	3
	OfflineExplorer/1.8	3
	SpamBayes	3
	vebidoo	3
	WebCopier\	3
	Mozilla/5.0(compatible;Pogodak.co.yu/3.1	3
	Wget1.6	3
	//www.facebook.com/externalhit_uatext.php)	3
	libcurl	3
	ia_archiverrr	3
	meanpathbot/1.0	3
	crawl31+	3
	peerindex	3
	MyNutchSpider	3
	ParaSite	3
	GetProxi.es-bot	3
	SnapSkout	3
	gcreep	3
	YahooFeedSeeker/2.0	3
	Curious	3
	Slurp*	3
	DieBlindeKuh	3
	trendictionbot	3
	rogerbot/1.0	3
	WallpapersHD	3
	WEBDAVClient	3
	rsstank	3
	baiduspider/2.0	3
	Hloader	3
	CloudEndureScanner	3
	TurnitinBot/2.0	3
	JBot	3
	BravoBrian	3
	EmailSpider	3
	KavamRingCrawler	3
	PaperLiBot/2.1	3
	internetVistamonitor	3
	AboutWebsite	3
	EventGuruBot	3
	SpiderBot/1.0	3
	HyperCrawl	3
	ia_archiver(OS-Wayback)	3
	Freeuploader	3
	PulsepointXT3webscraper	3
	AhrefsBo	3
	nutchUL	3
	ccubee/3.5	3
	LinkTiger	3
	rdfbot	3
	webspider	3
	Wonderbot	3
	Dataprovider	3
	iZSearch	3
	Twengabot	3
	Stan	3
	Zermelo	3
	EverbeeCrawler	3
	gsa-crawler-MOS	3
	Lmspider	3
	WPScan	3
	g2Crawler	3
	ZeBot_www.ze.bz	3
	SBSearch	3
	XGET	3
	masscan	3
	Nessus	3
	PeoplePal	3
	Searchestate	3
	LqwRobot	3
	webkicks-Robot	3
	SpyFu	3
	IntershopWebAdapterAgent	3
	Probethenet	3
	golem	3
	desertrealm	3
	eZooms	3
	YahooCacheSystem	3
	araneo	3
	tach_bw	3
	YangaWorldSearchBotv1.1	3
	coccocbot-image	3
	Ltx71	3
	GigaBot	3
	OrkashBot	3
	NaverBot/1.0	3
	Archive.org	3
	AdnormCrawler	3
	EMailWolf	3
	emailwolf	3
	libwww\	3
	Nigma.ru/3.0	3
	Sitevigil	3
	Yandex/1.03.000(compatible;Win16;M)	3
	Boardreader	3
	Craftbot	3
	Flunky	3
	YahooSlurpChina	3
	Chlooe	3
	BoardReader-Image-Fetcher	3
	cyberspyder	3
	JetBot	3
	gulliver	3
	emailcollector	3
	Pingdom.com_bot_version_1.4	3
	ZeBot	3
	MJ12	3
	JS-KitURLResolver	3
	googlebot+	3
	OntoSpider	3
	Attracta	3
	Attach	3
	calif	3
	magpie-crawler/1.1	3
	InternetCruiserRobot	3
	TwinglyRecon	3
	ImageFetch	3
	radian6	3
	Pavuk	3
	muncher	3
	Whatweb	3
	YaBrowser	3
	TurtleScanner	3
	christcrawler	3
	Ezooms	3
	CMC/0.01	3
	spiderline	3
	*#directedtoallotherspiders	3
	VWbot	3
	DynamiteData	3
	pricebest	3
	joebot	3
	Lsearch/sondeur	3
	jooble-bot	3
	Getintent	3
	Firefly/1.0	3
	dwcp	3
	VBProject	3
	dienstspider	3
	Flexum	3
	RSSingBot	3
	Gotit	3
	* #matchallotherbots.	3
	multicrawler	3
	Gigabot/1.0	3
	RoboCrawlSpider	3
	emailsiphon	3
	YandexBot	3
	Lightspeedsystems	3
	AlexaSpider	3
	Yahoo-MMCrawler/	3
	IlTrovatore-Setaccio	3
	Baiduspider-image+	3
	Searchie	3
	harvest	3
	CheckMarkNetwork	3
	Panscient	3
	MicrosoftDataAccess	3
	YandexMetrika/2.0	3
	genieo	3
	SosoSpider	3
	Nigma.ru	3
	libwww-perl/5.800	3
	phpdig	3
	WebZip*	3
	TeleportPro/1.28	3
	ltx71+-+	3
	Bubing	3
	digger	3
	PleaseCrawl	3
	NaverBot	3
	Sqlmap	3
	piltdownman	3
	ExaBot	3
	ATN_Worldwide	3
	Wget*	3
	EMailExtractor	3
	bwh3_user_agent	3
	yunyun	3
	WIJobRoboter	3
	//www.omgili.com/Crawler.html	3
	DotBot	3
	Sogou-Test-Spider/4.0(compatible;MSIE5.5;Windows98)	3
	JavaBee	3
	sharp-info-agent	3
	webcatcher	3
	Mozilla/5.0(compatible;worldwebheritage.org/1.0;[email protected])	3
	Spidex	3
	linguee	3
	TeeRaidBot	3
	HostedComplianceSheriff	3
	csgbot	3
	Mozilla/5.0(WindowsNT6.1;WOW64)AppleWebKit/537.36(KHTMLlikeGecko)Chrome/41.0.2272.89Safari/537.36	3
	//www.xovibot.net/)	3
	Yahoo!Mindset	3
	RocketCrawler	3
	AiHitBot	3
	TwistedPageGetter	3
	WebMoose	3
	KBroker	3
	Microsoft.URL.Control\	3
	SiteBot	3
	python-urllib	3
	Mozilla/4.0(compatible;MSIE6.0;WindowsNT5.1)	3
	Yeti	3
	noxtrumbot\	3
	GlutenFreeCrawler/1.0	3
	EducateSearchVxB	3
	nekstbot	3
	EvilRobot	3
	46.229.173.67	3
	panelbot	3
	nicerspro	3
	AndroidDownloadManager	3
	Id-search	3
	NDSpider	3
	katipo	3
	Iskanie	3
	XoviOnpageCrawler	3
	offline	3
	WDG_SiteValidator	3
	YandexAntivirus	3
	DownloadDevil	3
	Mojeek	3
	Mozilla/5.0(compatible;AMZNKAssocBot/4.0)	3
	MiaDev	3
	vegibot	3
	DimensioNet	3
	200PleaseBot	3
	Spiderbot/Nutch	3
	Meanpathbot	3
	auresys	3
	disco/Nutch-1.0-dev	3
	uptimebot	3
	Deusu	3
	WebCollage	3
	Verticrawlbot	3
	WBSearchBot+	3
	YahooSeeker/	3
	aiHitBot	3
	Charlotte/Nutch-1.12	3
	LibWeb	3
	SpurlBot	3
	OracleEnterpriseSearch	3
	293net	3
	Googlebot-Image<	3
	cis455crawler	3
	webfetcher	3
	Msrabot	3
	pagescorer	3
	Dirbuster	3
	mantabot	3
	LWNutch/Nutch-1.4	3
	//domainreanimator.com)[email protected]	3
	Unknownrobot(identifiedby'robot')	3
	fouineur	3
	ScountJet	3
	webcollage/1.96	3
	GoZilla	3
	backrub	3
	Baidubot	3
	searchch	3
	GigablastOpenSource/1.0	3
	extractorpro	3
	hit	3
	webcollage_save/1.93	3
	MSIECrawler\	3
	puf	3
	Seomoz	3
	NetScoop	3
	vsecrawler	3
	Theophrastus	3
	commons-httpclient	3
	rogerbot/1.1	3
	AppCodes	3
	Crowsnest/0.5	3
	Baiduspider-sfkr	3
	Ipselonbot	3
	[email protected][/EMAIL]	3
	Turnitinbot	2
	MicrosoftURLControlâ€“5.01.4511	2
	wanderer	2
	GomezAgent1.0	2
	BlitzBOT	2
	//js-kit.com/	2
	motor	2
	//otc.dyndns.org/webscan/)	2
	Linkdexbotv2.1	2
	CFNetwork/*	2
	TutorialCrawler*	2
	gama	2
	Mozilla/4.0(compatible;Win32)	2
	CazoodleBot/*	2
	InternetExplorer*	2
	Nudelsalat/*	2
	FavOrg	2
	singingfish	2
	[email protected]	2
	Google-Feedfetcher	2
	CapelFantom	2
	abot/*	2
	findfiles.net/*(Robot;[email protected])	2
	spider([email protected])	2
	Friendica	2
	WebWalker	2
	websnarf	2
	NextGenSearchBot(forinformationvisit)	2
	mwdsearch	2
	RankSonicBot	2
	NationalDirectory	2
	searchbe	2
	'\*bot'	2
	Gozilla/*	2
	SmartyBot	2
	DomainCrawler/3.0	2
	portalb	2
	Mozilla/5.0(compatible;OsO;*	2
	Slurp#Yahoo!	2
	HatenaBookmark/*	2
	HTTPFetch/*	2
	OCN-SOC/*	2
	SurveyBot/*	2
	grapnel	2
	Update	2
	//cognitiveseo.com/bot.html	2
	RSurf15a81	2
	webcheck1.10.4	2
	RoboCrawl	2
	kw-lp-suggest	2
	//www.cloudflare.com/always-online)AppleWebKit/534.34	2
	shaboyispider	2
	''	2
	CherryPicker/	2
	Sogouheadspider*	2
	gloomarbot	2
	WebCopierv4.0	2
	Python	2
	IWAgent/1.0-www.brandprotect.com	2
	GigaBlast	2
	'crawl'	2
	hometown	2
	Pcore-http	2
	Mozilla/5.0(compatible;Exabot-Images/3.0*)	2
	WebSnatcher*	2
	Ginxbot	2
	LMQueueBot/*	2
	LeGuideImgServer	2
	Occam	2
	ClarityBot	2
	SemrushBot/0.95	2
	site-valet	2
	VoilaBotCollector	2
	Baidu-YunGuanCe-Bot	2
	httpunit/*	2
	BuiBui-Bot	2
	macworm	2
	MonoBrowserCapabilitiesUpdater*	2
	Mozilla/4.0(compatible;TrendMicrotmdr1.*	2
	msnbot-media/1.0	2
	//fairshare.cc	2
	pimptrain	2
	A6-Indexer/1.0	2
	YodaoBot-Image	2
	3D-FTP/*	2
	pavuk/*	2
	Sqworm/*	2
	ruby	2
	linkidator	2
	T-H-U-N-D-E-R-S-T-O-N-E	2
	wadaino.jp-crawler*	2
	SogouPicSpider/*	2
	BeijingCrawler	2
	ickHTTP	2
	//code.google.com/p/crawler4j/)	2
	Java/1.6.0_14	2
	yahoo!	2
	MetaURIAPI/2.0+metauri.com	2
	imagelock	2
	Fooky.com/ScorpionBot/ScoutOut;*	2
	PageDown*	2
	havIndex	2
	gsa-crawler*	2
	madaali.de	2
	Scrapy/0.24.6	2
	'robot'	2
	DomainsBotBot/1.*	2
	DG_JUSTICE_CRAWLER	2
	BDCbot/2.0	2
	SemrushBot/0.98~bl	2
	eCairn-Grabber	2
	curiousgeorge-www.analyticsseo.com	2
	CydralSpider/*	2
	EmailSiphon*	2
	arianna	2
	NetShelter	2
	musobot/1.0	2
	Domnutch-Bot	2
	FetchAPI	2
	funnelweb	2
	netcarta	2
	Googlebot/	2
	WOW64	2
	objectssearch	2
	PSurf15a51	2
	BOTWSpider	2
	zoomRank/2.0	2
	intelliagent	2
	Mozilla/5.0(compatible;Webbot/*)	2
	Nutch/Nutch-0.9	2
	GarlikCrawler/1.1	2
	AshrefsBot	2
	bigbrother	2
	Biozilla	2
	EpsilonSoftWorks'MailMunky	2
	WebPix*	2
	icarus6	2
	Mozilla/4.0(compatible;MSIE5.0;WindowsNT)	2
	mediatoolkitbot	2
	URLspion	2
	Busiversebot	2
	heritrix/1.14.4	2
	NetSeer/Nutch-0.9	2
	iron33	2
	wallpaper	2
	FriendlyCrawler	2
	seplinkbot/1.0	2
	TalkroWeb-Shot/*	2
	WebRipper	2
	BitBeamer/*	2
	//*/crawler.html)	2
	Webzip*	2
	AmicoAlpha()Gecko/AmicoAlpha/	2
	DuckDuckGo-Favicons-Bot	2
	bayspider	2
	ProductionBot2016B	2
	Mozilla/4.0(compatible;MSIE7.0;WindowsNT5.2)	2
	AcadiaUniversityWebCensusClient	2
	Scrapy/1.0.1	2
	//*	2
	WordPress-Do-P-/2.*	2
	finnish	2
	hcat/*	2
	Mozilla/5.0gURLChecker/*	2
	Linkdexbotv2.0	2
	Mozilla/5.0(compatible;del.icio.us-thumbnails/;)KHTML/*(likeGecko)	2
	LetsCrawl.com/1.0*	2
	//www.findxbot.com)	2
	askivesbot	2
	JS-Kit	2
	Scooter2_Mercator_3-1.0	2
	IE/6.01(CP/M;8-bit*)	2
	'Unknown'	2
	bjaaland	2
	Atomic_Email_Hunter	2
	SEOkicks-Robo	2
	Mozilla/5.0(compatible;Seznamscreenshot-generator2.0;*)	2
	Panopta+v1.1	2
	roach.smo.av.com-1.0	2
	topiclink	2
	Exabot-Test/*	2
	httpgeneric	2
	ilse	2
	jubii	2
	*Extractor	2
	Larbin*	2
	Ocelli/1.4	2
	senrigan	2
	WebZip/5.0	2
	Mozilla/5.0(SnapPreviewBot)Gecko/Firefox/	2
	Scopia	2
	microsoft	2
	AraybOt	2
	YangaWorldSearch	2
	Konqueror/3.5	2
	SeznamBot/*	2
	//ponderer.org/*	2
	lftp/3.2.1	2
	SurfControl	2
	YandexWebmaster	2
	*email	2
	Search-Engine-Studio	2
	nhse	2
	evliyacelebi	2
	shareaza*	2
	Twitturly*	2
	Lexxebot	2
	DragonBot	2
	//www.forumseek.net/BOT_2.1	2
	MsnBot-Media	2
	//www.aihitdata.com/about)	2
	//www.changedetection.com/bot.html)	2
	//help.yahoo.com/help/us/ysearch/slurp	2
	MicrosoftOffice/(PictureManager*)	2
	gsa-nfsa	2
	BegunAdvertising	2
	parasite	2
	360Spider-Image	2
	SBL-BOT*	2
	KBeeBot/0.*	2
	Bot.AraTurka.com	2
	//filterdb.iss.net/crawler/	2
	Ilse	2
	'Java/1.6.0_04'	2
	Mozilla/5.0(compatible;NLCrawler/2.0.25/r6;Linux2.6.3-7;i686;en_US)KHTML/3.4.89(likeGecko)	2
	winona	2
	IrssiUrlLog/*	2
	'legs'	2
	YRL_ODP_CRAWLER	2
	Y!J-BRM/YATSD	2
	Rexyobot	2
	gromit	2
	favorstarbot/*	2
	FollowSite.com(*)	2
	Links4US-Crawler,*	2
	PhotoStickies/	2
	//www.scrubtheweb.com/abs/meta-check.html)	2
	NG-Search/*	2
	VengaBot/*	2
	* #Theserulesapplytoallcrawlers	2
	*spider	2
	Grammarly	2
	Sogou-Test-Spider	2
	Y!J-MBS	2
	Grub.org	2
	TEOMA#Ask.com	2
	CPython/2.7.3	2
	ferret	2
	ContactBot/0.1	2
	HatenaScreenshot*	2
	MFHttpScan	2
	ShowXML/1.0libwww/5.4.0	2
	infospider	2
	daumos	2
	fetchlibfetch/*	2
	OSSProxy*	2
	PartyBob	2
	araybot	2
	Crawler/*	2
	WebGatherer*	2
	//www.voila.com/)	2
	Teoma/AskJeeves	2
	SandCrawler	2
	msnbot-image	2
	sitetech	2
	curl*	2
	Googlebot#Google	2
	ProxyTester*	2
	//WebDataCentre.com/)	2
	WebInformacion	2
	AutoMate5	2
	JamesBOT-WebCrawler	2
	image.kapsi.net	2
	borg-bot/0.9	2
	tkwww	2
	Mozilla/4.0(compatible;Scumbot/;Linux/)	2
	Anonymisiert*	2
	DownloadSession*	2
	Photon	2
	HubSpotCrawler1.0	2
	MSProxy/*	2
	linkfluence	2
	gsa-crawler-A	2
	Libwww*	2
	Baidu*	2
	worm	2
	WISEbot/*	2
	NetSucker*	2
	//www.yodao.com/help/webmaster/spider/;)	2
	Phantom	2
	vwbot	2
	sEasyDL/*	2
	NV32ts	2
	[email protected])	2
	Yahoo-Test	2
	DownloadMaster*	2
	sna-0.0.*	2
	ICAP-IOD	2
	//domainsigma.com/robot)	2
	Cys	2
	YahooBot	2
	RealDownload/*	2
	Halebot	2
	'spider'	2
	spry	2
	logo_gif	2
	Tclhttpclientpackage*	2
	Webshuttle	2
	LinkValetOnline*	2
	ElectricMonk	2
	iGetter/*	2
	[email protected])	2
	hyperdecontextualizer	2
	diibot	2
	grub	2
	hitcrawler_0.*	2
	IDGCrawler	2
	Xenu\'sLinkSleuth1.1c	2
	nerdbynature	2
	semrush	2
	nagios	2
	E-MailAddressExtractor	2
	Vebidoobot	2
	Crawler	2
	HttpSession	2
	bingbot.$	2
	webwatch	2
	//www.kalooga.com/info.html?page=crawler)	2
	BOTforJCE	2
	Steeler/3.5	2
	WAP_Browser/5.0(compatible;YodaoBot/1.*)	2
	MultiCrawler	2
	SlySearch/*	2
	WebEnhancer*	2
	YourWebsite.net	2
	cmc	2
	Toatadragostea*	2
	Nutscrape	2
	//www.linktiger.com*)	2
	ZIBBCrawler(emailaddress/WWWaddress)	2
	cb	2
	Y!J-BRN/YATSA	2
	adidxbot/1.1	2
	feedthebot	2
	yandex.ru	2
	RecordedFuture	2
	metatagsdir/*	2
	PBrowse1.4b	2
	CRAZYWEBCRAWLER+0.9.8	2
	WebStripper/2.14	2
	Beamer*	2
	bot/(bot;[email protected])	2
	ezic.comhttpagent*	2
	HyperEstraier/*	2
	USyd-NLP-Spider*	2
	Mozilla/4.0(compatible;MSIE5.5;WindowsNT;PCLNNOCSiteScope)	2
	updown_tester	2
	Atomic_Email_Hunter*	2
	BasicHTTP/*	2
	//www.tasap.com)	2
	Mozilla/4.0(compatible;MSIE8.0;WindowsNT6.0;ClarityDailyBot)	2
	infoseek	2
	FastPartnerSiteCrawler	2
	'robots\.txt'	2
	DeepIndexer*	2
	//www.strategicboard.com)	2
	//www.diffbot.com)	2
	discoverybot*	2
	Mozilla/5.0(Java)outbrain	2
	WebsterPro*	2
	wishpond	2
	Mozilla/4.05[en]	2
	turtle	2
	Mozilla/5.0(compatible;Charlotte/;)	2
	//www.cmscrawler.com)	2
	HatenaStar	2
	Xenu\'s	2
	GulperWeb*	2
	NexToolsWebAgent*	2
	Msnbot-media	2
	HTTrack3	2
	/redirect.php	2
	OmtrBot/1.0	2
	Yahoo!+Slurp+China	2
	my-heritrix-crawler	2
	GetIntentCrawler	2
	shopstylebot/1.0	2
	boitho.com-robot	2
	webinator	2
	/advertiser/	2
	WebStripper*	2
	omgilibot/0.3	2
	AOLbot/4.0	2
	ilial	2
	lwp-trivial/5.810	2
	GoogleProducer	2
	Mozilla/5.0(compatible;IPCheckServerMonitor*)	2
	YodaoBot/1.()	2
	FastSearch	2
	Domain+Re-Animator+Bot	2
	Haosou	2
	FANGCrawl/*	2
	RixBot	2
	Netluchs/Nutch-0.9-dev	2
	labelgrabber.txt	2
	RedCarpet/*	2
	//www.dotnetdotcom.org/*)	2
	Pete-Spider/1.*	2
	//herbert.groot.jebbink.nl/?app=rssImages)	2
	StarDownloader/	2
	WordPress-B-/2.*	2
	Charlotte/1.1	2
	emacs	2
	MadebyZmEu@WhiteHatv0.*(www.WhiteHat.ro)	2
	WebMiner*	2
	Niki-Bot	2
	AllTheWeb	2
	InfoSeekSidewinder	2
	InfoSeekRobot1.0	2
	Mail.RU_Bot/Img	2
	perignator	2
	ksibot	2
	fdse	2
	A.NETWebCrawler	2
	FreshDownload/*	2
	//twitturls.com)	2
	Trident/4.0	2
	RSurf15a51	2
	/_sites	2
	WinHttp*	2
	RoboCrawler	2
	Visbot	2
	ThemeSpider*	2
	//www.exabot.com/go/robot)	2
	//www.softlist.us/)	2
	Pageload*	2
	WebsiteDownloader*	2
	SurfKnight	2
	WinHttpRequest	2
	Java/1.6.0_12	2
	WebThumbnail	2
	WeSEE_Bot	2
	Sogouinstspider/4.0	2
	hl_ftien_spider	2
	WebCorp/*	2
	SiteArcCrawler/0.1	2
	GameSpyHTTP/*	2
	alexa	2
	TechBOT	2
	6600/RS/PRIVACY_ENFAQ.jsp	2
	OfflineExplorer*	2
	WebImageCollector*	2
	CFNetwork	2
	BooboBot	2
	Internetseer.com	2
	PJbot	2
	webmoose	2
	Check&Get	2
	Find/*	2
	/*.pdf$	2
	ESI	2
	ProgramShareware1.0.2	2
	Python*	2
	URL2File/*	2
	//botw.org)	2
	Gocrawl	2
	RBot	2
	Crawlera	2
	[email protected]	2
	Zeus	2
	Prozilla*	2
	ProductionBot0116B	2
	blindekuh	2
	orb_search	2
	HyperEstraier	2
	USER_AGENT	2
	dnabot	2
	MSIECrawler	2
	//garlik.com/,[email protected])	2
	NetZipDownloader*	2
	UofTDB_experiment*([email protected])	2
	YACYBIT	2
	Anonymizied*	2
	CAST	2
	Attribot	2
	Mozilla/5.0(compatible;AboutUsBot/*)	2
	updated/0.1beta	2
	squid	2
	FranklinLocator*	2
	A1WebsiteDownload/1.()miggibot	2
	search_au	2
	webreader	2
	Roverbot*	2
	googlebot-Image	2
	DA*	2
	GomezAgent3.0	2
	Artera(Version*)	2
	ScoutAbout*	2
	phantom	2
	iexplore.exe	2
	sohu*	2
	webfoot	2
	nutch/1.2	2
	Sogou-Test-Spider/*	2
	GomezAgent2.0	2
	dotbot/1.1	2
	SimplePie/1.1.1	2
	voidbot	2
	Snapbot/*	2
	ExaleadNG/*	2
	JPluck/*	2
	NetID.comBot*	2
	Mozilla/4.0(compatible;MSIE7.0;WindowsNT5.1;Trident/4.0;.NETCLR1.1.4322;.NETCLR2.0.50727;.NETCLR3.0.4506.2152;.NETCLR3.5.30729)	2
	tiscali	2
	occam	2
	ssearcher	2
	YandexAdNet	2
	UtilMindHTTPGet	2
	Nozilla/P.N(JustforIDSworing)	2
	amazonaws.com	2
	//www.integromedb.org/Crawler	2
	jack	2
	postano	2
	FASTDataSearchDocumentRetriever	2
	SpaceBison/	2
	Fetch/*	2
	maxamine.com--robot	2
	*download	2
	BaySpider	2
	RSurf15a41	2
	Unknownrobot(identifiedby'*bot')	2
	PATROL/V3.6.50i(Linux;INETKM6.2.10200410050940)	2
	halebot	2
	//www.yama.info.waseda.ac.jp/~yamana/es/)	2
	MicrosoftDataAccessInternetPublishingProviderDAV*	2
	Spider	2
	Scrapy/0.24.4	2
	ArchitectSpider	2
	CFSCHEDULE*	2
	WestWindInternetProtocols*	2
	MQbot	2
	Googlebot-Mobile/2.1	2
	FreeNutch/Nutch-1.2	2
	//www.webalta.net/ru/about_webmaster.html)(Windows;U;WindowsNT5.1;ru-RU)	2
	Feedfetcher	2
	DownloadExpress*	2
	PSurf15aVA	2
	iconoclast	2
	iCopyrightConductor*	2
	Moatbot	2
	FeedInformer	2
	robocrawl	2
	ICE_GetFile	2
	//www.envolk.com/envolk*)	2
	MisterPIX*	2
	SmallProxy*	2
	superfish	2
	Solbot	2
	NAVER	2
	yodaoice	2
	nomad	2
	raven	2
	havindex	2
	//MapoftheInternet.com)	2
	Crawler4j	2
	Fast	2
	HTTrack*	2
	AmboBot	2
	NetcraftWebserverSurvey	2
	//www.busiverse.com/bot.php)	2
	SiteSucker/*	2
	//Anonymouse.org/*	2
	Mozilla/4.0(compatible;Win32;WinHttp.WinHttpRequest.5)	2
	blackwidow	2
	spam+bot	2
	exactseek-pagereaper-*([email protected])	2
	DomainMacroCrawler	2
	wlm	2
	HiddenMarket-*	2
	KolinkaForumSearch(www.kolinka.com)	2
	Nutch	2
	LexxeBot/1.0+([email protected])	2
	FASTCrawler	2
	BlueCoatProxySG	2
	Go!Zilla*	2
	12345	2
	IconSurffavicon.icoInfoNaviRobot	2
	PAD-bot	2
	Scrapy/0.16.5	2
	PagePeeker2	2
	'crawler'	2
	AdMuncher*	2
	/.doc$	2
	//www.tkl.iis.u-tokyo.ac.jp/~crawler/)	2
	BOT/0.1(BOTforJCE)	2
	abcdatos	2
	webfetch/*	2
	Zao/*	2
	TWMBot	2
	plumtreewebaccessor	2
	htmlparser	2
	monster	2
	//www.kapere.com)	2
	BuzzRankingBot	2
	copyrightsheriff(*)	2
	crawlbe	2
	Scrapy/0.24.5	2
	Arquivo-web-crawler	2
	ContactBot/*	2
	wired-digital	2
	javabee	2
	OFMsearch_robot.google.(compatible;)	2
	NL-Crawler	2
	Anonymous/*	2
	WebAuto/*	2
	*larbin	2
	compute-1.amazonaws.com	2
	GumGumBot	2
	Forschungsportal/*	2
	KontikiClient*	2
	Crawl_Application	2
	libWeb/clsHTTP*	2
	//www.aol-soft.com/)	2
	rbse	2
	brightnet	2
	Gimme60bot/1.0	2
	vegi-bot	2
	NextGenSearchBot1	2
	kapsi	2
	PSBot	2
	TerrawizBot/*	2
	*<BR>	2
	ArenaFutura	2
	NLCrawler	2
	//help.baidu.jp/system/05.html)	2
	AideRSS/2.0(aiderss.com)	2
	acapbot	2
	Java/1.7.0_04	2
	Adixbot	2
	WebCapture*	2
	IamSPAMER!	2
	Mozilla/4.0(compatible;CerberianDrtrs*)	2
	Taigawebspider	2
	WebZIP*	2
	IPiumBotlaurion(dot)com	2
	Anonymous*	2
	OFMsearch_robot	2
	urlfan-bot	2
	GooglePlusShare	2
	SimplePie/1.0.1	2
	northstar	2
	//flipboard.com/browserproxy)	2
	research	2
	PlumtreeWebAccessor	2
	LSSession	2
	/admin/	2
	PortHuronLabs	2
	NetCarta_WebMapper/*	2
	NetAttache*	2
	SiteWinder*	2
	Privoxy/*	2
	PulseCrawler/1.1	2
	sven	2
	XenuLinkSleuth	2
	java	2
	SynapticSearch/AICrawler1.?	2
	//www.easou.com/search/spider.html)	2
	Little-forest-lfi	2
	ColdFusion*	2
	OutfoxBot/*	2
	WinTools	2
	DownloadNinja*	2
	cubestat	2
	SynooBot	2
	//www.gigablast.com/spider.html)	2
	israelisearch	2
	Mozilla/4.0(compatible;MSIE6.0;BluecoatDRTR)	2
	eStyleSearch*(compatible;MSIE6.0;WindowsNT5.0)	2
	felix	2
	wolp	2
	resumerobot	2
	Solomono	2
	FlamingAttackBot*	2
	HatenaAntenna/*	2
	Linkdexbotv2.2	2
	CMC	2
	RED	2
	ExpertSearchSpider	2
	Mozilla/4.0(compatible;BorderManager*)	2
	Mozilla/4.0(compatible;MSIE4.01;Vonna.combot)	2
	openstat.ru/Bot	2
	Odnosniki.pl	2
	freesitemapgenerator	2
	nbot/2.0	2
	//arachnode.net*	2
	CobWeb/*	2
	ExtractorPro*	2
	PHP*	2
	Vegas95/*	2
	kelley-sitemap-crawler	2
	libwww-perl/5.803	2
	EMAILsearcher	2
	MicrosoftOfficeExistenceDiscovery	2
	mnogosearch	2
	cowbot	2
	Dergro\xdfeBilderSauger*	2
	Foobot*	2
	HTTrack	2
	Mozilla/4.0(compatible;AdvancedEmailExtractor*)	2
	SquigglebotBot/*	2
	woriobot*	2
	BPImageWalker/2.0(www.bdbrandprotect.com)	2
	Yeti-FeedItemCrawler	2
	WebSpider	2
	NetzCheckBot/1.0	2
	wsowner	2
	PSurf15a11	2
	NASASearch1.0	2
	uberspider	2
	Jeeves/Teoma	2
	checkbot	2
	download_express	2
	Atomic_Email_Hunter/*	2
	gnome-vfs/*	2
	lnbot	2
	DataFountains/DMOZDownloader*	2
	OpenWebAnalyticsBot*	2
	Pluggd	2
	archive*	2
	Y!J-BRM/YATSDcrawler	2
	SoftListBot	2
	Pilican	2
	ucsd	2
	BillyBobBot	2
	victoria	2
	CopyRightCheck*	2
	Twitturly	2
	//www.dead-links.com/)	2
	python-requests/1.2.0	2
	DISCoPump*	2
	shaihulud	2
	UnknownBot	2
	suggynutch/1.2	2
	envolk/1.7	2
	R6_FeedFetcher_	2
	tcl	2
	LinkCheck	2
	//corp.infocious.com/tech_crawler.php)	2
	MicrosoftOfficeProtocolDiscovery	2
	CommonCrawler	2
	SiteValet	2
	TunnelPro	2
	sherlock/*	2
	titan	2
	ICC-Crawler/2.0	2
	Custo*	2
	Feedfetcher-Google-iGoogleGadgets*	2
	Mozilla/5.0(compatible;nextthing.org/*)	2
	//www.meanpath.com/meanpathbot.html)	2
	Speedy_Spider	2
	//www.html2jpg.com	2
	NotYourBusiness!	2
	[email protected]	2
	data2lifebot	2
	IPWorks!/*	2
	YahooSeeker-Testing	2
	SilentSurf*	2
	DoyoucheckBot	2
	DeepCrawl	2
	Lynx	2
	aretha	2
	cybermapper	2
	MSNBot#MSN	2
	//www.jadynave.com/robot*	2
	JakartaCommons-HttpClient/3.0.1	2
	Webcrawler	2
	uptimefiles-websitemonitoringservice	2
	Labhoo	2
	Net_Vampire*	2
	-	2
	Mediapartners(Googlebot)	2
	Nsauditor/1.x	2
	BLP_bbot/1.0	2
	*harvest	2
	Poirot	2
	gulperbot	2
	webbandit/*	2
	Fishsearch	2
	WebAltaCrawler/1.3.26	2
	FOTOCHECKER	2
	Mozilla/5.0()VoilaBot	2
	WebWhacker*	2
	WinScripteriNetTools	2
	DealGatesBot	2
	Experibot	2
	PicaLoader*	2
	SuperHTTP/*	2
	NSO_Debugger_User/2.0	2
	gsa-crawler-B	2
	IAArchiver	2
	DeepIndex	2
	Webwhacker*	2
	XerkaWebBotv1.*	2
	TEOMA	2
	Motor	2
	ProoXiBot	2
	hrefsBot	2
	YodaoBot/*	2
	iwon	2
	openstat	2
	Tv<nn>_Merc_resh_26_1_D-1.0	2
	Harvest/*	2
	ibm	2
	HooWWWer/*	2
	//www.moreover.com;[email protected])	2
	Y!J-BRN/YATSAcrawler	2
	//www.pingdom.com/)	2
	IconSurf/2.*	2
	MicrosoftDataAccessInternetPublishingProviderProtocolDiscovery	2
	Titanium2005(4.02.01)	2
	rogerbot???	2
	//www.nameprotect.com/botinfo.html)	2
	smartspider	2
	msnbot/2.1	2
	lycra	2
	WebGet	2
	WebGrabber	2
	ActiveRefresh*	2
	SiteValetOnline*	2
	newsme/1.0;[email protected]	2
	w3m2	2
	wombat	2
	//www.genieo.com/webfilter.html	2
	simbot	2
	BackStreetBrowser*	2
	heritrix	2
	LucidMediaClickSense/4.?	2
	MicrosoftBITS/*	2
	Mozilla/5.0(compatible;YodaoBot-Image/1.*)	2
	*crawler	2
	ssearch_bot	2
	Mozilla/4.0(compatible;MSIE6.0;WindowsNT5.1;en)Opera8.50	2
	SurveyBot	2
	weblinker	2
	fish	2
	Googlebot-Images	2
	SmartDownload/*	2
	Nutscrape/(CP/M;8-bit)	2
	LSSRocketCrawler/1.0	2
	WoFindeIchRobot	2
	LTI/LemurProject	2
	kilroy	2
	naoFavicon4IE*	2
	RadiationRetriever*	2
	Feedfetcher-Google*	2
	8	2
	NewRelicPinger	2
	//riddler.io/about.html)	2
	meshexplorer	2
	BaiduSpiderBot	2
	ExtremePictureFinder	2
	mchp-crawler	2
	//www.yoow.eu)	2
	HenryTheMiragoBot	2
	sna-0.0.1	2
	//tweetedtimes.com)	2
	gonzo1[P]	2
	WWW-Mechanize/*	2
	BaiduSpider+	2
	roverbot	2
	BitTorrent/*	2
	BlockNote.Net	2
	STEROIDDownload	2
	VCIWebViewer*	2
	moba-crawler	2
	BountiiBot	2
	mobilegoo	2
	Cloudinary	2
	yie8	2
	SogouOrionspider/*	2
	litefinder	2
	WebTarantula.comCrawler	2
	CTerm/*	2
	gammaSpider	2
	Jambot/0.1.1	2
	Lorkyll.[email protected]	2
	Y!OASIS*	2
	TheWorldWideWebWorm	2
	Majestic(UK)	2
	slcrawler	2
	visionsearch	2
	Inet-EurekaApp	2
	Myzilla	2
	//tweetmeme.com/)	2
	Search-AU	2
	Hidden-Referrer	2
	Bimbot	2
	IU-MS-Crawler	2
	Chilkat/*	2
	EmailCollector*	2
	Sogoudevelopspider/*	2
	Lizard	2
	Lycos_Spider_(modspider)	2
	KDD-Explorer	2
	Pioneer	2
	uber	2
	LeechGet*	2
	Mozilla/*(TuringOS;TuringMachine;0.0)	2
	//herbert.groot.jebbink.nl/?app=WebImages?)	2
	Mozilla/4.0(compatible;Getleft*)	2
	ProductionBotDOT3016B	2
	LinkLint-checkonly/2.x.x	2
	*#allbots	2
	AESOP_com_SpiderMan	2
	AskJeeves/Teoma	2
	geturl	2
	rhcs	2
	wz101	2
	3wGet/*	2
	Mozilla/5.0(compatible;archive.org_bot*)	2
	ClickagyIntelligenceBot	2
	YahooMobile	2
	Snoopy*	2
	Microsoft-WebDAV-MiniRedir/*	2
	//www.apolloinsights.com/)	2
	muscatferret	2
	BilgiBot/*	2
	//hilfe.acont.de/bot.htmlACONTBOT	2
	strucr-tablet	2
	Informant	2
	,\.\;\/\\-]	2
	Mozilla/5.0(compatible;YodaoBot/1.*)	2
	Mozilla/5.0(Macintosh;IntelMacOSX)Excel/12.*	2
	dragonbot	2
	OpenSearchServer_Bot	2
	SearchMetrics	2
	KRetrieve/	2
	NetchartAdvCrawler*	2
	b2w/*	2
	LinkStatsBot	2
	ECCP/1.2.1	2
	WebStripper/2.62	2
	check_http	2
	TwengaQualityBot	2
	LinkpadBot/1.07	2
	RPT-HTTPClient/*	2
	BOT_2.1	2
	vlasna-traveller	2
	whatuseek	2
	Mozilla/(compatible;OffByOne;Windows)WebsterProV3.*	2
	OfflineDownloader*	2
	007AC9	2
	kw-lp-suggestCount	2
	idmarch	2
	IRLbot/*	2
	no_user_agent	2
	Atomz/1.0	2
	tlspider	2
	AutoHotkey	2
	//www.inetbot.com/bot.html)	2
	Holmes/*	2
	sogoujsrobot(*)	2
	xget	2
	123peoplebot/1.0	2
	LightningDownload/*	2
	GoogleFeedfetcher	2
	PigBlock(WindowsNT5.1;U)*	2
	linkchecker	2
	Xenuâ€™sLinkSleuth1.1c	2
	eit	2
	magpie	2
	Chatalogica.comCGIScript	2
	libcurl-agent/*	2
	DesktopSidebar*	2
	CJNetworkQuality	2
	//search.thunderstone.com/texis/websearch/about.html)	2
	WIRE/(LinuxBot,Robot,Spider,Crawler)	2
	8484BostonProject*	2
	www4mail/	2
	LinkextractorPro*	2
	Mozilla/5.0(compatible;Exabot/3.0*)	2
	WebAltaCrawler/*	2
	X-CAD-SE	2
	HLoader	2
	IWAgent/*	2
	AppEngine-Google	2
	Ezooms/1.0;[email protected]	2
	YangaWorldSearchBotv1.1/beta	2
	MFCFoundationClassLibrary*	2
	FDM1.x	2
	PEval1.4b	2
	Insitesbot	2
	AdsBot-Google	2
	WebMagnet*	2
	diffbot	2
	WebDataCentreBot/1.0	2
	w3search/Nutch-0.9	2
	empas	2
	linkscan	2
	BNaBot	2
	atomz	2
	netmechanic	2
	ScollSpider/2.*	2
	XSpider*	2
	pixfinder/*	2
	opensiteexplorer.org	2
	BLEXBotCrawler	2
	yahoo-blogs/v3_9	2
	CE-Preload	2
	GurujiBot/1.*	2
	INetURL/*	2
	mp3Spidercn-search-develatyahoo-incdotcom	2
	TheInformant*	2
	hbtronix.spider	2
	//www.conductor.com/caliperbot)	2
	Anonymizer/*	2
	//www.forumseek.net	2
	wmir	2
	atn	2
	MetaProductsDownloadExpress/*	2
	Botster.LinkChecker	2
	//www.baidu.jp/spider/)	2
	rules	2
	EmailWolf*	2
	MSNbot/Bingbot	2
	Mozilla/5.0(compatible;acapbot/0.1;treatlikeGooglebot)	2
	safetynetrobot	2
	Icarus6j	2
	TRankBot	2
	uMBot-IC	2
	techbot	2
	//liferea.sf.net/)	2
	NetVampire/*	2
	YetiBot	2
	B-l-i-t-z-B-O-T	2
	Presto/2.8.119	2
	/adresy/	2
	123peoplebot	2
	architext	2
	hi	2
	JUST-CRAWLER(*)	2
	QCrawl	2
	dialogsearch.com	2
	Mnogosearch	2
	MQbot*	2
	)(Linux*)	2
	Mozilla/4.0(compatible;MSIE6.0;WindowsNT5.1;.NETCLR1.0.3	2
	ABACHOBot	2
	//www.yunyun.com/spider.html)	2
	008*	2
	ScreenerBotCrawlerBeta2.0	2
	//gnomit.com/)Gecko/*Gnomit/1.0	2
	Huasai	2
	Vedma	2
	XoviBo	2
	Atomz.comSearchRobot	2
	/</pre>	2
	1.9b5)Gecko/2008032620Firefox/3.0b5	2
	adidxbot/2.0	2
	'Java/1.7.0_21'	2
	InternetNinja*	2
	SpeedDownload/*	2
	DBot	2
	SLURP	2
	123people	2
	WebAltaCrawler/1.3.34	2
	Y!J-BRL/YATSScrawler	2
	octopus	2
	valkyrie	2
	naver	2
	EasyDL/*	2
	linkdexbot-mobile	2
	cgireader	2
	cactvschemistryspider	2
	pioneer	2
	Google-AdSense	2
	FGet*	2
	SogouPushSpider/*	2
	Mozilla/5.0(compatible;ViralheatBot/*)	2
	PageNest/*	2
	TutorGigBot/*	2
	//www.descargarprogramagratis.com/)	2
	DA7.0	2
	Baiduimagespider	2
	incywincy	2
	Aranha	2
	httpclient*	2
	XaldonWebSpider*	2
	*#AllSpiders	2
	sdcresearchlabs-testbot	2
	ClarityDailyBot	2
	Jetsetter	2
	Storebot	2
	WebsiteExtractor*	2
	mssearch5.0robot	2
	jobs.de-robot	2
	Sunrise/0.*	2
	Mozilla/5.0(WindowsNT6.2)Insitesbot/1.0	2
	void-bot	2
	MicrosoftURLControlâ€“6.00.8169	2
	ahoythehomepagefinder	2
	fetchrover	2
	facebookexternal	2
	/something/</pre>	2
	ParsijooBot	2
	AdxPsfFetcher-Google	2
	Mozilla/4.0(Windows98;US)Opera10.00[en]	2
	Mozilla/4.0(compatible;Spider;Linux)	2
	PEARHTTP_Request*	2
	SCOUTjET	2
	NusearchSpider	2
	Java/1.6.6_26	2
	momspider	2
	Mozilla/5.0(compatible;Theophrastus/*)	2
	serkiaBot	2
	InternetExploiter/*	2
	Sentibot	2
	spbot/2.1	2
	Mozilla/5.0(Macintosh;U;MacOSX;)AppleWebKit/()Pandora/2.*	2
	sch-fast-se-crawl04.osl.basefarm.net	2
	XenuLinkSleuth1.2g	2
	igdeSpyder	2
	Naver	2
	cassandra	2
	Busiversebot/v1.0	2
	eCatch*	2
	knil	2
	CFNetwork*	2
	cienciaficcion	2
	Google-Youtube-Links	2
	HTMLParser/*	2
	gsa-crawler-C	2
	Netsprint	2
	Orangebot	2
	SpankBot*	2
	SuperBot/*	2
	webinatorbot	2
	Python-urllib/2.7	2
	overture	2
	jobot	2
	//buytaert.net/crawler/)	2
	InternetArchive/*	2
	FATbot	2
	webwalker	2
	ProWebWalker*	2
	//www.200please.com/bot)	2
	solfo-linkchecker	2
	pitkow	2
	//www.clixsense.com/)	2
	Steeler/*	2
	w3index	2
	cz32ts	2
	Mozilla/5.0(compatible;BuzzRankingBot/*)	2
	xpymep.exe	2
	MS_Search	2
	DNAbot	2
	Yahoo!Slurp/SiteExplorer	2
	SmeshBot	2
	CheckDogBt	2
	Lockon	2
	coolbot	2
	MJ12bot#Donotallowmajestic12-usesgigsofdataperday	2
	Pajaczek/*	2
	Mozilla/3.01	2
	Shim?Crawler*	2
	WebDownloader/*	2
	about	2
	ISpider	2
	shelobv1.*	2
	Netprospector*	2
	spiderview	2
	Pockey*	2
	//www.miragorobot.com)	2
	FyberSpider*	2
	Go-Ahead-Got-It*	2
	snap.combetacrawlerv0	2
	//www.avantbrowser.com)	2
	infoseeksidewinder	2
	strucr-phone	2
	DownloadWonder*	2
	trendictionbot0.5.0	2
	Mozilla/5.0(compatible;NGBot/*)	2
	//showyou.com/crawler)	2
	InternetShinchakubin	2
	Y!J-SRD	2
	^NetShelter	2
	poppi	2
	suntek	2
	wada	2
	GetRight/*	2
	MicrosoftWindowsNetworkDiagnostics	2
	NP/*	2
	cliqzbot	2
	//open.etao.com/dev/EtaoSpider)	2
	us	2
	emcspider	2
	ScheduledCache	2
	Marfeel-crawler	2
	POEGrubCrawler	2
	pka	2
	DataCha0s/*	2
	Nutch/0.?(OpenXSpider)	2
	Slurp/2.0	2
	downloadexpress	2
	srmse/Nutch	2
	squirrly	2
	Panopta	2
	rixbot	2
	CerberianDrtrs/*	2
	//www.oneriot.com)	2
	//www.scoutjet.com/)	2
	Mozilla/4.0(compatible;MSIE?.0;SaferSurf*)	2
	WebsiteeXtractor*	2
	bot-pge.chlooe.com/1.0.0	2
	crawler-unibwm*	2
	Megaindex.ru	2
	HeinrichDerMiragoBot	2
	Baiduspider*	2
	CligooRobot	2
	merzscope	2
	//www.mindbreeze.com	2
	FLATARTS_FAVICO	2
	Zao-Crawler	2
	icq	2
	coccoc/1.0	2
	BaiDuSpider+	2
	Lexxe	2
	SOFTWING_TEAR_AGENT*	2
	MicrosoftURLControl-6.00.8862	2
	Puu	2
	//www.baidu.jp/search/s308.html)	2
	Bot)	2
	zspider	2
	HatenaRSS/*	2
	MicrosoftDataAccessInternetPublishingProviderCacheManager	2
	NetAnts*	2
	ng/*	2
	bingpreview	2
	Java/1.6.0_35	2
	AdShadow	2
	curiousgeorge	2
	freecrawl	2
	robbie	2
	Bloglines/3.1	2
	Camcrawler*	2
	Victoria	2
	informant	2
	GetRightPro/*	2
	PycURL/*	2
	Checkbot	2
	OnPageBot	2
	archive.org	2
	SSurf15a11	2
	Xenuâ€™s	2
	lockon	2
	FASTDataSearchCrawler	2
	NPBot*	2
	Tarantula/*	2
	Adsbot-Google*	2
	kdd	2
	PlantyNet_WebRobot*	2
	/.pdf$	2
	DownloadNinja7.0	2
	Superfeedrbot	2
	//www.entireweb.com/about/search_tech/speedy_spider/)	2
	Sqeobot/0.*	2
	tutorgig	2
	exabot.com	2
	PrivacyAware	2
	LOOQ/0.1*	2
	Mozilla/5.0(compatible;NetcraftSurveyAgent/1.0;*[email protected])	2
	Shelob([email protected])	2
	SiteParser/*	2
	NetcraftWebServerSurvey	2
	sgscout	2
	cruiser	2
	XSpider	2
	httperf/*	2
	Spiderline	2
	DataFountains	2
	Larbin	2
	DownloadDemon*	2
	MissiguaLocator*	2
	virus_detector*	2
	JetBrainsOmeaReader*	2
	addthis.com	2
	RubrikkBot	2
	wwwc	2
	TurnitinBot/*	2
	sensis	2
	Mozilla/5.0(X11;compatible;semantic-visions.comcrawler;HTTPClient3.1)	2
	SynthesioCrawlerreleaseMonaLisa(contactatsynthesiodotfr)	2
	VSAgent	2
	Google-AdsBot	2
	InternetExplore*	2
	NetPumper*	2
	NewsGator/*	2
	voila	2
	CerberianDrtrs	2
	POEGrubCrawler/0.01	2
	SPbot	2
	netscoop	2
	search-info	2
	Netseer	2
	'Unknownrobot'	2
	seplinkbot	2
	Atomic_Email	2
	GetSmart/*	2
	Mozilla/5.0(compatible;DKIMRepBot/*)	2
	sogouwebspider*	2
	SpiderMan	2
	getbot	2
	Goodzer/2.0	2
	polybot?*	2
	Bjaaland	2
	'bot\*'	2
	shaggy	2
	CyberPatrol*	2
	//github.com/cgiffard/node-simplecrawler.git)	2
	ShoeMoneyToolsBot	2
	MicrosoftInternetExplorer	2
	bingbot#Bing	2
	Grub	2
	Y!J-BRL/YATSS	2
	Senrigan	2
	rookee-bot	2
	TweakMASTER	2
	MicrosoftVisioMSIE	2
	POE-Component-Client-HTTP/*	2
	Arachnys/Nutch-1.12	2
	Cocoal.icio.us/()*	2
	HTTPGrab	2
	cuwhois	2
	//www.semrush.com/bot.html)	2
	NativeHost	2
	ingenieur	2
	converacrawler	2
	TwengaBot/1.1	2
	Urlbot	2
	francoroute	2
	//www.legalx.net)	2
	FairAdClient*	2
	BDCbot/1.0	2
	Mozilla/5.0(Twiceler*)	2
	P3PClient	2
	WebDownloader*	2
	MovableType/*	2
	FastCrawler	2
	/private/	2
	ADSAComponent	2
	slaskdatacenter	2
	Baiduspider-imagem	2
	sch-fast-se-crawl02.osl.basefarm.net	2
	360Spider-Video	2
	Java/1	2
	MerzScope	2
	d2cbot	1
	TutorGigBot	1
	Bot;	1
	amorphicCrawler	1
	Scopiacrawler1.2	1
	ContactBot	1
	Bixocrawler	1
	Atlas	1
	/cgi-bin/	1
	Xenu	1
	ReasearcherCrawler	1
	/2013-10-24-09-04-01/after-the-race?_escaped_fragment_PGER_2013_*	1
	Simple/5.803	1
	Easouspider	1
	OmniExplorer_Bot/1.	1
	Archive-It	1
	HY_crawler	1
	vebot	1
	ChatCatcher	1
	affectv	1
	Simple/5.814	1
	xChaos_Arachne	1
	PhpDig	1
	Showyoubot	1
	Twitturls	1
	binlar_2.6.3	1
	CATExplorador	1
	mmcrawler	1
	baiduspider	1
	Googlebot-	1
	KolinkaForumSearch	1
	PlantyNet	1
	IRLCrawler	1
	linkCheckV3.0	1
	PythonUrlLib	1
	xbingbot	1
	Yandex	1
	WinkBot/0.06	1
	,\.\;\/\\-]bot	1
	NetResearchServer	1
	ASpider(AssociativeSpider)	1
	Monster	1
	PopScreen	1
	TheJubiiIndexingRobot	1
	MicrosoftPrototypeCrawler	1
	IDentity	1
	Links4US-Crawler	1
	seebot	1
	Pagespeedbot	1
	Search4Free	1
	sitecheck.internetseer.com	1
	yoono	1
	//www.linkdex.com/m/bots/	1
	Aboundex/0.3	1
	*#appliestoallrobots	1
	Qryos	1
	HomepageClone	1
	I,Robot	1
	Screaming+Frog+SEO+Spider	1
	AlexaToolbar	1
	GenCrawler	1
	quipu/1.0	1
	Python-urllib/1.17	1
	/*id=	1
	MapoftheInternet.com	1
	Minerva	1
	FreeWebMonitoringSiteChecker/	1
	SEOlyticsCrawler/3.0	1
	Y!J-ASR/0.1	1
	BaboomBot	1
	DoCoMo/2.0N905i(c100;TB;W24H16)	1
	libwww-perl.XXXX	1
	popIn_Agent	1
	GooglebotGooglebot-MobileGooglebot-ImageMediapartners-GoogleAdsbot-GoogleSlurpmsnbotmsnbot-mediaYahoobotMicrosoftbot	1
	Mini-Crawl	1
	Rogerbot/	1
	//napoveda.seznam.cz/en/seznambot-intro/	1
	LinkFeatureBot	1
	RankLite	1
	Java/1.5.0_02	1
	MiragoBot	1
	ThePythonRobot	1
	ComputingSiteRobi/1.0	1
	Feedster	1
	semetrical	1
	WinInet	1
	architextspider	1
	HatenaRSS	1
	Presto/2.6.30	1
	CoinCornerBot/1.1	1
	werelate	1
	Pimptrain	1
	Evri	1
	Exabot	1
	Google-Adwords-Instant	1
	/rew	1
	FaceBookbot	1
	//nutch.apache.org/bot.html	1
	Hobbit_bbtest-net/4.2.0	1
	OptimizationCrawler	1
	Googlebot-Image	1
	statdom.ru/Bot	1
	CompeteCrawler	1
	twitmatic	1
	Speedy\Spider	1
	semalt.semalt.com	1
	CSULibrariesNutch	1
	HTTrack	1
	ConfuzzledBot	1
	107.150.49.242	1
	Speedy+Spider+	1
	Subscribe.Ru/1.0	1
	rorrimBot	1
	robots.txt	1
	PageNestFreeEdition/3.10	1
	Xpider	1
	WhatsApp	1
	TutorGig	1
	SmartSpider	1
	SpryWizardRobot	1
	dotbot#drugoimezaAhrefsBota	1
	AVSearch	1
	Kyoto-Tohoku-Crawler/v1	1
	Node/simplecrawler0.5.2	1
	Openstat/0.1	1
	//www.profound.net/urlappendbot.html	1
	mlbot.578EBFDE8D919BCD87B0AF957CA754D7	1
	factbot	1
	GazoPabot	1
	GrifinBot/0.01	1
	NimbleCrawler	1
	EITLinkVerifierRobot	1
	RoadHouseCrawlingSystem	1
	RankvalBot	1
	TANNER	1
	BlogTraffic/1.3Feed-Fetcher	1
	LSSRocketCrawler/1.0LightspeedSystems	1
	spray-can	1
	traffic2cash.xyz	1
	Mediapartners-Bing	1
	123Peoplebot-Image	1
	Bot,	1
	//www.webmasterworld.com/search_engine_spiders/4427797.htm	1
	TechnoratiSnoop	1
	NPT	1
	WebCopierv3.6	1
	BlogStreetBot	1
	Slurp#YahooSlurp	1
	Baidu#	1
	Apache-HttpClient	1
	DuckDuckGo-Favicons-Bot/1.0	1
	HooWWWer	1
	FluidDynamicsSearchEnginerobot	1
	RobotFrancoroute	1
	dialect	1
	30	1
	Java1.3.1_03	1
	Pcore-HTTP/v0.24.5	1
	TheInformant	1
	POPrlBot	1
	Java/1.6.0_03	1
	feedspider0.wise-guys.nl	1
	CB/Nutch-1.7	1
	Thumbtack-Thunderdome	1
	ananzi	1
	WebReaperv9.8–www.webreaper.net	1
	ConveraCrawler*	1
	evc-batch/2.0	1
	MaxPointCrawler/Nutch-1.1(maxpoint.crawleratmaxpointinteractivedotcom)	1
	FeedFetcher	1
	MacFinder1.0.xx	1
	Zango	1
	Unknownrobot(identifiedbyemptyuseragentstring)	1
	grub-client-1.5.3	1
	MSNPTC/1.0(compatible;MSIE6.0;WindowsNT5.2;MyIE2;.NETCLR1.1.4322;.NETCLR1.0.3705)	1
	InsightsCollector/0.1	1
	MicrosoftScheduledCacheContentDownloadService	1
	Nutch-Staff	1
	boithocom-dc	1
	robot	1
	[email protected]	1
	DownloadNinja5.0	1
	[email protected]	1
	JubiiRobot	1
	Kyoto-Tohoku-Crawler	1
	Digincore	1
	Vegi*	1
	Jetty	1
	HHSGoogleTest	1
	goo	1
	/cgi-bin/	1
	ZEEFscraper	1
	w3af.org	1
	antibot	1
	sahabatdinarbot	1
	Imagelock	1
	JungleeBot	1
	WISEbot	1
	BeetleBot	1
	dloader(NaverRobot)\/1.5	1
	YebolBot	1
	mozilla/5.0	1
	FDSE	1
	Mozilla/[email protected]	1
	CrownPeakSearchG2Crawler	1
	semiocast	1
	Sosoblogspider	1
	DigimarcMarcSpider	1
	Googlebot-news	1
	82.233.155.60	1
	gsa-crawler(Enterprise;M2-KQWU4PEKDA2JA;[email protected])	1
	www.kb.nl	1
	Magnolia	1
	Slurp#дабыненагибалсайт	1
	netestate	1
	WhatWeb/0.4.8-dev	1
	FLR-Bot	1
	Net_Vampire	1
	iCCrawler	1
	Suke	1
	Googlebot-Image	1
	TwengaBot*	1
	Semrush-SA	1
	CharlesUserAgent	1
	Java1.4.0_01	1
	alexasiteaudit	1
	bingbot\	1
	ObjectsSearch	1
	ZipppBot	1
	libwww	1
	Pajaczek	1
	Lingewoud	1
	Metalogger	1
	TeleportPro/1.29.1632	1
	Domnutch-Bot/Nutch	1
	accelovation	1
	100	1
	PortalJuiceSpider	1
	FASTEntepriseCrawler	1
	BoogleBot2	1
	NetCartaWebMapEngine	1
	kmSearchBot	1
	Baiuspider	1
	207.46.13.146	1
	MicrosoftURLControl.6.00.8xxx	1
	Mozilla/4.0(compatible;MSIE8.0;WindowsNT5.1;Trident/4.0;.NETCLR2.0.50727;.NETCLR3.0.04506.648;.NETCLR3.5.21022;	1
	inktomisearch.com	1
	MicrosoftCorp	1
	LARBIN-EXPERIMENTAL([email protected])	1
	LinkedInBot/1.0	1
	badbot	1
	Slurp#дабыненагибалфорум	1
	Ahrefs-Bot/2.0	1
	Uptimerobot	1
	gridBOT	1
	zoomRank/3.0	1
	IstellaBot/1.18.81	1
	BoardReaderImageFetcher	1
	Googlebots	1
	LinqiaScrapeBot/1.0	1
	VeriCiteCrawler/Nutch-1.9	1
	Java/1.6.0_33	1
	GooglebotImages	1
	//www.youdao.com/help/webmaster/spider/	1
	TURNITINBOT	1
	blockdotbot	1
	Centric	1
	WWW-Mechanize/1.12	1
	YandexVideo/3.0	1
	gsa-crawler+	1
	//fulltext.seznam.cz/)	1
	aol	1
	/	1
	EDI	1
	Email	1
	//www.linguee.com/bot	1
	Openindex	1
	//scrapy.org)	1
	lycos#Lycos	1
	Netscape	1
	logo_gif_crawler	1
	Szukacz/1.5	1
	Slurp#Yahoo	1
	Searchbot	1
	Pinterest/Nutch-2.3	1
	Ponyfish	1
	Jetbot/	1
	DiBot+Java	1
	loadka.ru	1
	Mediapartners-Google,yahoo,bling,uc,opera	1
	ResumeRobot	1
	NHNCorp	1
	Pompos/1	1
	/*.jpg$	1
	VorbossWebCrawler	1
	BIGLOTRON	1
	gsa-weizmann-crawler	1
	Akamai	1
	TclW3Robot	1
	XenuÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂ¢ÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂsLinkSleuth1.1c	1
	SearchNZ	1
	DIIbot\/1.2	1
	Blazer1.0	1
	//tools.geek-tools.org	1
	008	1
	linkedinbot	1
	your-search-bot	1
	google-bot	1
	Slurp#Yahoo?	1
	Link/1.0	1
	YandexRobot	1
	Robot	1
	TurnitinBot#blocked2/1/2012-usedfortermpapers	1
	NINJA	1
	CACTVSChemistrySpider	1
	BingRobot	1
	plinki	1
	MSIE4.01;WindowsNT;MSSearch6.0Robot	1
	Drupal	1
	kame-rt	1
	best-seo-software.xyz	1
	//www.sygol.net	1
	yahoo-MMAudVid	1
	Xenu\\\'s	1
	/wp-admin	1
	DomainSONOCrawler	1
	darbonis	1
	Yahoo!Slup	1
	Superfeedrbot/2.0	1
	Mozilla/5.0(compatible;Goodzer/2.0;[email protected])	1
	Hatena-Mobile-Gateway	1
	yahoo.net;NS1.YAHOO.COM;NS2.YAHOO.COM;NS3.YAHOO.COM;NS4.YAHOO.COM;NS5.YAHOO.COM	1
	Sspider_080915	1
	NorthStar	1
	NetCarta_WebMapper	1
	Twiceler/	1
	delicious-thumbnails	1
	Knowings	1
	ToutiaoSpider	1
	i-search-crawler	1
	^.SemrushBot.	1
	WWW-Mechanize/1.74	1
	BCGovSearch	1
	heeii/Nuts	1
	/Zen_Files/	1
	RPT-HTTPClient	1
	XYLEMERobot	1
	AURESYS/1.0	1
	searchbot	1
	Download	1
	cuwhois/1.0	1
	Gets	1
	IDGCrawler/Nutch-1.8	1
	twenga	1
	Unknownrobot(identifiedby'crawl')	1
	Mediatoolkit	1
	Modiphibot	1
	^.dotbot.	1
	LinqiaMetadataDownloaderBot/1.0	1
	Deepnet	1
	*	1
	WbSrch	1
	lexxebot	1
	Spock	1
	Mail.Ru_Bot	1
	Java/1.4.2_04	1
	Java/1.5.0_03	1
	siclab	1
	FeedBurner/1.0	1
	eSobiSubscriber	1
	betabot	1
	Spider_Monkey	1
	YahooAdMonitoring	1
	Whitevector	1
	ScreamingFrogSEOSpider/3.2	1
	GetintentCrawler	1
	//napoveda.seznam.cz/cz/seznambot/	1
	Gib	1
	Scirus	1
	woobot/2.0	1
	//support.sitesell.com/contact-support.html)	1
	WebFindBot	1
	AdSonarBot	1
	xx)	1
	usasearch	1
	EasouSpider*	1
	BoardReaderFaviconFetcher	1
	ws\zone\crawler\/Nutch-1.4	1
	scrapbot	1
	DownloadNinja	1
	GuggenBot	1
	libsys-crawler	1
	Bingbot-media	1
	//www.unchaos.com	1
	SiteCheck-sitecrawl	1
	node.io	1
	zealbot	1
	GoogleMobileAdSense	1
	Pinterest/0.1	1
	DigitalIntegrityRobot	1
	Muncher	1
	WWWCVer0.2.5	1
	YandexVideo	1
	R6_FeedReader	1
	waseda_koba_bot	1
	Java/1.6.0_01	1
	TitIn	1
	EZOOMS	1
	Wow64	1
	About	1
	ICEBrowser	1
	noxtrumbot/1.0+	1
	AntBot	1
	linkdex.com/v2.1	1
	metauri	1
	Xenu'sLinkSleuth1.1c	1
	iSiloX	1
	SearchengineLicenceSheep	1
	MicrosoftURLControlÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂ¢ÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂ6.00.8169	1
	185.40.4.41	1
	okhttp	1
	Sindup	1
	//git.io/tl_S2w	1
	InktomiCorp	1
	msn.com	1
	ShopSalad	1
	CoolBot	1
	*	1
	AITCSRobot/1.1	1
	webindex	1
	Vegibot(wefollowyourrobots.txtsettingsbeforecrawling,youcanslowdownthebotbychangetheCrawl-Delayparameterinthesettings.ifyouhave	1
	Googlebot-IA	1
	ArabyBot	1
	Yoono	1
	Googlebot-*	1
	slurp/si	1
	htdig/3.1.5(root@localhost)	1
	DoCoMo/2.0P900i(c100;TB;W24H11)	1
	NetcraftSpider	1
	UrlCrawler	1
	duggmirror	1
	WWWC	1
	HopperBot	1
	MailRU	1
	SOTScraper	1
	InfociousBot	1
	SobiSubscriber	1
	Goolebot	1
	WmailSiphon	1
	MaMaCaSpEr	1
	Ning	1
	Synapse #synapsecrawler	1
	MerchantCentricBot	1
	gonzoP	1
	marvin/infoseek	1
	MetaSpider	1
	Pu_iN	1
	Site-Shot	1
	special_archiver/3.1.1	1
	//www.picsearch.com/bot.html	1
	YodaoBot-Image/1.0	1
	Java/1.6.0_29	1
	Slurp	1
	MissouriCollegeBrowse	1
	Bingbot-	1
	watson-url-fetcher	1
	atomicbot/1.0	1
	//www.feeddemon.com/	1
	EmailSiphon,ExtractorPro,Teleport,NICErsPRO,EmailCollector,CherryPickerSE/1.0,CherryPickerElite/1.0,EmailWolf1.00,CrescentInternetToolPakHTTPOLEControlv.1.0,EmailSiphon,SearchmetricsBot	1
	BazQuxCrawler	1
	//www.webintegration.at/jobroboter_suchmaschine	1
	//www.webmasterworld.com/search_engine_spiders/3895299.htm	1
	FELIXIDE	1
	pr-cy.ru	1
	Bling	1
	geobot	1
	Hatena-UserAgent/0.02	1
	larbin_2.6.3\[email protected]	1
	LARBIN-EXPERIMENTAL	1
	Scopiacrawler1.1	1
	Fast-Webcrawler	1
	psbot(picsearch)	1
	autokrawl	1
	UCSDCrawl	1
	pinterest	1
	BingPreview/1.0b	1
	openbot	1
	sna-0.0.1\[email protected]	1
	AuditMyPC	1
	linkCheck	1
	linko	1
	YandexMedia/3.0	1
	185.17.24.0/255	1
	//mj12bot.com/	1
	//search.goo.ne.jp/option/use/sub4/sub4-1/	1
	AdFenixSegmentationCrawler	1
	webdownloader	1
	ccBot	1
	MyNutchSpider/Nutch-1.9	1
	/includes/	1
	Pagebull	1
	php	1
	WGET	1
	Search.Aus-AU.COM	1
	SindiceFetcher	1
	//www.aspseek.org	1
	Facebook	1
	CommentReader	1
	Sohu	1
	ZemantaAggregator	1
	CuriousGeorge-www.analyticsseo.com	1
	Istellabot	1
	KingKevinBot	1
	wisenutbot	1
	IIITBOT/1.1	1
	node.js	1
	Siteluxbot/1.0	1
	Linkdexbot/2.1	1
	//www.cherrypicker.com/>	1
	Ahrefs-Bot/3.0	1
	Monkeybot/0.1	1
	free-social-buttons.xyz	1
	Vonna.combot	1
	WebWatch	1
	[almaden.ibm.com...]	1
	Quantcastbot	1
	mediaBot	1
	HatenaScreenshot	1
	booch_1.0.7	1
	Shelob+([email protected])	1
	Twingly	1
	Java/1.4.2	1
	Nekstbot	1
	ATNWorldwide	1
	//www.youdao.com/help/webmaster/spider/;)	1
	YaDirectFetchert	1
	ScreamingFrogSEOSpider/7.2	1
	Distilled-Cache	1
	Java/1.4.2_05	1
	ichiro/3.0	1
	ClickTalebot	1
	SSG/3.0	1
	//riddler.io/about)	1
	Mingw	1
	EBISearch	1
	DulanceBot	1
	shadow	1
	SnookBot	1
	Emacs-w3SearchEngine	1
	reddot-scraper.unicef.org	1
	w2gbot/1.0	1
	kagoobot	1
	Yandex11111111111111	1
	PowerPivot	1
	WebsterProV3.4	1
	WeCrawlForThePeace	1
	Googlebot-Image#Googlebot	1
	//help.naver.com/customer/etc/webDocument02.nhn	1
	Websnarf	1
	xtmobile.vn	1
	Feedtrace	1
	Contextly	1
	EuripBot	1
	Mozilla/4.0(compatible;AdvancedEmailExtractorv2.xx)	1
	PeerFactorCrawler	1
	Rapleaf	1
	PowerMapper	1
	A\.NETWebCrawler	1
	//www.sitesell.com/sbider.html)	1
	URL_Spider_SQL	1
	WebVulnCrawl.unknown/1.0libwww-perl/5.803	1
	Mozilla/5.0(WindowsNT5.1)AppleWebKit/535.19(KHTML,likeGecko)Chrome/18.0.1025.162Safari/535.19	1
	bhcBot	1
	Infoseek	1
	msnbot	1
	OpenIntelligenceData	1
	TalkroWeb-Shot	1
	Scooter//ForAltavistaBot	1
	Extractorpro	1
	AtoshoFeedCrawler/1.0	1
	Domaincrawler#roguebot,noinfoatwww.domaincrawler.com	1
	backlinkrastreador	1
	WebDownloader/5	1
	WebStripper/2	1
	linklift	1
	//www.wesee.com/bot/	1
	FaceBook[Linkcheck]	1
	docomo_snr/1.0	1
	Sirketce	1
	Yahoo!Slurp;	1
	Mozilla/4.0(compatible;MSIE5.5;WindowsNT5.0)FetchAPI	1
	fusionbot	1
	Mediaparthners-Google	1
	DigitalOcean	1
	bingbot-newsblogs	1
	DYNAMIC	1
	Votaybot	1
	MSNBot#Microsoft	1
	Java/1.4.1_02	1
	Y!J-DLC/1.0	1
	NATE.ROBOT	1
	IXECrawler	1
	miggibot	1
	fr_org_viewer	1
	ru-robot	1
	iGetter	1
	ShoppimonAgent	1
	TSurf15a11	1
	Linkapediabot	1
	libmetha-agent	1
	Guardcrwlr	1
	//www.changedetection.com/bot.html	1
	H?m?h?kki	1
	WebStripper/2.61	1
	Mozilla/3.0(compatible;scan4mail(advancedversion)	1
	mediapartners-google*	1
	linkcheck	1
	citycrawler	1
	PC\SOFT\Framework	1
	quipu/2.0	1
	Java/1.6.0_26	1
	Yandex#ПравиладляпоисковикаЯндекс	1
	GroupHigh	1
	Anonymous/0.0	1
	/glasfunds.com/	1
	LabelGrabber	1
	RegatorBot	1
	Xenu.sLinkSleuth1.1c	1
	Googlebot#Google#CUSTOM	1
	Calypsov/0.01	1
	RivalSeek.com-Bot	1
	GetRight/4.x[a-e]	1
	Java/1.6.0_36	1
	NutchBOA/Nutch-1.0-dev	1
	.*	1
	aiHitBot-DS	1
	Mediatoolkitbot([email protected])	1
	tineye-bot	1
	Webreaper	1
	//www.gigablast.com/spider.html	1
	OrbSearch	1
	bot-	1
	IIITBOT	1
	msnbot#msn	1
	SpamBayes/1.1a3+	1
	MSIndianWebcrawl	1
	NameProtectRobot	1
	Websquashcom	1
	Szukacz	1
	Cusco	1
	PiltdownMan	1
	theWorldWideWebWanderer	1
	bookler.nichost.ru	1
	Garlik	1
	BuiBui	1
	Digimind	1
	Setooz\/Nutch-1.2	1
	//www.miragorobot.com/scripts/deinfo.asp)	1
	mediapartners-Google	1
	VegiBot	1
	boithocom-robot	1
	ZyBORG	1
	GoForIt.com	1
	Majestic	1
	2ADAMbot/1.0	1
	Putin	1
	Zeabot	1
	fastbuzz.com	1
	MizzuLabs2.2	1
	//help.goo.ne.jp/help/article/1142/)	1
	Gigamega	1
	Hivemind	1
	Clipper	1
	Pinterest/iOS	1
	iecrawler	1
	Ezooms.bot	1
	WISENutBot	1
	TweetmemeBot/3.0	1
	ascribeBot	1
	Pagespeed/1.1Fetcher	1
	Wazzup	1
	Yasnibot-image	1
	PlantyNet_WebRobot	1
	magpie-crawler#	1
	www.21seek.com	1
	Digincorebot	1
	influencebot	1
	TackBot	1
	//dir.com/pompos.html	1
	coccobot-web	1
	//www.facebook.com/externalhit_uatext.php	1
	moreover	1
	Java/1.4.2_08	1
	AOL	1
	crawlpaper	1
	EmeraldShield.comWebBot	1
	YandexBlogs/0.99	1
	Netscape-Proxy	1
	CORE	1
	baypup	1
	izsearch.com	1
	WebCopiervx.xa	1
	PrintfulBot	1
	Fouineur	1
	Echo2	1
	YahooVideoSearch	1
	NIXStatsbot	1
	tweetmemebot	1
	Route53	1
	DuckDuckGoBot	1
	zmeu	1
	SogouOrion	1
	lwp-trivial\/1.40	1
	asafaweb	1
	cuil.com	1
	vision-search	1
	kyklo	1
	KOCMOHABT	1
	JyxoBot	1
	followsite	1
	MSIE8.0	1
	com_viewer	1
	JetBrains	1
	EvriNid	1
	Mattie	1
	seocharger-robot	1
	majestic12.co.uk	1
	BecomeBot	1
	WebQuest	1
	CASANOVAO.1	1
	BizBot04kirk.overleaf.com	1
	REAP-crawler	1
	palseek	1
	W3C-gsa	1
	109.207..	1
	USyd-NLP-Spider	1
	EvliyaCelebi	1
	showyoubot	1
	SOAbot	1
	lindex	1
	wget	1
	Dumbot	1
	moget/2.0([email protected])	1
	echocrawl	1
	AlkalineBOT	1
	Araneo	1
	antibot-V1.4.7	1
	ppclabs_bot	1
	LinkCheckbySiteImprove.com	1
	mr-webcrawler	1
	HostHarvest	1
	image.coccoc/1.0	1
	DownloadAccelerator(downloader)	1
	DeWeb(c)Katalog/Index	1
	RBSESpider	1
	TLSpider	1
	selfbot	1
	78.157.216.128/255	1
	Slack	1
	Unknownrobot(identifiedbyhiton'robots.txt')	1
	Scumbot	1
	bieshu	1
	BINGbot	1
	MJ12bot/v1.4.4	1
	socialbm_bot	1
	heretrix	1
	spotinfluence	1
	webmeup.com	1
	[email protected]	1
	sensis.com.au	1
	blogspotbot	1
	linklooker	1
	Bizbot003	1
	//ruki.rezko.net)	1
	StanbyCrawler	1
	N/A	1
	sistrex	1
	2ADAMbot	1
	aboutthedomain	1
	MOMspider	1
	/api/	1
	amzn_assoc	1
	Rika	1
	cognitiveseo	1
	googleboot	1
	PeerFactor404crawler	1
	Dillo\/0.8.6-i18n-misc	1
	PerlCrawler1.0	1
	*MSNBot	1
	MegaSheep	1
	LinkVerifier	1
	//www.brandwatch.net)	1
	grub-client	1
	77.134.170.38	1
	iSiloWeb	1
	Tarantula\Experimental\Crawler	1
	Java/1.4.2_03	1
	Pagespeed/1.1	1
	//igde.ru/doc/tech.html)	1
	jjBot	1
	//linkfluence.net/;[email protected])	1
	CRAZYWEBCRAWLER0.9.0	1
	IchiroRobot	1
	Toweya.com	1
	bingBot	1
	HIVmd_robot	1
	DomainAppender/1.0	1
	amaya	1
	NetcraftSurveyAgent	1
	piratenpartei.yacy	1
	Twiceler	1
	Googlebot-Image*	1
	uptime.com	1
	Java/1.6.0_28	1
	pipeLiner	1
	//pompos.iliad.fr	1
	unwrapbot	1
	DirBuster	1
	AppleWebKit/412	1
	/akamai/akamai-sureroute-test-object.htm	1
	hl_spider	1
	//www.analyticsseo.com/the-analytics-seo-crawler-curious-george/	1
	//webmeup-crawler.com/)	1
	JetBot/1.0.	1
	Haiula	1
	SheerBoredom.Experimental.Robot	1
	Spade	1
	groupangle	1
	Toplistbot	1
	jobcrawler	1
	vcbot	1
	ScorpionBot	1
	Yahoo!Slurp#Yahoo	1
	IUPUIResearchBotv1.9a	1
	baiduspider#Baidu	1
	WSBWebCrawlerV1.0(Beta),[email protected]	1
	snap.com	1
	VisBot	1
	//www.botopedia.org/user-agent-list/search-bots/item/340-yeti-naverbot	1
	Sisi	1
	downloadninja	1
	kspider	1
	FindLinks	1
	*.choopa.net	1
	HKUWWWOctopus	1
	WEPSearch00	1
	Charon	1
	YandexBot#	1
	BlogScope/1.0	1
	Robot_Name	1
	heritrix/1.8.0	1
	Scopiacrawler1.0	1
	GooglebotVideo	1
	KFSW-Bot	1
	YandexBot#stopcommoncrawlbotfromcrawlingthesite.	1
	Jobot	1
	Go.Zilla	1
	Sphere\Scout	1
	offlineexplorer	1
	CollapsarWEB	1
	DienstSpider	1
	IntelliAgent	1
	Mozilla/5.0(compatible;AhrefsBot/5.0	1
	kulokobot	1
	GoogleAdsBot	1
	Fetch	1
	*#Forallotherrobots	1
	CoPubbot	1
	bot	1
	//help.goo.ne.jp/help/article/853/	1
	SemrushBot\/0.91	1
	Crawl#Yahoo	1
	pr-cy	1
	Parser	1
	NG	1
	Pita	1
	cmscrawler	1
	um-IC/1.0	1
	ahoy	1
	KnowItAll	1
	//www.bing.com))	1
	blogmuraBot	1
	sufog.com	1
	Zapier	1
	TCGfetch	1
	woriobotheritrix	1
	CakePHP	1
	md5sum	1
	python-requests/1.2.3CPython/2.7.3Linux/3.3.8-gcg-201308121035	1
	IncyWincy	1
	msnbot	1
	65.52.109.194	1
	QroboBOT	1
	msnbot-media*	1
	Papers	1
	Hyper-Decontextualizer	1
	PageAnalyzer/1.1	1
	TheNorthStarRobot	1
	Y!J-BRW/1.0crawler	1
	duggmirror\	1
	Talkwater	1
	eCommerceBot	1
	Googlebot-images	1
	Amfibibot	1
	IzumSearch	1
	TheTkWWWRobot	1
	WallPaper(aliascrawlpaper)	1
	MJ12bot #majesticCrawler	1
	Architextspider	1
	"SevenvalFIT"	1
	HatenaBookmark	1
	//nationaltaxreports.com	1
	TwitterFeed3	1
	InsightsCollector	1
	Mediapartners-Google/2.1	1
	SLCrawler/2.0	1
	Sift	1
	93.104.208.0/255	1
	Seznamscreenshot-generator2.1	1
	Teleport+Pro/1.29	1
	Mp3Bot/1.0+	1
	Bot_	1
	Watchfire\WebXM	1
	Abonti/0.92	1
	wscheck.com	1
	EmeraldShieldcomWebBot	1
	Anthill	1
	77.75.124.128/255	1
	darcbot	1
	//freemyfeed.com/	1
	SemrushBot-A	1
	Java/1.4.1_01	1
	followlove.ru	1
	SimBot	1
	KO_Yappo_Robot	1
	sunteksearchengine	1
	148.251.1.115	1
	sfFeedReader	1
	domcom	1
	Bilbo/1.2+WAP	1
	scooter	1
	Mediapartners-Google\	1
	TailsweepBot	1
	spray-can/1.2.1	1
	VoilaRobot	1
	Custo3(Netwu.com)	1
	UniversalRobot/1.0	1
	Aretha	1
	WeblogMonitor	1
	Opera/9.0(WindowsNT5.1;U;en)	1
	CentricBot	1
	peopleman/1.6	1
	008/0.83	1
	wordpress	1
	Nudelsalat	1
	i-search-crawler/2.0	1
	Twitterfeed	1
	renlifangbot	1
	/Unrelated_Watkins/	1
	WebStripper	1
	LucidMedia	1
	gimme60	1
	c14542.sgvps.netWebEnhancer	1
	NjuiceBot	1
	Y!J-BRI/0.0.1crawler	1
	semantic-visions.com	1
	SOA	1
	Program\Shareware	1
	Web-sniffer/1.1.0	1
	VorbossWebCrawler/Nutch-2.3	1
	GetRightPro/6.0beta2	1
	webharvest01.kb.nl	1
	WhiteVector	1
	msnbot	1
	Bumblebee	1
	Edacious	1
	AhrefsBot#	1
	Itah	1
	MerchantCentric	1
	WordPress\/3.2.1	1
	BattleztarBazinga/0.01	1
	Image2play	1
	MyGreatUA/2.0	1
	zzabmbot	1
	microsoft.url.control	1
	FTRF	1
	rogerbot #MozCrawler	1
	Zscho.deCrawler	1
	YandexZakladki	1
	Down2Web	1
	feedly	1
	rojerbot	1
	feedworker/1.0crawler	1
	snapsitemap	1
	Java/1.5.0_04	1
	rootlink	1
	Seosys/Nutch-2.3	1
	best-seo-offer	1
	YowedoBot	1
	Dhoondho.com	1
	Go--client	1
	jyxobot/1	1
	balihoo	1
	YandexMarketpicturerobot	1
	openstat.ru	1
	^.YandexBot.	1
	evc-batch	1
	SetCronJob/1.0	1
	Python-urllib/2.6	1
	CollapsarWEBqihoobot	1
	Yanxdex	1
	Xenu\\\'sLinkSleuth1.1c	1
	YandexImages#	1
	CTBOT	1
	checkgzipcompression.com	1
	mezhpozvonochnoi	1
	WebReaper	1
	Kilroy	1
	WakameCrawler/0.01	1
	webalta*	1
	endeca*	1
	//www.baidu.com/search/spider.htm;Ubuntu;Compatible;Version;Platform;.NETCLR2.0.50727;Version)	1
	GerURL	1
	cfetch	1
	Microsoft-WebDAV-MiniRedir/5.1.2600	1
	ProductAdsBot/1.0	1
	BilgiBot	1
	SiteVigil	1
	MicrosoftURLControl.6.00.8169	1
	Calypso	1
	VWbot_K	1
	Mediapartners-Google*\	1
	newsme	1
	mnsbot/1.0+	1
	Gigabot/2.0att	1
	crawl/0.4	1
	//letscrawl.com/	1
	holmes/3.12.2+	1
	Googlebot-Image\	1
	Applebot/	1
	/trap/	1
	Pulsepoint	1
	AhrefsBot.Feeds	1
	listicka	1
	PRIVACY_ENFAQ.jsp	1
	Java/1.6.0_38	1
	Ukonline	1
	Synaps	1
	SafetyNetRobot	1
	WebStolperer	1
	FASTEnterpriseCrawler/5.3.4([email protected])	1
	DotBot/	1
	Seotome	1
	WebLinker	1
	WebReapervWebReaperv7.3–www,otway.com/webreaper	1
	WebVac+	1
	xcms_search_engine	1
	HeinrichDerMiragoRobot	1
	AdReport	1
	ChristCrawler.com	1
	ArchiveGridCrawler	1
	//www.pcaccessoriesparts.com)	1
	Boomtrain-Content-Bot	1
	promobit	1
	RollCrawl	1
	LinkSleuth	1
	pingbot	1
	ALeadSoftbot	1
	NoteworthyBot	1
	GWPImages/1.0	1
	StumbleUponInc.	1
	Emacs	1
	M12botMJ12botYandexBLEXBot	1
	Syndic8	1
	Jorgee	1
	SkreemRBot	1
	//www.fastbot.de)	1
	OpenTextSiteCrawler	1
	ScreamingFrogSEOSpider/5.1Beta2	1
	Stratagems	1
	WebQL	1
	psbot	1
	nfsagsa	1
	Peter\Wang\/Nutch-0.9	1
	*#CUSTOM	1
	Mike-Crawler	1
	//Dig	1
	BecomeBot	1
	157.55.17.194	1
	Mobilemaps	1
	googlemediabot*	1
	HTTPFetcher	1
	googlebot #allowGooglecrawler	1
	Bot\	1
	Bluecoat	1
	TurtleScanner/1.4	1
	AssociativeSpider	1
	Haiula/1.4	1
	SNKScreenshotBot/0.20	1
	ramBotxtremex.x #(BAD)UnknownURL	1
	TEOMA#Ask.com#CUSTOM	1
	zzabmbot/1.0	1
	cuill.com	1
	InspectorWeb	1
	MacWWWWorm	1
	W3M2	1
	GACheck	1
	scrutiny/4	1
	WeAreNotEvil	1
	NorthernLightGulliver	1
	Katipo	1
	Mediapartners*	1
	AmazonCloudFront	1
	botlist	1
	Bot.	1
	//www.garlik.com/	1
	Sunrise	1
	FlipboardRobot	1
	integrity/5	1
	linkoatlbot	1
	TurnitinBot#appliestoTurnitinBotrobots	1
	MediaFox	1
	WebReaperv9.1–www.otway.com/webreaper	1
	93.104.215.255	1
	gsa-crawler-du	1
	CRAZYWEBCRAWLER	1
	//www.google.com/bot.html	1
	//medical-info.de/	1
	ProPowerBot\/2.14	1
	evc/2.0	1
	JetBrains5.0	1
	WebCookies/1.0	1
	WhatWeb	1
	Mozilla/5.0(WindowsNT6.1;WOW64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/32.0.1700.107Safari/537.36	1
	dcbspider	1
	Patric	1
	BegunRobotCrawler	1
	gocrawl	1
	finbot	1
	SpokeSpider/1.0	1
	Najdi.si	1
	WebZIP	1
	HTTPClient	1
	synthesio	1
	mycrawler	1
	skynutchcrawler/Nutch-1.9	1
	Jetbot	1
	//www.synoo.de/bot.html;[email protected])	1
	WASALive	1
	crawler.sistrix.net	1
	Baidumobaider	1
	Viralheat	1
	visbot	1
	COMODOspider/Nutch-1.0	1
	ScreamingFrogSEOSpider/2.55	1
	LinkValidator	1
	CLEWWA-BOT	1
	SiteSearchASP.NET	1
	ng	1
	Pogodak!	1
	WebDownloader/6	1
	Spiderlytics/1.0	1
	SLurp	1
	AddCatalog	1
	Prlog/1.0	1
	V1.0/1.2	1
	//tweetmeme.com/	1
	GetRight/4.x	1
	botwSpider	1
	Semr	1
	FeedBucket	1
	llnw.net	1
	roboto	1
	8484BostonProject	1
	KeybotTranslation-Search-Machine	1
	Bot+	1
	Yahoo-VerticalCrawler	1
	RED.	1
	msiecrawler	1
	LinkWalker/3.0	1
	Megaindex.ru/2.0	1
	ContextWeb	1
	//home.snafu.de/tilman/xenulink.html	1
	bot\/1.0	1
	Dispatch/0.11.0	1
	eBayRelevanceAdCrawler*	1
	* #Matchallbots	1
	/camille/	1
	Lycos_Spider_(T-Rex)	1
	ukpetmartbot	1
	googlebot-news#onlythenewsservice	1
	ChinaSlurp	1
	JemmaTheTourist	1
	MLbot	1
	WebsiteExplorer	1
	WooRank	1
	//www.kosmix.com/crawler.html	1
	page-store	1
	ILU	1
	Mozzila/Nutch-1.0	1
	SearchImprove-crawlerbySiteimprove.com	1
	SpiderOne	1
	WebCopier	1
	Feedspotbot/1.0	1
	tivraSpider	1
	aggregatorVocusBot	1
	PentonMediabot	1
	Pimptrain.com'srobot	1
	fetcher	1
	SMTBot(similartech.com/smtbot)	1
	/magentoxyz_bkp/	1
	SystemCenterOperationsManager20076.1.7221.13	1
	googlebot#Googlespecificsettings	1
	Mozilla/4.0+(compatible;+MSIE+5.01;+Windows+NT+5.0)+RPT-HTTPClient/0.3-3E	1
	Steeler	1
	spider14.yandex.ru	1
	Mj4	1
	LOOQ	1
	Nutch-1.0-dev	1
	OnlineDomainTools-OnlineWebsiteLinkChecker/1.2	1
	Inkomi	1
	portal-crawler	1
	//my.nosto.com/tagging)	1
	ClockworkDataVault	1
	ThingFetcher	1
	fbot	1
	googlebot.com	1
	zumBot	1
	WebMechanic	1
	Gigabot	1
	Virusdiecrawler/2.1	1
	CombineSystem	1
	RoboFox	1
	Flock	1
	girafa	1
	bingbot/	1
	FlightDeckReportsBot	1
	SemrushBot/0.99~bl	1
	awcheckBot	1
	INGRID	1
	dotsemantic	1
	seznam.cz	1
	GriffinBot	1
	phpSiteCheck	1
	GoogleMobile	1
	SimmanyRobotVer1.0	1
	//www.vocus.com/vnhs.html)	1
	//www.wrensoft.com/zoom/support/useragent.html	1
	influencebo	1
	gamsup.ru	1
	NING	1
	PopularIconoclast	1
	Jooble-bot	1
	mediawords	1
	BizzInformation	1
	Tip-ExRobot	1
	URLSpiderPro	1
	WebsiteQuester–www.asona.org	1
	Scout.Vortex	1
	msngbot	1
	OmniExplorer_Bot/3.	1
	Java/1.6.0_23	1
	//help.naver.com/customer_webtxt_02.jsp)	1
	SecurityResearch.bot	1
	iViaPageFetcher	1
	DirBuster-0.12	1
	dahoms	1
	Yandexot	1
	netEstateNECrawl	1
	kgbody/2.0	1
	Nomad	1
	SpiderTraficDublu	1
	adsbot-Google	1
	justvisualbot	1
	Mozbot	1
	DataSift	1
	Clearware\web\browser	1
	go.mail.ru	1
	ShagSeeker	1
	NIF/1.1	1
	SogouWebspider	1
	//www.semantissimo.de/	1
	crawl-13.cuill.com	1
	compatibleZyBorg	1
	ParchBot	1
	OvalEarthBot	1
	SlurpBot	1
	RuLeS	1
	staticICEbot	1
	BackRub/.	1
	Yabnex	1
	puf/1.0.0	1
	//www.useragentstring.com/Yahoo!%20Slurp_id_75.php	1
	heritrix	1
	AntyStatyczny	1
	PackRat	1
	MJ12bot(Majestic)	1
	BSpider/1.0libwww-perl/0.40	1
	//ahrefs.com/	1
	//www.feedly.com/fetcher.html	1
	//sitecheck.internetseer.com/>	1
	cn_com_viewer	1
	AWSbot	1
	Googlebot\	1
	Bingbot/2.0	1
	IconSurf	1
	TACHBlackWidow	1
	Scrapy/1.1.2	1
	CloudFront	1
	vebidoobot-Image	1
	Metaspinner	1
	LargeSmall\Crawler	1
	Operations	1
	212.100.254.105	1
	worldwebheritage	1
	aranea#roguebot,botchedpythoncrawler	1
	OvalEarthBot/2.2.0	1
	mj4	1
	SynapticSearch	1
	AnzwersCrawl	1
	West\Wind\Internet\Protocols	1
	updownerbot	1
	AddCatalog/2.1	1
	HenriLeRobotMirago	1
	Rogerbot#AddedbyAB-MozCrawler	1
	sudo	1
	TITAN	1
	SocraRobot	1
	VegeBot#	1
	Duckduckgo	1
	ResearchBot	1
	WinHttp.WinHttpRequest.5	1
	Zumbot	1
	185.5.53.94	1
	RobotdeGoogle	1
	[Ww]eb[Bb]andit	1
	BigliSEO	1
	LightningDownload/1.0.1	1
	Casterly	1
	Dispatch	1
	Speedy	1
	TheWebWombat	1
	yodaobot/1.0	1
	Yandex#	1
	Yahoo-mmcrawler	1
	snap.combetacrawler	1
	vscooter	1
	flexum	1
	DomainSigmaCrawler/0.1	1
	Java/1.6.0_31	1
	Adsbot-Go	1
	PNWalker	1
	adsbot	1
	007ac9.net	1
	Wget/*	1
	Presto/2.6.30Version/10.62	1
	TinEye-bot/0.61	1
	HolmesBot	1
	datagnion	1
	AcunetixSecurityScanner	1
	Linkidator	1
	* #AllBots	1
	//moz.com/researchtools/ose/dotbot,notmozilla!	1
	applenewsbot	1
	TeomaTechnologies	1
	NewsGatorOnline	1
	93.104.209.59	1
	ArchiveTeam	1
	Qippobot	1
	dotobot	1
	Filpboard	1
	SEMRushbot	1
	Putinspider	1
	37.9.58.0/127	1
	BINGBot	1
	PGPKeyAgent	1
	SLCrawler	1
	Mediaparteners-Google*	1
	MoCollege1.9	1
	185.25.148.240	1
	som-gsa-crawler-two	1
	ia_archiver	1
	Mozzila	1
	Slurp	1
	findLinks	1
	HouxouCrawler/Nutch-0.9	1
	Mozilla/5.0(Linux;U;Android2.3.6;en-us;MB865Build/5.5.1-175_EDMR1.25)AppleWebKit/533.1(KHTML,likeGecko)Version/4.0	1
	Java/1.5.0_01	1
	CRAZYWEBCRAWLER0.9.1	1
	HyperCrawl/0.2	1
	InsightsCollector/0.1beta	1
	Whitehat	1
	LinkSpammer	1
	BLP_bbot\/0.1	1
	CloudServerMarketSpider/1.0	1
	python-requests/1.2.3CPython/2.7.4Linux/3.8.11-ec2	1
	BorderManager3.0	1
	Mediapartners-Ask	1
	64.94.186.110	1
	altavista	1
	HttpComponents/1.1	1
	URLmetriques	1
	Semrushbot-SI	1
	/lists/	1
	GoogleBot-Image	1
	Moreoverbot/5.1	1
	SiteSnagger	1
	html_analyzer	1
	ZumBot/1.0	1
	eSyndiCat	1
	googlebot*	1
	Xenu.s	1
	Googlebot/2.X	1
	Wells\Search\II	1
	UdmSearch	1
	*-Google	1
	EbuzzingFeedBot1.0	1
	xMind	1
	qsdb	1
	MSNBot-News	1
	Mozilla/4.0(compatible;MSIE8.0;WindowsNT5.1;Trident/4.0;.NETCLR1.1.4322;.NETCLR2.0.50727;.NETCLR3.0.04506.30;	1
	RU_Bot/2.0	1
	libwww-perl/5.814	1
	rexyobot	1
	Scarlet	1
	Acme.Spider	1
	Unknown	1
	Bot*	1
	conceptbot	1
	ShopWiki*	1
	bingbox	1
	eSyndiCat+Bot	1
	Akamai_Site_Analyzer	1
	AdIdXBot	1
	Liferea	1
	Cowbot	1
	GuestbookAutoSubmitter	1
	Baiduspider2	1
	page_verifier	1
	80LEGS.com	1
	umd-gsa-crawler	1
	msn-media	1
	net-profits.xyz	1
	*Disallow/wwwboard/Disallow/wwwboard/messages/	1
	ScreamingFrogSEOSpider/5.1	1
	WebCopierDanilo	1
	WebBug	1
	Vegebot	1
	InfoseekSideWinder	1
	duke-crawler	1
	HtTrack	1
	//www.omni-explorer.com)	1
	psbot	1
	SearchProcess	1
	GoogleWebPreview	1
	Botv	1
	Webmole	1
	sukibot_heritrix/3.1.1	1
	WebReaperv9.7–www.webreaper.net	1
	NetinfoBot/1.0/Nutch-0.9	1
	Twitterfeedv3	1
	с.новым.годом.рф	1
	ThumbnailCZrobot	1
	Nutch	1
	NameOfAgent(CMSSpider)	1
	//tab.search.daum.net/aboutWebSearch.html)Daumoa/3.0	1
	Яндекс	1
	YokleBot	1
	SpokeSpider	1
	Web-sniffer	1
	woobot/1.1	1
	UNIONSELECT	1
	WebBandit/1.0	1
	Digger	1
	RavenSearch	1
	DKIMRepBot	1
	WordPress	1
	Reedah	1
	Northwestern-Search	1
	daum	1
	ConveraBot	1
	CorenSearchBot	1
	A1SitemapGenerator*	1
	SalesIntelligent	1
	zibb	1
	PuxaRapido	1
	lechenie	1
	SystemCenterOperationsManager2007	1
	BorderManager	1
	LocalcomBot/1.3.0	1
	PageAnalyzer/1.5	1
	zgrab/0.x	1
	OmniExplorer_Bot/2.	1
	archiver	1
	Mwd.Search	1
	Teoma	1
	ShablastBot1.0	1
	Presto/2.2.15	1
	FeedChecker	1
	firstdirectory-bot	1
	virus_detector	1
	Java/1.6.0_34	1
	Mail.Ru/2.0	1
	Verticrawl	1
	*entriesfortheAdsBot-GoogleandforGooglebot	1
	MetaMoJiCrawler	1
	NASASearch	1
	ejabat.google.com	1
	python-requests/1.2.3CPython/2.7.2+Linux/3.0.0-16-virtual	1
	Teleport	1
	Eurosoft-Bot	1
	TosCrawler/Nutch-1.6	1
	008 #www.80legs.comcrawler	1
	WeBoX/0.97	1
	PShop-Nutch\/Nutch-1.1	1
	Pockey-GetHTML\/4.11.6	1
	Online24-Bot	1
	ask	1
	Bingboot	1
	goku	1
	HRCrawler	1
	NetLyzerFastProbe	1
	Statastico	1
	Orbsearch/1.0	1
	GoogleSmartphone	1
	Muninn	1
	//fulltext.sblog.cz/robot/)	1
	sputnik	1
	BNSBot/1.0	1
	Helix	1
	nutch-solr-integration/Nutch-1.2	1
	4webmasters.org	1
	MyIPTest	1
	Spinne	1
	ELFINBOT	1
	sai-crawler	1
	SaSSearch	1
	viview.inspsearch.com	1
	40	1
	Webcopier	1
	fess	1
	HometownSpiderPro	1
	NHSEWebForager	1
	WhoWhereRobot	1
	Yahoo-MMCrawler/3.x(mmsdashmmcrawlerdashsupportatyahoodashincdotcom)	1
	VKRobot	1
	Mozilla/5.0(compatible;seo-audit-check-bot/1.0)	1
	P.Arthur\1.1	1
	ZixyBot	1
	cutestat	1
	twenga2.com	1
	RobbietheRobot	1
	Linkapedia	1
	Busiverse	1
	proxy.aol.com	1
	Mediapartners-GoogleMediapartners(Googlebot)	1
	entireweb	1
	http_get	1
	Bot/	1
	Msn	1
	seplinkbox	1
	live.com	1
	Lipperhey-Kaus-Australis/5.0	1
	DAUMOA	1
	Ultraseek	1
	IntrafindBot(intrafind.de)	1
	WebZinger	1
	AntsArmy	1
	OracleSecureEnterpriseSearch	1
	Baiduspider #Baidu	1
	ROBOT_MEDIAVIDEOUNO	1
	Shai\'Hulud	1
	SCPS(Enterprise;T3-KUJ2NYX3RWSGG;[email protected])	1
	Java/1.5.0	1
	Pinterest/0.2	1
	Amber	1
	msnbot-Products*	1
	IRLbot	1
	68.64.172.29	1
	Baiduspider-anúncios	1
	enllepuntocom/Nutch-1.9	1
	SiteTech-Rover	1
	UniversalFeedParser	1
	//help.yandex.com/search/robots/agent.xml	1
	Gazz	1
	blogbridge	1
	Mercury	1
	Mozilla/4.0(compatible;Arachmo)	1
	MyEngines-Bot	1
	AdidxBot	1
	*	1
	spider00.logika.net	1
	Nutch-1.2	1
	treato-bot	1
	iajaBot	1
	JaydeCrawler	1
	Gooblogsearch/2.0	1
	my6sense	1
	//help.yahoo.com/kb/search/SLN22600.html?impressions=true	1
	sape.bot	1
	*	1
	MegaIndex.com	1
	Googlebotdoesthesame,soitneedsitsownsectiontoo.	1
	SkyGrid	1
	bingbot?	1
	SafeDNSsearchbot/Nutch-1.9	1
	gsa-crawler(Enterprise;M2-PQTGDPEKCA6JT;[email protected])	1
	1Noonbot1.0	1
	Wget/1.4.0	1
	BingPreview#Microsoft	1
	isiteamspider	1
	dlvr.it	1
	//user-agents.me/crawler/yisouspider	1
	Cloudtrawl	1
	SiteCon	1
	larbin_indexer	1
	DaviesBot	1
	LiteFinder/1.0	1
	Findxbot/1.0	1
	Webcorpuscrawler	1
	Googlebot-Image(Googlebot)	1
	Java1.4.0_02	1
	RHCS	1
	XenuÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂ¢ÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂs	1
	imparser09.yandex.ru	1
	WinkBot/1.0	1
	AcoonRobot	1
	SiteExplorer/1.0	1
	mandalay	1
	gsa-crawler+(Enterprise;+T3-HKV9FY7X3YWBK;[email protected])	1
	Cugillionbot	1
	LarbinWebCrawler	1
	NostoCrawlerBot	1
	DRWAdminContact	1
	Python-httplib2/0.7.7(gzip)	1
	Java1.4.0	1
	FetchRover	1
	Rapleafbot	1
	AcoonBot/4.11.1	1
	gigablast.com	1
	BuiBui-Bot/1.0	1
	python-requests/1.1.0CPython/2.6.8Linux/3.4.48-45.46.amzn1.x86_64	1
	choopa.net	1
	oBot/	1
	Poppi	1
	duggmirror	1
	Sogoudevelopspider	1
	5.62.8.0/255	1
	Java/1.4.2_06	1
	googlebot	1
	Shopit	1
	mail\.ru	1
	www.petitsage.frsitedetector0.4	1
	MFGPagesBot	1
	Medubot	1
	PHPbot	1
	NutchOrg	1
	siteuptime	1
	GetURL	1
	exactseek-pagereaper	1
	iVia	1
	majestic#	1
	zumbot	1
	Cuill	1
	Java/1.6.0_06	1
	Image2play/0.1	1
	BLaaG	1
	Contextured	1
	WebsiteQuester–www.esalesbiz.com/extra/	1
	teomaagent1	1
	BrightEdgeCrawler	1
	Megaindex	1
	//www.brandwatch.com/how-it-works/	1
	Gaisbot	1
	WebmasterWorldvbBot	1
	//www.snap.com)	1
	//www.80legs.com/webcrawler.html	1
	Revolution\	1
	123SubmitPRO	1
	MoodleBot	1
	//www.iplexx.at)	1
	google*	1
	AtraxSolutions	1
	Python-urllib/2.4	1
	Twengabot-2.0	1
	chushou	1
	downloader	1
	MyWireServiceBot	1
	SEOMoz	1
	StackRambler/2.0(MSIEincompatible	1
	Yahoo-Newscrawler*	1
	servage.net	1
	//www.omni-explorer.com)JobsCrawler	1
	SitemapGenerator	1
	webarchiv	1
	//www.weiterbildungsprofis.de/sitemap_index.xml	1
	crawlerboy	1
	Googlebot	1
	aibot	1
	cybo.com	1
	telnet	1
	InterNaetBoten	1
	StudioFACASearch	1
	nlcrawler	1
	Faxobot	1
	IBrowse	1
	BLEXBot*	1
	Kyoto-Crawler	1
	MultiText	1
	ia_archiver	1
	EuripBot/2.0	1
	OpenfosBot	1
	GetRight/3.x.x	1
	Java/1.6.0_32	1
	//www.FeedBurner.com)	1
	Msnbot-	1
	archive.bibalex.org_bot	1
	nu_tch-princeton/Nu_tch-1.0-dev	1
	BLEXBot#	1
	^.statdom.ru.	1
	RukiCrawler	1
	SuperGet	1
	RedCarpet/1.3	1
	CareerBot/1.1	1
	Mozilla/5.0(compatible;MSIE9.0;WindowsNT6.1;Win64;x64;Trident/5.0)	1
	Yandex/1.0	1
	Pockey-GetHTML	1
	NmapScriptingEngine	1
	publiclibraryarchive.org/1.0	1
	WeLikeLinks	1
	5.62.24.0/255	1
	Presto/2.9.168	1
	abotv1.0	1
	Popdexter	1
	ahrefsBot	1
	YandexCatalog/3.0	1
	Webtrends	1
	Webclipping	1
	ScreamingFrogSEOSpider/5.0	1
	Mozilla/5.0(compatible;iaskspider/1.0;MSIE6.0)	1
	ADmantXPlatformSemanticAnalyzer-ADmantXInc.-www.admantx.com-support@admantx.com	1
	buttons-for-website.com	1
	ThePeregrinator	1
	UpikBot	1
	Mozilla/3.01C-PBWF-ip3000.com-crawler	1
	appie1.1(www.walhello.com)	1
	VocusBot0.4	1
	/it/	1
	PulseCrawler	1
	Seznam	1
	DownloadNinja2.0	1
	Liberate	1
	Nutch-0.9	1
	Yahoo-Yahoo-YSM	1
	Hobbit_bbtest-net	1
	BWC\/0.3	1
	ShablastBot	1
	BOIA-Accessibility-Agent/PR1.0	1
	/cabinet	1
	Changedetection	1
	TurnitinBot/1	1
	MarcSpider	1
	Space\Bison	1
	HRCrawler/2.0	1
	vBSEO	1
	SEOENGBot	1
	FindexaCrawler	1
	Twitterbot/0.1	1
	Ahoy!TheHomepageFinder	1
	mnsbot	1
	Grabber(SDSC)	1
	ntentbot	1
	Mozilla/4.0(compatible;MSIE7.0;WindowsNT6.1;WOW64;Trident/5.0;SLCC2;.NETCLR2.0.50727;.NETCLR3.5.30729;.NETCLR	1
	http_load\29jun2005	1
	Slurp#Yahoo!#CUSTOM	1
	mycrowl/Nutch-1.9	1
	Yahoo-MMCrawler*\	1
	WiredDigital	1
	askbot	1
	bot/	1
	SiteCheck	1
	RankurBot	1
	RebelMouse	1
	ProCogBot	1
	AURESYS	1
	buibui[at]dadapro[dot]com)	1
	FollowSite	1
	Mail.ru	1
	ecoresearch	1
	ecoresearch/0.9	1
	Domnutch	1
	MacInroy	1
	ReferrerKarma/2.0	1
	ichiro/mobile	1
	Arachnida	1
	msnbotbingbot	1
	MJ12bot/v1.4.7	1
	FunnelWeb	1
	JoeBot	1
	ReplazBot	1
	LinkMasterBot	1
	CoinCornerBot	1
	TeleportPro	1
	TheWebfootRobot	1
	BigmirSpider	1
	bjtelecom.net	1
	/race-information/photo-video	1
	MSNbot	1
	/MagNewxyz_bkp/	1
	Turnitin	1
	Mozilla/4.0+(compatible;+MSIE+6.0;+Windows+98;+Win+9x+4.90;+sureseeker.com;+.NET+CLR+1.1.4322)	1
	inktomisearch	1
	sohu-search	1
	QuepasaCreep	1
	bloodhound	1
	spiderlytics	1
	BTWebClient	1
	sherlock	1
	EPiServerLinkChecker	1
	website-analyzer.info	1
	Java/1.6.0_30	1
	Moni	1
	AffectvRobot	1
	SemrushBot-Desktop	1
	Purebot*	1
	Scopiacrawler	1
	OmniExplorerBot	1
	ClickMeter	1
	psbot # psbot(Picsearch)usedbymsnbot-	1
	CloudFlare-AlwaysOnline/1.0	1
	Sphere(www.sphere.com)scoutatspheredotcom	1
	JenkersBot/1.0	1
	WIRE	1
	Offline	1
	IA	1
	jeeves	1
	FlipboardProxy/1.1	1
	timboBot	1
	som-gsa-crawler-one	1
	//deepcrawl.co.uk/bot.html)	1
	BlogRangerCrawler	1
	mbot	1
	Yelpspider	1
	Yanga\WorldSearch\Bot	1
	APHPscript	1
	gsa-crawler#CUSTOM	1
	UnivOfOuluNutch	1
	UniLeipzigASV	1
	Hostnoc	1
	DigimarcMarcspider/CGI	1
	DownLoadExpress	1
	msnbot?	1
	Java/1.6.0_05	1
	"Yahoo!Slurp"	1
	tomsk.ru	1
	Openstat	1
	Molbsy	1
	Snooper	1
	SiteSearcher	1
	archive_crawler	1
	//service.kskrk.ru/sitemap.xml	1
	Sitemapdoc	1
	Thumbtack	1
	Conceptbot	1
	GCreep	1
	Mail.RU/2.0	1
	TrustedSite	1
	hivaBot	1
	Slurp	1
	TANNER\Spider/Nutch-1.1	1
	Wget\/1.8.2	1
	//www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)	1
	Faraday	1
	Sleek	1
	sfr	1
	btcrawler	1
	Art-Online.com\0.9(Beta)	1
	GenericBot	1
	gsa-crawler-allied	1
	webcheck	1
	btbot	1
	EmailCollector/1.0	1
	atomicbot	1
	Mozilla/5.0(X11;U;Linux;en-US)AppleWebKit/532.4(KHTML,likeGecko)Qt/4.6.3Safari/532.4	1
	GoogleFeedFetcher	1
	Rexyobot/1.12	1
	baiduSpider	1
	AntiVirusPro	1
	Prismatic	1
	78.157.215.0/127	1
	*Bot	1
	nys-crawler	1
	Baiduspider#blocked2/1/2012-Chinese	1
	CrawlerBoy+Pinpoint.com	1
	ubermetrics	1
	bingbot#Bing#CUSTOM	1
	ABCdatosBotLink	1
	spiderBot	1
	ArchiveBot	1
	sohu	1
	Cision	1
	hunchan	1
	Java/1.6.0_27	1
	www.creativeresearchindexer.com	1
	TheWebMoose	1
	Linkdexbot/2.2	1
	HubSpotLinksCrawler	1
	msnbot-mm	1
	//www.sogou.com/docs/help/webmasters.htm#07	1
	SkimBot/1.0	1
	proximic*	1
	Xenu's	1
	StubHub	1
	Snoopy\v1.2.4	1
	dj-research	1
	NetResearchServer/4.0	1
	hotpage.fr	1
	JCE	1
	slurp/2.0	1
	Yeti\/1.0	1
	WmailWolf	1
	Baiduspider#appliestoBaiduspiderrobots	1
	//www.omni-explorer.com)CarsCrawler	1
	EDI/1.6.0(Edacious&IntelligentWebCrawler)	1
	iZSearch.com	1
	Mailbot	1
	SEOstats2.1.0	1
	PicSearch	1
	MwdSearch	1
	LocalcomBot	1
	woriobot+	1
	Convera	1
	INKTOMISEARCH.COM;inktomi.com	1
	WordPress\	1
	JBotJavaWebRobot	1
	Addurlrobot	1
	Teezir	1
	DomainsDB*	1
	SIMgroep	1
	SiteSucker/2.3.6	1
	lwp-trivial\/1.38	1
	GooglebotNews	1
	Ramblerbot	1
	Ruky-Roboter	1
	Blogcensus	1
	stq_bot	1
	Magpie	1
	woriobotsupport	1
	integrity	1
	bingbot#Microsoft	1
	SoGou	1
	NutchSpider/Nutch-1.0-spb	1
	InterNaetBoten/0.99	1
	BecomeBot\	1
	Skymob.com	1
	reenterbot	1
	Linkdexbot/2.0	1
	Gecko/2008032620	1
	/oneshot/	1
	//help.naver.com/robots/</a>)	1
	"TinEye"	1
	//www.entireweb.com/	1
	HostHarvest/0.4.28	1
	hvvc	1
	JumpStation	1
	GetterroboPlusPuu	1
	MFCFoundationClassLibrary4.0	1
	cherrypicker	1
	baffinbot02	1
	Java/1.4.2_01	1
	phpSiteCheck1.0	1
	sna	1
	shoppertom	1
	KRetrieve	1
	XerkaWebBot	1
	www.wesee.com	1
	BlackboardSafeassign	1
	Yahoo!Inc.	1
	GoForIt	1
	static.reverse.softlayer.com	1
	cuill	1
	semanticdiscovery/0	1
	/fancy/	1
	ichiro/mobilegoo	1
	wubaiyiSpider	1
	Clewwa-Bot/Nutch-1.0	1
	//www.discoveryengine.com	1
	//www.cmscrawler.com	1
	OptimizationCrawler/0.2	1
	Monitis	1
	Haste	1
	JoBoJavaWebRobot	1
	InktomiSearch	1
	nys-qa-crawler	1
	PostRank/2.0	1
	Robbie	1
	//www.authoritativeweb.com/crawl)	1
	IAB	1
	GetProxi.es-bot/1.1	1
	toplink24.de	1
	WebBanditWebSpider	1
	Mozilla/4.08[en](Win98;U;Nav)	1
	Aghaven\/Nutch-1.2	1
	NPBot	1
	UptimeBot	1
	VocusBot*	1
	Siteluxbot	1
	SNKScreenshotBot	1
	samhsagsa-crawler	1
	[email protected]	1
	BWAgent	1
	mfibot	1
	OMGCrawler1.0	1
	VMBot	1
	Meltawer	1
	Ahrefs-Bot/4.0	1
	xenu	1
	Java1.3.0	1
	SSL-Crawler	1
	Mozilla/5.0(X11;Linuxx86_64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/52.0.2743.116Safari/537.36	1
	www.cuil.com	1
	HÃ¤mÃ¤hÃ¤kki	1
	Mediapartners-Google*	1
	"Yahoo!Slurp/3.0"	1
	nutraspace	1
	TapuzBot	1
	MSNBot#MSN#CUSTOM	1
	imspider	1
	SG-Scout	1
	augurnfind	1
	DFBot1.0	1
	LC-Crawler	1
	VoilaBotCollectorBETA0.1	1
	Arale	1
	primo	1
	178.79.147.120	1
	libWeb\/clsHTTP	1
	WordPress\/2.6.2	1
	MJ12bot#stopMajestic12UKbot01-31-12	1
	MindSpider	1
	VengaBot	1
	Iphone	1
	agadine	1
	scrapinghub	1
	Begun	1
	DonQuichote/1.2a-unixMode/Get	1
	Java/1.6.0_39	1
	GoogleInc.	1
	DownloadExpress	1
	DesertRealmSpider	1
	SpiderView(tm)	1
	//www.alltheweb.com	1
	WSB+WebCrawler+V1.0+(Beta),[email protected]	1
	PerMa	1
	Prlog	1
	BruinBot	1
	BoxSeaBot	1
	gammaSpider,FocusedCrawler	1
	logo.gifCrawler	1
	WebCopierv5.4	1
	crawlerboy+pinpoint.com	1
	Nutch-1.7	1
	Gigabot/2.0/gigablast.com/spider.html	1
	TECNOSEEK	1
	whatUseekWinona	1
	*googlebot	1
	Googlebot-Video(Googlebot)	1
	Majestic-SEO	1
	Bot-	1
	redditbot	1
	PDFBot	1
	TeleportPro/1	1
	growerideas	1
	BusinessBot	1
	gulper	1
	gsa-crawler-intranet-amsterdam	1
	Y!J-BRO/YFSJcrawler	1
	probethenet	1
	Java/1.4.2_10	1
	Webscout	1
	WebCopier+v4.3	1
	FavIconizer	1
	//www.baidu.com/search/spider.htm	1
	Mozilla/3.0(compatible)	1
	Robofox	1
	libwww-perl/5.47	1
	GulperBot	1
	InfoseekSidewinder	1
	Valkyrie	1
	Page2RSS	1
	UltraSpider3000	1
	Acoon\	1
	T8Abot/v0.0.7-beta	1
	web-revenue.xyz	1
	Krugle	1
	/autorenimages/	1
	Semrushbot-SA	1
	MS_FrontPage	1
	Yahoo!'sWebCrawler	1
	WebNews+Arianna	1
	[email protected]	1
	iaarchiver	1
	wgetrc	1
	HTMLgobble	1
	//www.altavista.com	1
	Blah	1
	CopyscapePlagiarismChecker-DuplicateContentDetectionSoftware	1
	GagaRobot	1
	12.0)Gecko/20100101Firefox/12.0	1
	360spider-image	1
	wscheck.com/1.0.0	1
	a.pr-cy.ru	1
	VSE/.	1
	A6	1
	archive.is	1
	msnbot/1.0-MM	1
	ie_crawler	1
	SpotBot	1
	GalaBuzz	1
	Griffon	1
	//www.musdetal.ru/sitemap_images.xml	1
	Ahrefs-Bot/5.0	1
	twiceler	1
	Shim-Crawler	1
	ICF-VNUCHGOOGLE	1
	ScreamingFrogSEOSpider/3.1	1
	INGRID/0.1	1
	Shareaholicbot	1
	Esther	1
	SemanticBot	1
	Jakartacommons-httpclient	1
	Gulliver/1.3	1
	POGS/2.0	1
	fetch	1
	MJ12bot#appliestoMJ12robots	1
	DownloadAccelerator	1
	Berry	1
	larbin_2.6.2	1
	Y!J*	1
	Scrapy/1.0.3	1
	//www.informedusa.com/t/phantom7.15.html	1
	Coccoc	1
	KIT-Paperball	1
	CuriousGeorge-www.analyticsseo.com/crawler	1
	PageAnalyzerv4.0	1
	Feedly	1
	AnswerBus	1
	Contiki	1
	HiddenMarket	1
	Yanga+WorldSearch+Bot+v1.1	1
	uMBot-LN/1.0	1
	SygolBot	1
	bjtelcom	1
	Simplecrawler	1
	PageWeight	1
	UndertheRainbow2.2	1
	Arquivo	1
	DomainRe-Animator	1
	SimplePie/1.2	1
	Clushbot	1
	Fast-WebCrawler	1
	DirectHitGrabber	1
	CrawlerBoy	1
	digimarc	1
	arianna.libero.it	1
	Alexbot	1
	Google*	1
	JAVA	1
	*Yandex	1
	GetRight/4.5xx	1
	LinexBot	1
	Searchspider	1
	URLmetriche	1
	SOASearch	1
	MagusBot	1
	//support.addthis.com/)	1
	web-archive-net.com	1
	Nachobot	1
	newspaper	1
	Bloglines	1
	hostmonitor	1
	docomo	1
	SqwidgeBot	1
	Kalooga	1
	EMCSpider	1
	photon	1
	SBIder	1
	Imspider	1
	Pixray-Seeker/2.0	1
	googlebot-third	1
	findlinks/2.1.5	1
	JakartaCommons-HttpClient/3.1	1
	//yacy.net/bot.html	1
	sukibot	1
	Georgios	1
	SpiderlineCrawler	1
	Screaming	1
	libhtt	1
	aiHitBot/2.9	1
	OpenfindRobot	1
	PopScreenBot	1
	awcheck	1
	Nigma	1
	BackRub/.	1
	bzBot	1
	MSNBot-Products	1
	mnoGoSearchsearchenginesoftware	1
	quintura-crw	1
	MuckRack	1
	BazQux/2.4	1
	violabot	1
	spbot/4.2.0	1
	python-requests/2.10.0	1
	webcrawler.com	1
	itsapic.com_crawler	1
	ARIADNE	1
	Pcore-HTTPpossiblyGoogleBot?	1
	Ezooms*	1
	Statastico/4.0	1
	IncyWincy/1.0b1	1
	Thumbnail.CZrobot	1
	Mediapartners-Yahoo	1
	BlogramCrawler/	1
	cIeNcIaFiCcIoN.nEt	1
	[email protected]	1
	EARTHCOM.info	1
	Java/1.8.0_121	1
	Java/1.6.0_02	1
	QlikView	1
	SheerBoredom.Experimental.Robot/0.2	1
	w3cbot	1
	statdom	1
	Golem	1
	Crawl	1
	disco/nutch-1.0-dev	1
	10	1
	share-buttons.xyz	1
	SafariBookmarkChecker	1
	gsa-cawler	1
	Slurp<BR>	1
	AddSearchBot	1
	EventMachineHttpClient	1
	//www.sync2it.com/susie	1
	Nozilla	1
	Sygol	1
	JaydeNicheBot	1
	iqonbot	1
	WeRelateBot	1
	baconsbot	1
	Justdialbot/1.0	1
	mtbot	1
	URLCheck	1
	spider26.picsearch.com	1
	freefind	1
	HappyFunBot/1.1	1
	CatchBot/1.0	1
	bright.netcachingrobot	1
	SiteImprove	1
	TV33_Mercator	1
	OracleUlraSearch	1
	md5sum\x22	1
	Giant/1.0	1
	//www.miragorobot.com/scripts/mrinfo.asp)	1
	pagespeed	1
	aipbot	1
	YahooPipes	1
	redditbot/1.0	1
	Domnutch-Bot/Nutch-1.0	1
	WBSearchBot/1.1	1
	Calif	1
	Baiduspider-anÃÂºncios	1
	SQUID_configured_as_described_at_/help/faq/cache	1
	Slurpchina	1
	BLEX	1
	peoplecheck	1
	CMRadar	1
	FelixIDE	1
	istellabot/t.1.13	1
	gsa-crawler2	1
	//northernlight.com/)	1
	HagenDerRobotMirago	1
	accoonabot	1
	gsa-crawler-unimi	1
	Exabot-Test	1
	NP	1
	bleeding-sucker-bot	1
	CRAZYWEBCRAWLER0.9.7	1
	Lwebis	1
	SsearchCrawler	1
	Grapnel/0.01Experiment	1
	JCrawler	1
	Yandexbot\|SemanticScholarBot\|SemrushBot\|Baiduspider\|Sogou\|Slurp	1
	COMPANYNAMECrawler	1
	HatenaAntenna #(BAD)UnknownURL	1
	FAST-WebCrawler	1
	Yandex#willuseallYandexrobots	1
	Instapaper	1
	Lachesis	1
	Cassandra	1
	Sven	1
	ExactSearch	1
	yahoo-Blogs	1
	HTTrackoff-linebrowser	1
	MeMoNewsBot	1
	CyberSpyderLinkTest	1
	PositronicBot	1
	SitemapBot	1
	suchmaschinenoptimierung.de	1
	TheImageScapeRobot	1
	MicrosoftURLControlÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂ¢ÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂÃÂ5.01.4511	1
	Googlebotsmartphone	1
	AskTbUT2V5	1
	keyword	1
	Shagseeker	1
	Speedy#appliestoSpeedyrobots	1
	ccbot#stopcommoncrawlbotfromcrawlingthesite.Itwascausingerrors.	1
	//www.biz360.com	1
	Sogou#appliestoSogourobots	1
	YandexDirect/3.0	1
	SemrushBot#	1
	TweetedTimesBot/1.0	1
	StudioFACA	1
	ia_archiver #disblealexapreviewandwaybackmachine	1
	HI(HTMLIndex)Search	1
	PageBoy	1
	PHP-SoHosted	1
	Mozilla/4.0(compatible;MSIE8.0;WindowsNT6.1;WOW64;Trident/4.0;SLCC2;.NETCLR2.0.50727;.NETCLR3.5.30729;.NETCLR	1
	Java/1.4.2_09	1
	Faradayv0.8.8	1
	BlacklinkCrawler	1
	WebCore/Roots	1
	OpenTextIndexRobot	1
	UnknownRobot*	1
	SolomonoBot/1.05	1
	zelist	1
	//www.thefind.com/main/CrawlerFAQs.fhtml)	1
	Acoona-AI-Agent/1.1.2+	1
	Go1.1packagehttp	1
	Jobboerse.com	1
	EccolaBot	1
	Baidu-spider	1
	Sphere	1
	backlinktest	1
	Pcore-HTTP/v0.25.0	1
	Flamingo	1
	Webscout/1.0	1
	//www.omni-explorer.com)WorldIndexer	1
	EbiNess	1
	TheNWIRobot	1
	PicoSearch/1.0	1
	MEGAUPLOAD	1
	//support.alexa.com/hc/en-us/articles/200450194	1
	MSNBot/BingBot	1
	//64.5.245.11/faq/faq.html)	1
	OfflineExplorer	1
	[email protected]	1
	Teoma#(Ask/Teoma)	1
	eduCrawler	1
	TodoExpertosBot	1
	MJ12bot/v1.3.2	1
	Ggooglebot	1
	NutchBOA	1
	TAMU_CS_IRL_CRAWLER	1
	appie	1
	NaverBot	1
	LetsCrawl.com	1
	ASPseek/1.2.10	1
	MetaURIAPI/2.0	1
	Admantx	1
	ZeusLinkScout	1
	//www.google.com/bot.html)*	1
	CydralSpider	1
	MissaugaLocate1.0.0	1
	xmsnbot	1
	/wp-content/uploads/	1
	ValueClick	1
	FANGCrawl	1
	GigabotSiteSearch	1
	produkte24	1
	rdfbot/Nutch-1.0-dev	1
	Lickity_Split_Spider	1
	50	1
	oleanebot	1
	//www.nict.go.jp/en/univ-com/plan/crawl.html	1
	Becomebot	1
	Visvo	1
	larbin_2.6.2\[email protected]	1
	Java/1.6.0_37	1
	plaNETWORKBotSearch	1
	//www.omni-explorer.com)InternetCategorizer	1
	gsa-crawler-internet-amsterdam	1
	BlogPulse	1
	SEMrushBot-SA	1
	microsoftoffice	1
	//www.omni-explorer.com)InternetIndexer	1
	larbin	1
	IndustryProgram1.0.x	1
	BaiduImagespider+	1
	*=meansthatitaffectsallspiders	1
	Feedly/1.0	1
	[email protected]	1
	WebCookies	1
	Borg-Bot	1
	DWCP(Dridus'WebCatalogingProject)	1
	savetheworldheritage	1
	Daumoa-image	1
	SwishSpider	1
	kalooga\/KaloogaBot	1
	xrss	1
	gonzo	1
	Pinterest/	1
	//moz.com/help/pro/what-is-rogerbot-	1
	AhrefsBot #ahrefsCrawler	1
	Cuil	1
	qingdao	1
	MicrosoftURLControl.5.01.4511	1
	OnlineDomainTools-OnlineWebsiteLinkChecker	1
	Y!J-DSC	1
	Sosospider#blocked2/1/2012-Chinese	1
	Youdao	1
	/promo/	1
	ShowXML	1
	BacaBeritaApp	1
	ScreamingFrogSEOSpider/2,55	1
	BotLink	1
	LiteFinder*	1
	SquigglebotBot	1
	DAUM	1
	ec_robot	1
	twitterFeed	1
	HamBot	1
	Xenu\Link\Sleuth	1

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment