spider-rewrite.conf

RewriteCond %{HTTP_USER_AGENT} Googlebot [OR]
RewriteCond %{HTTP_USER_AGENT} ia_archiver [OR]
RewriteCond %{HTTP_USER_AGENT} sp.ask.com [OR]
RewriteCond %{HTTP_USER_AGENT} Jeeves/Teoma [OR]
RewriteCond %{HTTP_USER_AGENT} Mediapartners [OR]
RewriteCond %{HTTP_USER_AGENT} msnbot [OR]
RewriteCond %{HTTP_USER_AGENT} Slurp [OR]
RewriteCond %{HTTP_USER_AGENT} MSIECrawler [OR]
RewriteCond %{HTTP_USER_AGENT} www.voila.com [OR]
RewriteCond %{HTTP_USER_AGENT} vspider [OR]
RewriteCond %{HTTP_USER_AGENT} http://www.WISEnutbot.com [OR]
RewriteCond %{HTTP_USER_AGENT} heritrix [OR]
RewriteCond %{HTTP_USER_AGENT} CASANOVA [OR]
RewriteCond %{HTTP_USER_AGENT} sogou\sspider [OR]
RewriteCond %{HTTP_USER_AGENT} YahooSeeker [OR]
RewriteCond %{HTTP_USER_AGENT} WeRelateBot [OR]
RewriteCond %{HTTP_USER_AGENT} Zeusbot [OR]
RewriteCond %{HTTP_USER_AGENT} Entireweb [OR]
RewriteCond %{HTTP_USER_AGENT} BDFetch [OR]
RewriteCond %{HTTP_USER_AGENT} ConveraCrawler [OR]
RewriteCond %{HTTP_USER_AGENT} FDSE\srobot [OR]
RewriteCond %{HTTP_USER_AGENT} LinkWalker [OR]
RewriteCond %{HTTP_USER_AGENT} wwwster [OR]
RewriteCond %{HTTP_USER_AGENT} aipbot [OR]
RewriteCond %{HTTP_USER_AGENT} Vagabondo [OR]
RewriteCond %{HTTP_USER_AGENT} HTTrack [OR]
RewriteCond %{HTTP_USER_AGENT} larbin_ [OR]
RewriteCond %{HTTP_USER_AGENT} ZanranCrawler [OR]
RewriteCond %{HTTP_USER_AGENT} www.yacy.net