Update spiders.txt

catalog/includes/spiders.txt

@@ -1,54 +1,95 @@
 $Id$
 almaden.ibm.com
-appie 1.1
-architext
-ask jeeves
-asterias2.0
-augurfind
+abachobot
+accoona-ai-agent
+addsugarspiderbot
+arachmo
 baiduspider
-bannana_bot
-bdcindexer
-crawler
-crawler@fast
-docomo
+becomebot
+beslistbot
+bimbot
+bingbot
+dataparksearch
+dotbot
+envolk[its]spider
+exabot
+fast enterprise crawler
 fast-webcrawler
-fluffy the spider
-frooglebot
-geobot
+furlbot
+fyberspider
+galaxybot
+gigabot
+girafabot
 googlebot
-gulliver
-henrythemiragorobot
+googlebot-image
+holmes
 ia_archiver
-infoseek
-kit_fireball
-lachesis
-lycos_spider
-mantraagent
-mercator
-moget/1.0
-muscatferret
-nationaldirectory-webspider
-naverrobot
-ncsa beta
+ichiro
+igdespyder
+irlbot
+l.webis
+larbin
+ldspider
+lexxebot
+linguee bot
+linkwalker
+lmspider
+lwp-trivial
+mabontland
+magpie-crawler
+mediapartners-google
+mj12bot
+mnogosearch
+mojeekbot
+moreoverbot
+morning paper
+msnbot
+mxbot
 netresearchserver
-ng/1.0
-osis-project
+netseer crawler
+newsgator
+ng-search
+nicebot
+noxtrumbot
+nutchcvs
+obot
+oozbot
+orangebot
 polybot
-pompos
-scooter
-seventwentyfour
-sidewinder
-sleek spider
-slurp/si
-[email protected]
-steeler/1.3
-szukacz
-t-h-u-n-d-e-r-s-t-o-n-e
+psbot
+pycurl
+scoutjet
+scrubby
+seekbot
+seochat::bot
+seznambot
+shim-crawler
+shopwiki
+shoula robot
+silk
+snappy
+sogou spider
+sosospider
+speedy spider
+stackrambler
+suggybot
+synoobot
 teoma
+thumbnail.cz robot
+tineye
 turnitinbot
-ultraseek
+twengabot
+urlfilebot
 vagabondo
-voilabot
-w3c_validator
-zao/0
-zyborg/1.0
+voltron
+websquash.com
+yacy
+yahoo! slurp
+yahooseeker
+yandexbot
+yandeximages
+yeti
+yodaobot
+youdaobot
+zealbot
+zyborg