diff --git a/catalog/includes/spiders.txt b/catalog/includes/spiders.txt index 11dd705dd..7a20e2b41 100644 --- a/catalog/includes/spiders.txt +++ b/catalog/includes/spiders.txt @@ -1,54 +1,95 @@ $Id$ almaden.ibm.com -appie 1.1 -architext -ask jeeves -asterias2.0 -augurfind +abachobot +accoona-ai-agent +addsugarspiderbot +arachmo baiduspider -bannana_bot -bdcindexer -crawler -crawler@fast -docomo +becomebot +beslistbot +bimbot +bingbot +dataparksearch +dotbot +envolk[its]spider +exabot +fast enterprise crawler fast-webcrawler -fluffy the spider -frooglebot -geobot +furlbot +fyberspider +galaxybot +gigabot +girafabot googlebot -gulliver -henrythemiragorobot +googlebot-image +holmes ia_archiver -infoseek -kit_fireball -lachesis -lycos_spider -mantraagent -mercator -moget/1.0 -muscatferret -nationaldirectory-webspider -naverrobot -ncsa beta +ichiro +igdespyder +irlbot +l.webis +larbin +ldspider +lexxebot +linguee bot +linkwalker +lmspider +lwp-trivial +mabontland +magpie-crawler +mediapartners-google +mj12bot +mnogosearch +mojeekbot +moreoverbot +morning paper +msnbot +mxbot netresearchserver -ng/1.0 -osis-project +netseer crawler +newsgator +ng-search +nicebot +noxtrumbot +nutchcvs +obot +oozbot +orangebot polybot -pompos -scooter -seventwentyfour -sidewinder -sleek spider -slurp/si -slurp@inktomi.com -steeler/1.3 -szukacz -t-h-u-n-d-e-r-s-t-o-n-e +psbot +pycurl +scoutjet +scrubby +seekbot +seochat::bot +seznambot +shim-crawler +shopwiki +shoula robot +silk +snappy +sogou spider +sosospider +speedy spider +stackrambler +suggybot +synoobot teoma +thumbnail.cz robot +tineye turnitinbot -ultraseek +twengabot +urlfilebot vagabondo -voilabot -w3c_validator -zao/0 -zyborg/1.0 +voltron +websquash.com +yacy +yahoo! slurp +yahooseeker +yandexbot +yandeximages +yeti +yodaobot +youdaobot +zealbot +zyborg