# Note: Robots that access TWiki.org at a high rate are
# automatically added to a blacklist and will not be able 
# to access content anymore. Download the TWiki package,
# it contains all TWiki.org pages you need to get started
# with your own TWiki.

User-agent: iecrawler
User-agent: ie_crawler
User-agent: e-SocietyRobot
Disallow: /

User-agent: msnbot
Crawl-delay: 120
Disallow: /cgi-bin/attach
Disallow: /cgi-bin/changes
Disallow: /cgi-bin/edit
Disallow: /cgi-bin/geturl
Disallow: /cgi-bin/installpasswd
Disallow: /cgi-bin/logon
Disallow: /cgi-bin/mailnotify
Disallow: /cgi-bin/manage
Disallow: /cgi-bin/oops
Disallow: /cgi-bin/passwd
Disallow: /cgi-bin/preview
Disallow: /cgi-bin/rdiff
Disallow: /cgi-bin/register
Disallow: /cgi-bin/rename
Disallow: /cgi-bin/save
Disallow: /cgi-bin/search
Disallow: /cgi-bin/statistics
Disallow: /cgi-bin/upload
Disallow: /cgi-bin/viewauth
Disallow: /cgi-bin/viewfile
Disallow: /cgi-bin/view/Sandbox
Disallow: /cgi-bin/view/*?

Disallow: /cgi-bin/survey
Disallow: /cgi-bin/twikicvs
Disallow: /cgi-bin/awstats/

Disallow: /cgi-bin/b/bin/

User-agent: Slurp
# Custom extension: (see http://help.yahoo.com/help/us/ysearch/slurp/slurp-03.html)
Crawl-delay: 120
Disallow: /cgi-bin/attach
Disallow: /cgi-bin/changes
Disallow: /cgi-bin/edit
Disallow: /cgi-bin/geturl
Disallow: /cgi-bin/installpasswd
Disallow: /cgi-bin/logon
Disallow: /cgi-bin/mailnotify
Disallow: /cgi-bin/manage
Disallow: /cgi-bin/oops
Disallow: /cgi-bin/passwd
Disallow: /cgi-bin/preview
Disallow: /cgi-bin/rdiff
Disallow: /cgi-bin/register
Disallow: /cgi-bin/rename
Disallow: /cgi-bin/save
Disallow: /cgi-bin/search
Disallow: /cgi-bin/statistics
Disallow: /cgi-bin/upload
Disallow: /cgi-bin/viewauth
Disallow: /cgi-bin/viewfile
Disallow: /cgi-bin/view/Sandbox
Disallow: /cgi-bin/view/*?

Disallow: /cgi-bin/survey
Disallow: /cgi-bin/twikicvs
Disallow: /cgi-bin/awstats/

Disallow: /cgi-bin/b/bin/

User-agent: *
Crawl-delay: 120
Disallow: /cgi-bin/attach
Disallow: /cgi-bin/changes
Disallow: /cgi-bin/edit
Disallow: /cgi-bin/geturl
Disallow: /cgi-bin/installpasswd
Disallow: /cgi-bin/logon
Disallow: /cgi-bin/mailnotify
Disallow: /cgi-bin/manage
Disallow: /cgi-bin/oops
Disallow: /cgi-bin/passwd
Disallow: /cgi-bin/preview
Disallow: /cgi-bin/rdiff
Disallow: /cgi-bin/register
Disallow: /cgi-bin/rename
Disallow: /cgi-bin/save
Disallow: /cgi-bin/search
Disallow: /cgi-bin/statistics
Disallow: /cgi-bin/upload
Disallow: /cgi-bin/viewauth
Disallow: /cgi-bin/viewfile
Disallow: /cgi-bin/view/Sandbox
Disallow: /cgi-bin/view/*?

Disallow: /cgi-bin/survey
Disallow: /cgi-bin/twikicvs
Disallow: /cgi-bin/awstats/

Disallow: /cgi-bin/b/bin/