# Whitelisted user-agents are allowed
User-agent: googlebot
User-agent: slurp
User-agent: msnbot
User-agent: teoma
User-agent: Echo
User-agent: Scooter
User-agent: ArchitextSpider
User-agent: InfoSeek Sidewinder
User-agent: T-Rex
Disallow: /cgi
Disallow: /files
Disallow: /js
Disallow: /Java
# Disallow all others
User-agent: *
Disallow: /