# This robot.txt file is for every spider
User-Agent: *
# They are disallowed access to files in these directories or beginning with these names
Disallow: /_private/
Disallow: /archive/
Disallow: /cgi/
Disallow: /data/
Disallow: /docs/
Disallow: /images/
Disallow: /js/
Disallow: /logs/
Disallow: /special/
Disallow: /test