# /robots.txt file for http://webcrawler.com/
# mail webmaster@webcrawler.com for constructive criticism

User-agent: *
Disallow: /inc
Disallow: /menu
Disallow: /images
Disallow: /customerrors
Disallow: /pdf
Disallow: /scripts
Disallow: /gbcf-v3
Disallow: /public_html