# /robots.txt file for http://webcrawler.com/
# mail webmaster@webcrawler.com for constructive criticism

User-agent: *
Disallow: /stylesheet.css
Disallow: /error.aspx
Disallow: /login.aspx
Disallow: /logout.aspx
Disallow: /terms.aspx
Disallow: /terms.xml
# Disallow: /privacy.aspx
Disallow: /ppipn.aspx
Disallow: /confirm.aspx
Disallow: /billing.aspx
Disallow: /acctreg.aspx
Disallow: /Transforms
Disallow: /Templates
Disallow: /MyAccount

# Don't restrict the W3C link checker
User-Agent: W3C-checklink