# All robots will spider the domain
User-agent: *
# All robots will not spider the fol directories
Disallow: /2004_Site/
User-agent: Slurp
Crawl-delay: 5