User-agent: * # this is wildcard for all bots
Disallow: /scotmail/ # directories to not allow bots in ex /downloads/
Disallow: /worksite/
disallow: /devsite/
Disallow: /stats/
Disallow: /cgi-bin/
Request-rate: 1/5 # maximum rate is one page every 5 seconds
Visit-time: 0100-0245 # only visit between 1:00 AM and 2:45 AM UT (GMT)
Sitemap: http://commoditycrudeoil.com/sitemap.xml # site map