# Robots.txt for www.theitg.com


# Comments appear after the "#" symbol at the start of a line, or after a directive
# User-agent: BadBot # replace the 'BadBot' with the actual user-agent of the bot
# Disallow: / # keep them all out
# Crawl-delay: 5 # Limit crawl rate to 5 seconds per page
# Visit-time: 0600-0845 # only visit between 06:00 and 08:45 UTC (GMT)
# You can use multiple & seperate Sitemaps on a single website


User-agent: *
Request-rate: 1/5
Sitemap: http://www.theitg.com/sitemap.xml
Allow: /
Disallow: /babyboo4
Disallow: /babyboomerplace/files
Disallow: /babyboomerplace/downloader
Disallow: /Backups
Disallow: /bundles/files
Disallow: /careerportfolios/files
Disallow: /cgi-bin
Disallow: /dds
Disallow: /dds/admin
Disallow: /dds/cart
Disallow: /dds/completed
Disallow: /dds/DOWNLOADFILESFORSALE
Disallow: /dds/downloads
Disallow: /dds/images
Disallow: /dds/includes
Disallow: /dds/js
Disallow: /digitalstore/cgi-bin
Disallow: /digitalstore/info
Disallow: /digitalstore/install
Disallow: /digitalstore/updates
Disallow: /digitalstore/files
Disallow: /download
Disallow: /downloads
Disallow: /files
Disallow: /files/Download
Disallow: /html_email/files
Disallow: /itvguide
Disallow: /pc2tv
Disallow: /PornVideoPlayer_files
Disallow: /PornVideoPlayer2_files
Disallow: /pwd
Disallow: /shop
Disallow: /theitg
Disallow: /vistaconstructionboise
Disallow: /xfer