robots.txt


User-agent: Slurp
Crawl-delay: 100
Disallow:

User-agent: gsa-crawler-www 
Crawl-delay: 100

User-agent: Googlebot
Crawl-delay: 100

User-agent: Mediapartners-Google
Disallow:

User-agent: Yahoo-NewsCrawler
Disallow:

User-Agent: msnbot
Crawl-delay: 100
Disallow:

User-Agent: *
Disallow: /config/
Disallow: /handlers/
Disallow: /includes/
Disallow: /interceptors/
Disallow: /layouts/
Disallow: /logs/
Disallow: /model/
Disallow: /plugins/
Disallow: /views/
Allow: /