User-agent: *
Allow: /modules
Disallow: /css
Disallow: /images
Disallow: /inc


#
#Error 404 page
#
Disallow: /404.asp


#Sitemap: http://www.covesys.com/sitemap.xml

#
#Sitemap line is supported by Google, MSN, Yahoo, ASK
#

#---
# Start of Blacklisted bots
#---

User-agent:libwww
Disallow:/

User-agent:LinkWalker
Disallow:/


#---
#End of Blacklisted Bots
#---


# User-agent: Googlebot
# Disallow: /marklit
# Allow: /marklit/something.pdf
#
# Some major crawlers support the line 'Allow'
#


User-agent: Googlebot
Crawl-delay: 0.5

User-agent: Slurp #Yahoobot
Crawl-delay: 0.5

#
# Several crawlers support a crawl delay parameter.
# The number indicates seconds before the bot can make another request to the server.
#


#
# If a bot is not asking for robots.txt, then you'll need to ban it via .htaccess instead.
#
# RewriteEngine On 
# RewriteCond %{HTTP_USER_AGENT} BADBOT 
# RewriteRule .* - [F]