# allow all web spiders
User-agent: *
# paths to NOT allow crawling
Disallow: /App_Code/
Disallow: /App_Themes/
Disallow: /App_WebReferences/
Disallow: /Bin/
Disallow: /includes/
Disallow: /javascript/
Disallow: /styles/