# Robots.txt file for http://www.flamencodancer.co.uk/
#
# All robots will spider the domain

User-agent: Googlebot
Disallow: /*.flv$
Disallow: /*.fla$
Disallow: /*.swf$
Disallow: /*.mp3$
Disallow: /*.divx$
Disallow: /*.txt$
# disallow all files with ? in url
Disallow: /*?

# allow the Google image crawler to crawl all pages
User-agent: Googlebot-Image
Allow: /

# allow the Google bot to craw all pages for its mobile index
User-agent: Googlebot-Mobile
Allow: /

User-agent: ia_archiver
User-agent: archive.org_bot
# disallow certain search engines such as the Internet Wayback Machine so that old versions of your site can no longer be accessed
Disallow: /

User-agent: slurp
Disallow: /*.flv$
Disallow: /*.fla$
Disallow: /*.swf$
Disallow: /*.mp3$
Disallow: /*.divx$
Disallow: /*.txt$



User-agent: msnbot
Disallow: /*.flv$
Disallow: /*.fla$
Disallow: /*.swf$
Disallow: /*.mp3$
Disallow: /*.divx$
Disallow: /*.txt$
# disallow all files with ? in url
Disallow: /*?

# Exclude Files From All Robots:
User-agent: *
Disallow: /css/
Disallow: /cgi-bin/
Disallow: /Working_test
Disallow: /Scripts/
Disallow: /js/
Disallow: /gbcf-v3/
Disallow: /?detectflash=false

Sitemap: http://www.flamencodancer.co.uk/sitemap.xml