# robots.txt for http://www.reflexionrecords.com/ # User-agent: ia_archiver # User-agent: WebCrawler User-agent: * Disallow: /_vti_bin/ Disallow: /_vti_cnf/ Disallow: /_vti_log/ Disallow: /_vti_pvt/ Disallow: /_vti_txt/ Disallow: /cgi/ Disallow: /cgi-bin/ Disallow: /conf/ Disallow: /img/ Disallow: /images/ Disallow: /logs/ Disallow: /ns-icons/ User-Agent: NPBot Disallow: / User-Agent: sitecheck.internetseer.com Disallow: /file-to-disallow ## http://www.singingfish.com/spider.html indexes media files: # User-agent: asterias # Disallow: /