# Begin all other Search Engine defaults User-agent: * Disallow: /MP3 # END all other Search Engine defaults # -------------------------------------- # YAHOO Slurp webcrawler engine # Slow down frequency of access for indexing # In August 2007, Yahoo Slurp used 156 MB of bandwidth by itself # Avoid indexing the audio MP3 files User-agent: Slurp Crawl-delay: 5 Disallow: /MP3 # End YAHOO Slurp specific instructions # -------------------------------------- # -------------------------------------- # MSNBot webcrawler engine # Slow down frequency of access for indexing # Avoid indexing the audio MP3 files User-agent: msnbot Disallow: /*.MP3$ # End MSNBot Slurp specific instructions # --------------------------------------