# disallow specific bots from indexing or crawling the site at all
# Most recent additions first. 

User-agent: Yandex bot
disallow: /

User-agent: Slurp
Crawl-delay: 1

User-agent: *
Crawl-delay: 1