# Restrictions to all bots

User-agent: *
Disallow: /_Layouts/
Disallow: /Lists/
Disallow: /_catalogs/
Disallow: /_vti_bin/
Disallow: /WorkflowTasks/
Disallow: /search/
Disallow: /Documents/Forms/
Crawl-delay: 7


# Set crawl delay for important bots
# Note Google ignores this

User-agent: bingbot
Crawl-delay: 20

User-agent: msnbot
Crawl-delay: 20

User-agent: Yandex
Crawl-delay: 20

User-agent: Wget
Crawl-delay: 10


# Probably OK, not blocked
# Qwantify - https://www.distilnetworks.com/bot-directory/bot/qwantify/


# Blacklist, based on logs

User-agent: BLEXBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: psbot
Disallow: /

User-agent: DotBot
Disallow: /


# Blacklist, based on others

User-agent: magpie-crawler
Disallow: /