User-agent: Scrapy
Allow: /

User-agent: bingbot
Disallow: /
User-agent: crawl
Disallow: /
User-agent: mj12bot
Disallow: /
User-agent: robot
Disallow: /
User-agent: slurp
Disallow: /
User-agent: bot[\s_ :,\.\;\/\\-]
Disallow: /
User-agent: bingpreview
Disallow: /
User-agent: seznambot
Disallow: /
User-agent: scanner
Disallow: /
User-agent: phantom
Disallow: /
User-agent: favicon
Disallow: /
User-agent: java\/[0-9]
Disallow: /
User-agent: msnbot\-media
Disallow: /
User-agent: spider
Disallow: /
User-agent: yandex
Disallow: /
User-agent: ia_archiver
Disallow: /
User-agent: exabot
Disallow: /
User-agent: baiduspider
Disallow: /
User-agent: surveybot
Disallow: /
User-agent: libwww
Disallow: /