# We know these support <meta name="robots" content="noarchive"/>
User-agent: googlebot
User-agent: Googlebot
User-agent: Applebot
User-agent: slurp
User-agent: msnbot
User-agent: bingbot
User-agent: teoma
User-agent: askjeeves
User-agent: yandex
User-agent: baiduspider
User-agent: Twitterbot
Crawl-delay: 1
Disallow: /cgi/
Disallow: /cgi/pt/search
Allow: /cgi/pt
Allow: /cgi/mb

User-agent: Twitterbot
Allow: /cgi/pt
Allow: /cgi/mb
Allow: /cgi/imgsrv

User-agent: *
Crawl-delay: 1
Disallow: /cgi/