#Robots.txt
User-agent: *
Disallow: /mycgi/  # mycgi directory
Disallow: /cgi/    # keep robots out of the executable tree
Disallow: /temp/   # temp directory
Disallow: /stats/  # stats directory
Disallow: /mailer/  # mailer directory
Disallow: /tell_to_friend.php
Disallow: /m/
Disallow: /search.cgi
User-agent: Mediapartners-Google

User-agent: bibgbot
Disallow: /
User-agent: Mail.RU_Bot
Disallow: /
User-agent: Baiduspider
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: YandexBot
Disallow: /
User-agent: proximic
Disallow: /
User-agent: Exabot
Disallow: /
User-agent: Scrapy
Disallow: /