# lyme.net robots.txt template
#
# $Id: robots.txt,v 1.4 2013/08/14 14:08:19 root Exp $
#
# (ref http://info.webcrawler.com/mak/projects/robots/norobots.html)
# http://www.robotstxt.org/wc/exclusion.html
#
# $Log: robots.txt,v $
# Revision 1.4  2013/08/14 14:08:19  root
# Updated from flash.lyme.net settings
#
# Revision 1.3  2013/08/12 16:45:33  root
# Block legacy HTML forum links
#
# Revision 1.2  2009/02/08 20:50:54  root
# allow /ubb/ now that spider-friendly URLs enabled in UBB
#
# Revision 1.5  2008/08/24 18:04:49  root
# add YodaoBot
#
# Revision 1.5  2008/08/24 17:59:09  root
# add YodaoBot
#
# Revision 1.4  2008/04/06 15:17:27  root
# add psbot, twiceler
#
# Revision 1.3  2008/01/29 03:38:52  root
# update disallow entries
#
# Revision 1.2  2008/01/26 14:55:56  root
# add robot extensions
#
# Revision 1.1  2008/01/26 14:53:40  root
# Initial revision
#
#
# Allow selected access to WWW robots (wanderers/spiders)
# twb, Tue Feb 18 20:20:34 EST 1997
# twb, Tue Jan 30 11:29:17 EST 2001 - update directories
# twb, Wed Mar 24 09:58:03 EST 2004 - add dialectizer disallow
# twb, Sun Feb  8 15:50:27 EST 2009 - allow /ubb/ and /scripts/ now that spider-friendly URLs enabled in UBB
# twb, Sun Mar 27 13:54:15 EDT 2016 - add AhrefsBot
#
#
# Disallow: /files/
# Disallow: /ubb/
# Disallow: /scripts/

User-agent: *
Disallow: /icons/
Disallow: /ubb/Forum1/HTML/
Disallow: /ubb/Forum2/HTML/
Disallow: /ubb/Forum3/HTML/
Disallow: /ubb/Forum4/HTML/
Disallow: /ubb/Forum5/HTML/
Disallow: /ubb/Forum6/HTML/
Disallow: /ubb/Forum7/HTML/
Disallow: /ubb/Forum8/HTML/
Disallow: /ubb/Forum9/HTML/
Crawl-delay: 5
Request-rate: 1/5
Visit-time: 0200-0600
Sitemap: http://www.lymenet.org/sitemap.xml.gz
Host: www.lymenet.org

# http://www.rinkworks.com/dialect/disallow.shtml
User-agent: dialect
Disallow: /

# http://www.picsearch.com/menu.cgi?item=Psbot
User-agent: psbot
Disallow: /

# http://www.cuil.com/info/webmaster_info/
User-agent: twiceler
Disallow: /

# http://www.yodao.com/help/webmaster/robot/004/
User-agent: YodaoBot
Disallow: /

# https://ahrefs.com/robot
User-agent: AhrefsBot
Disallow: / 

# SEOPROFILER.COM
User-agent: spbot
Disallow: /

# ezooms.com - One of the absolute must to block in every way you can from spying on you !!!
User-agent: ezooms
Disallow: /

# sistrix
User-agent: sistrix
Disallow: /

# Yandex bot - A rule breaker, just as Baidu spiders
#User-agent: Yandex
#Disallow: /

# proximic.com/info/spider.php
User-agent: proximic
Disallow: /php/

# http://filterdb.iss.net/crawler/
User-agent: oBot
Disallow: /

# MJ12bot - http://mj12bot.com/
User-agent: MJ12bot
Disallow: /

# https://megaindex.com/crawler
User-agent: MegaIndex.ru
Disallow: /

User-agent: megaindex.com
Disallow: /

User-agent: Mail.RU_Bot 
Disallow: /

User-agent: Mail.Ru 
Disallow: /