# $Id: robots.txt,v 1.2 1995/10/30 04:15:40 fielding Exp $ # robots.txt for http://www.c4systm.com/TheHat # see for an explanation. Sitemap: http://www.goetheanscience.org/sitemap.php?do=showsitemap&sm=sitemap.xml.gz User-agent: MOMspider # The Multi-Owner Maintenance Spider User-agent: * # All other spiders should avoid Disallow: /Bakup # Back-up files Disallow: /Bakup/ # Back-up files Disallow: /BookShop/data # Book Shop files Disallow: /BookShop/data/ # Book Shop files Disallow: /Text # Text files Disallow: /Text/ # Text files Disallow: /cgi-bin # Script files Disallow: /cgi-bin/ # Script files Disallow: /maintain # Maintenance sub-directory Disallow: /maintain/ # Maintenance sub-directory Disallow: /Orders/maintain # Maintenance sub-directory Disallow: /Orders/maintain/ # Maintenance sub-directory Disallow: /include # Included files sub-directory Disallow: /include/ # Included files sub-directory Disallow: /bin # Binary files sub-directory Disallow: /bin/ # Binary files sub-directory Disallow: /etc # Miscellaneous files sub-directory Disallow: /etc/ # Miscellaneous files sub-directory Disallow: /logs # Log files sub-directory Disallow: /logs/ # Log files sub-directory Disallow: /old # Old files sub-directory Disallow: /old/ # Old files sub-directory Disallow: /maps # Maps sub-directory Disallow: /maps/ # Maps sub-directory