# robots.txt for http://hshlnb.hsh.com/ # Last update: 03/25/08 # Dunno if this will do what I want but better than mucking # with /etc/httpd/conf/httpd.conf # Altavista (Altavista search engine only) User-agent: Scooter # FAST/AllTheWeb (AllTheWeb search engine) User-agent: fast # Google (Google Search Engine) User-agent: Googlebot # Inktomi (Anzwers, AOL, Canada.com, Hotbot, MSN, etc.) User-agent: slurp # Info: http://www.robotstxt.org/wc/robots.html # beta 'robots.txt' checker at http://www.tardis.ed.ac.uk/home/sxw/robots/check/ # DON'T CRAWL HSHLNB.HSH.COM User-agent: * Disallow: / # Sitemap: http://www.hsh.com/sitemap.xml#