# /robots.txt file for http://spacehulk.barsoom.cc/ # Allow the wget util User-agent: wget Disallow: # Allow www.archive.org User-agent: ia_archiver Disallow: # Selectively allow www.google.com User-agent: googlebot Disallow: /gsinv Disallow: /logs Disallow: /sh-campgn # Block all others User-agent: * Disallow: /