User-agent: * Disallow: /cgi-bin/ Disallow: /awstats/ Disallow: /usage2/ Disallow: /phpmailer/ Disallow: /mailform_template/ # but allow htdig to index our doc-tree #User-agent: susedig #Disallow: # disallow stress test user-agent: stress-agent Disallow: /