User-agent: *
Allow: /
Allow: /html/
Allow: /html.cgi
Disallow: *
Disallow: *.pdf
Disallow: *.cgi
Disallow: *.php
Disallow: *.mod_perl