User-Agent: * Disallow: /dev/ Disallow: /cgi-bin/ Disallow: /scripts/ # disallow all files ending with these extensions Disallow: /*.asp$ Disallow: /*.aspx$ Disallow: /catalogue/2002cat.pdf # allow browsershots User-agent: Browsershots Disallow: