# All robots, disallow message bases and images User-agent: * Disallow: /cgi-bin/ Disallow: /errors/ Disallow: /gifs/ Disallow: /yabb/ # Google ads, allow everything User-agent: Mediapartners-Google* Disallow: # Google crawler, allow everything except cgi-bin User-agent: Googlebot Disallow: /cgi-bin/ # ShopWiki, disallow everything (nothing to buy here) User-agent: ShopWiki Disallow: / # BecomeBot, ditto above User-agent: BecomeBot Disallow: / # Attentio, blog crawler User-agent: Attentio Disallow: /