#Allowing W3Cs Link Checker
User-Agent: W3C-checklink
Allow /
Disallow: /cgi-bin/

#Allowing WayBackMachine's robot:
User-agent: ia_archiver
Allow /
Disallow: /cgi-bin/
Disallow: /experiment/
Disallow: /conventies/
Disallow: /css/

#Allowing Google to cache all but the below:
User-agent: Googlebot
Allow /
Allow: /sitemap.xml
Disallow: /cgi-bin/
Disallow: /experiment/
Disallow: /conventies/
Disallow: /css/

#Disallowing all other robots/spiders:
User-agent: *
Disallow: /