# <field>:<optionalspace><value><optionalspace>
User-Agent: Mediapartners-Google*
Disallow:
User-Agent: *
Disallow: crosslink-submit.cgi
Disallow: crosslink.cgi
Disallow: filename-search.cgi
Disallow: giwerundean-lesson.cgi
Disallow: mailing-list.cgi
Disallow: ngb-sign-in.cgi
Disallow: qar-complete.cgi
Disallow: random-giwer-lie.cgi
User-Agent: Scooter
Disallow: holocaust-hb.html
Disallow: holocaust-is.html
# Inktomi/Hotbot
User-Agent: Slurp
Disallow: holocaust-av.html
Disallow: holocaust-is.html
User-Agent: Infoseek
Disallow: holocaust-av.html
Disallow: holocaust-hb.html
#
#Search Engine: User-Agent
# AltaVista:    Scooter
# Infoseek:     Infoseek
# Hotbot:       Slurp
# AOL:          Slurp
# Excite:       ArchitextSpider
#Google:       Googlebot
# Goto:         Slurp:
# Lycos:        Lycos
# MSN:          Slurp
# Netscape:     Googlebot
# NorthernLight:   Gulliver
# WebCrawler:   ArchitextSpider
# Iwon:         Slurp
# Fast:         Fast
# DirectHit:    Grabber
# Yahoo Web Pages: Googlebot
# Looksmart Web Pages: Slurp