# exclude help system from robots
User-agent: *
Crawl-delay: 100
Disallow: /manual/ 
Disallow: /cgi-bin/
Disallow: /doc/ 
Disallow: /gif/
Disallow: /rempt.xs4all.nl/andal//languages/denden/grammar/cases.html
Disallow: /rempt.xs4all.nl/andal//languages/denden/grammar/lexicon.html
Disallow: /rempt.xs4all.nl/andal//languages/denden/grammar/verbal_morphology.html
Disallow: /rempt.xs4all.nl/andal//literature/raygin.html
Disallow: /rempt.xs4all.nl/andal//music/introduction.html
Disallow: /rempt.xs4all.nl/andal//peoples_and_places/charya/brothels.html
Disallow: /rempt.xs4all.nl/andal//peoples_and_places/charya/charcult.html
Disallow: /rempt.xs4all.nl/andal//peoples_and_places/charya/money.html
Disallow: /rempt.xs4all.nl/andal//peoples_and_places/charya/sexuality.html
Disallow: /rempt.xs4all.nl/andal//peoples_and_places/clothes/broi_clothes.html
Disallow: /rempt.xs4all.nl/andal//peoples_and_places/people/bijfiguren.html
Disallow: /rempt.xs4all.nl/andal//peoples_and_places/people/index.html
# but allow htdig to index our doc-tree
User-agent: susedig
Disallow:
# disallow stress test
user-agent: stress-agent
Disallow: /
# disallow wisenut
user-agent: ZyBorg
Disallow: /
user-agent: TranSGeniKBot
Disallow: /
user-agent: TurnitinBot
Disallow: /
user-agent: msnbot
Disallow: /
User-agent: MLBot
Disallow: /
User-agent: Twiceler
Disallow: /