# Comment tag - here is the start....

User-agent: webcrawler
Disallow:

User-agent: lycra
Disallow: /

User-agent: *
Disallow: /tmp
Disallow: /logs
Disallow: /webalizer

# /robots.txt for webcrawler.com frontend machines

User-agent: *
Disallow: /cgi-bin/
Disallow: /relocate


# Comment tag - here is the end....