User-agent: *
Disallow:

# Allow all, for now.

# http://en.wikipedia.org/wiki/Robots_exclusion_standard

# This keeps all robots out:

# User-agent: *
# Disallow: /

# The next is an example that tells all crawlers not to 
# enter four directories of a website:

#User-agent: *
#Disallow: /cgi-bin/
#Disallow: /images/
#Disallow: /tmp/
#Disallow: /private/

#Example that tells a specific crawler not to enter one specific directory:

#User-agent: BadBot # replace the 'BadBot' with the actual user-agent of the bot
#Disallow: /private/

#Example that tells all crawlers not to enter one specific file:

#User-agent: *
#Disallow: /directory/file.html