# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file # # To ban all spiders from the entire site uncomment the next two lines: # User-agent: * # Disallow: / User-agent: ShopWiki User-agent: twiceler # http://www.cuill.com/twiceler/robot.html User-agent: voyager # http://www.kosmix.com/html/crawler.html User-agent: echrigy # http://www.techrigy.com/ User-agent: voilabot # http://www.voila.fr/ Disallow: / # User-agent: Googlebot User-agent: * Disallow: /blogs/rss Disallow: /blogs/shorten_url Disallow: /action/blogs/shorten_url Disallow: /blogs/atom Disallow: /action/spellcheck Disallow: /action/blogs_admin Disallow: /action/blogs_admin/subscribe Disallow: /blogindex Disallow: /action/signup/create Disallow: /action/nav/upload_photo Disallow: /action/product_review/thumbs_up Disallow: /action/product_review/thumbs_down Disallow: /realestateandhomes-search Disallow: /action/agents/show_score Disallow: /action/blogs/comments Disallow: /reset_password Disallow: /topbloggers Disallow: /arcaptcha