# dpsource.robots.txt # $Id: dpsource.robots.txt,v 1.6 2008/05/06 20:57:15 dferruggia Exp $ # This is for yahoo User-agent: Slurp Crawl-delay: 10 # For all robots User-agent: * # This is for anyone else Request-rate: 1/1 # maximum rate is one page every 1 seconds # Don't put anything in the shopping cart Disallow: cart.pl Disallow: /cgi-bin/cart.pl Disallow: /decor/cart.pl # Don't try to track orders Disallow: getTracking.pl Disallow: /cgi-bin/getTracking.pl Disallow: /decor/getTracking.pl # Don't index these pages Disallow: /decor/myAccount.pl # Don't look at obsolete programs Disallow: advSearch.pl Disallow: drillDown.pl Disallow: rawSearch.pl Disallow: /cgi-bin/advSearch.pl Disallow: /cgi-bin/drillDown.pl Disallow: /cgi-bin/rawSearch.pl Disallow: /decor/advSearch.pl Disallow: /decor/drillDown.pl Disallow: /decor/rawSearch.pl # Don't index product.pl and search.pl (Re: Bug 6598) Disallow: /decor/search.pl Disallow: /decor/product.pl # This is to block Fatbot and www.thefind.com who we never got #sales from anyways and gave wrong prices for us anyways User-agent: Fatbot Disallow: /