# Robots.txt # Be nice. # this allows the FAST Enterprise crawler to crawl your sites User-agent: FAST Enterprise Crawler 6 Disallow: # User-agent: Mediapartners-Google* Disallow: # User-agent: YahooFeedSeeker Disallow: http://nashuatelegraph.robocaster.com/ # User-agent: MSIECrawler Disallow: / # User-agent: Topix.net Disallow: / # User-agent: * Disallow: /csp/cms/sites/dt.cms.ad.AdServer.cls Disallow: /csp/cms/sites/CabinetPress/templates/stories/full/enlarge.csp