User-agent: msnbot Crawl-Delay: 10 Disallow: User-agent: Slurp Crawl-Delay: 10 Disallow: # Google doesn't have a throttle option, and absolutely hammers the site # when it gets stuck in the events section, so we're just making sure it # doesn't get indexed for now User-agent: googlebot # Disallow: /en/festivals/calendar/event/ # Disallow: /fr/festivals/calendar/event/ # if Google is fail, it's safe to assume some others will be too User-agent: * Disallow: /en/festivals/calendar/event/ Disallow: /fr/festivals/calendar/event/