User-agent: * Crawl-delay: 10 # Wait 10 seconds between requests Request-rate: 1/10 # No more than one page per 10 seconds Disallow: /edit/