# robots.txt - Server Exclusion file User-agent: * Disallow: /cgi-bin/ Disallow: /sc/ Disallow: /ss/ Disallow: /*.cgi$ Crawl-delay: 3 #User-agent: AdsBot-Google #Disallow: /cgi-bin/ #User-agent: CrazyWebCrawler-Spider #Disallow: /cgi-bin/ #User-agent: XoviBot #Disallow: /cgi-bin/ #UK bot http://mj12bot.com/ #User-agent: MJ12bot #Crawl-Delay: 5