User-agent: *
Allow: /
Disallow: /admin/
Disallow: /includes/
Disallow: /config.php
Disallow: /backup/

Sitemap: http://www.chunakeji.com/sitemap.xml

# 爬虫指令
Crawl-delay: 1

# 允许的爬虫
User-agent: Googlebot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: Bingbot
Allow: /

# 禁止的爬虫
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /