# Robots.txt for PanHunt - 搜索引擎蜘蛛爬行规则
# 针对百度、谷歌、必应、360、搜狗、神马等所有搜索引擎优化

User-agent: *
Allow: /

# 禁止爬行后台和私密页面
Disallow: /admin.html
Disallow: /favorites.html

# 允许抓取所有分类页、文章页、站点详情页
Allow: /category/
Allow: /article.html
Allow: /site.html
Allow: /blog.html
Allow: /submit.html
Allow: /sale.html
Allow: /search.html

# Sitemap地址 - 帮助搜索引擎快速发现所有页面
Sitemap: https://panhunt.com/sitemap.xml
Sitemap: https://panhunt.com/sitemap.txt

# 各大搜索引擎蜘蛛专属规则
User-agent: Baiduspider
Allow: /
Crawl-delay: 1

User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: bingbot
Allow: /
Crawl-delay: 1

User-agent: 360Spider
Allow: /
Crawl-delay: 1

User-agent: Sogou web spider
Allow: /
Crawl-delay: 1

User-agent: YisouSpider
Allow: /
Crawl-delay: 1

User-agent: Bytespider
Allow: /
Crawl-delay: 1