# MiCount Technology - Robots.txt
# Updated: 2026-11-02
# Website: https://micount.cn
# Multi-language low-code platform learning center
#
# AI crawlers & agents are WELCOME on public HTML, llms*.txt, ai.txt,
# sitemap-index.xml, and structured JSON APIs (/data/api/).
# Prefer start URLs: https://micount.cn/llms.txt · https://micount.cn/ai.txt
# AI coding tools hub: https://micount.cn/llms-ai-tools.txt（35 款 / 11 场景 / 20 维）
# Agent discovery: /.well-known/agent.json · /.well-known/ai-plugin.json · /.well-known/mcp.json
# FAQ: /data/api/faq.json · Compare: /data/api/compare-matrix.json · AI tools: /data/api/ai-tools.json
# Note: /api/* is Express-proxied (WeChat); static JSON for crawlers lives under /data/api/

# ============================================
# 全AI爬虫 robots.txt 优化模板（含Grok专属规则）
# 置于 User-agent: * 之前，确保非标准 UA 不被通配 Disallow 误伤
# 2026-11 复核：国内引擎 UA 以官方/实测为准（Kimi-SearchBot·Kimi-User、
# DoubaoBot、DeepSeekBot、Baiduspider-render、GLMBot 等）
# ============================================

# ============================================
# Grok / xAI Live Search 爬虫（非标准UA，必须显式声明）
# 关键：xAI和Grok这两个UA不含"bot"标识符，默认配置会误屏蔽
# ============================================
User-agent: xAI
Allow: /

User-agent: Grok
Allow: /

User-agent: xAI-Bot
Allow: /

# ============================================
# OpenAI 爬虫 → ChatGPT / Copilot
# ============================================
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# ============================================
# Anthropic 爬虫 → Claude
# ============================================
User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# ============================================
# Perplexity 爬虫
# ============================================
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# ============================================
# Google Gemini 训练爬虫
# ============================================
User-agent: Google-Extended
Allow: /

# ============================================
# Apple Intelligence 爬虫
# ============================================
User-agent: Applebot-Extended
Allow: /

# ============================================
# Common Crawl（多个AI训练用）
# ============================================
User-agent: CCBot
Allow: /

# ============================================
# Amazon Alexa AI 爬虫
# ============================================
User-agent: Amazonbot
Allow: /

# ============================================
# 字节跳动爬虫 → 豆包
# ============================================
User-agent: Bytespider
Allow: /

# ============================================
# 夸克AI搜索爬虫（阿里系，遵守robots.txt）
# ============================================
User-agent: Quark-Spider
Allow: /

# ============================================
# 通义千问训练爬虫（阿里系）
# ============================================
User-agent: QwenBot
Allow: /

User-agent: Qwen-User
Allow: /

# ============================================
# 通配符规则（放在所有特定UA之后）
# ============================================
User-agent: *
Allow: /

# Sitemaps（sitemap-index 为入口，子地图见各 Allow）
# 说明：/sitemap.xml 为全站单文件（推荐直接提交给 Google / Bing）
#       /sitemap-index.xml 为分片索引（GSC 分区块看覆盖率）
#       /sitemap-ai.xml 为 AI 引擎专用高价值子集
Sitemap: https://micount.cn/sitemap-index.xml
Sitemap: https://micount.cn/sitemap.xml
Sitemap: https://micount.cn/sitemap-ai.xml
Sitemap: https://micount.cn/sitemap-images.xml
Sitemap: https://micount.cn/blog/sitemap.xml
Sitemap: https://micount.cn/sitemap-baidu-core.xml
Sitemap: https://micount.cn/sitemap-pages-root.xml
Sitemap: https://micount.cn/sitemap-pages-cn.xml
Sitemap: https://micount.cn/sitemap-pages-tw.xml
Sitemap: https://micount.cn/sitemap-pages-intl.xml
Sitemap: https://micount.cn/sitemap-network.xml
Sitemap: https://micount.cn/CN/news/sitemap.xml
Sitemap: https://micount.cn/CN/news/blog-sitemap-index.xml
Sitemap: https://micount.cn/CN/partner/sitemap.xml
Sitemap: https://micount.cn/CN/job/sitemap.xml
Sitemap: https://micount.cn/tw/sitemap-index.xml

# IndexNow（Bing / Yandex / Naver 即时推送）：key 文件见站点根目录 <key>.txt
# 提交入口（站长工具）：https://www.bing.com/webmasters

# International English site - Low priority for CN-focused business
# If English market is not primary, consider Disallow: /en/
Allow: /en/

# Crawl-delay (be gentle with crawlers)
Crawl-delay: 1

# ==========================================
# Disallow: Non-content files and directories
# ==========================================

# Python scripts and build tools
Disallow: /*.py$
Disallow: /*.ps1$
Disallow: /*.bat$
Disallow: /*.log$

# AI Knowledge Base Files (explicitly allowed before wildcard txt block)
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /llms-zh.txt
Allow: /llms-blog.txt
Allow: /llms-compare.txt
Allow: /llms-faq.txt
Allow: /llms-platforms.txt
Allow: /llms-news-index.txt
Allow: /llms-ai-tools.txt
Allow: /llms-wechat.txt
Allow: /ai.txt
Allow: /robots.txt
Allow: /sitemap-index.xml
Allow: /sitemap.xml
Allow: /sitemap-baidu-core.xml
Allow: /sitemap-ai.xml
Allow: /blog/sitemap.xml
Allow: /news-feed.xml
Allow: /*/llms.txt
Allow: /tw/llms.txt
Allow: /.well-known/
Allow: /.well-known/security.txt
Allow: /.well-known/ai-plugin.json
Allow: /.well-known/ai-policy.json
Allow: /.well-known/mcp.json
Allow: /.well-known/agent.json

# API & structured data for AI agents (must precede wildcard json block)
# Prefer /data/api/*.json — nginx proxies /api/* to Express (WeChat), so static /api/*.json returns 404.
Allow: /api/
Allow: /api/*.json
Allow: /data/api/
Allow: /data/api/*.json
Allow: /data/api/faq.json
Allow: /data/api/compare-matrix.json
Allow: /data/api/index.json
Allow: /llms-platforms.txt
Allow: /llms-compare.txt
Allow: /llms-faq.txt
Allow: /llms-news-index.txt
Allow: /blog/sitemap
Allow: /blog/sitemap.xml
Allow: /CN/news/sitemap.xml
Allow: /CN/news/blog-sitemap-
Allow: /CN/network/sitemap.xml
Allow: /CN/partner/sitemap.xml
Allow: /CN/job/sitemap.xml
Allow: /tw/sitemap-index.xml
Allow: /tw/sitemap
Allow: /data/api/blog.json
Allow: /data/api/news.json
Allow: /blog/feed.xml
Allow: /blog/
Allow: /llms-blog.txt
Allow: /llms-ai-tools.txt
Allow: /ai-tools.html
# AI 编程工具 GEO 面（35 款 / 11 场景 / 20 维）
Allow: /CN/ai-tools/
Allow: /CN/ai-tools/index.html
Allow: /CN/ai-tools/data/tools.json
Allow: /data/api/ai-tools.json
Allow: /.well-known/agent.json
Allow: /.well-known/ai-plugin.json
Allow: /.well-known/ai-policy.json
Allow: /.well-known/mcp.json
Allow: /.well-known/openapi.yaml

Disallow: /*.txt$
Disallow: /*.json$
Disallow: /*.md$
Disallow: /*.docx$

# Cache and checkpoint files
Disallow: /_*

# Backup and temporary files
Disallow: /*_backup*
Disallow: /*_temp*
Disallow: /*_old*
Disallow: /*_bak*
Disallow: /*_broken*
Disallow: /*_fixed*
Disallow: /*.bak$

# Non-content directories
Disallow: /.qoder/
Disallow: /.vscode/
Disallow: /file/
Disallow: /components/
Disallow: /generate_articles.html

# ==========================================
# Allow: Content pages and static resources
# ==========================================

# Static resources (explicitly allow for rendering)
Allow: /js/
Allow: /css/
Allow: /styles/
Allow: /libs/
Allow: /images/
Allow: /*.css$
Allow: /*.js$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.gif$
Allow: /*.svg$
Allow: /*.webp$
Allow: /*.ico$
Allow: /*.woff2$
Allow: /*.woff$

# All HTML content pages
Allow: /*.html$

# ==========================================
# Search engine specific directives
# ==========================================

User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Baiduspider
Allow: /
Crawl-delay: 1

User-agent: Sogou web spider
Allow: /
Crawl-delay: 2

User-agent: 360Spider
Allow: /
Crawl-delay: 2

User-agent: Yandex
Allow: /
Crawl-delay: 2

User-agent: NaverBot
Allow: /
Crawl-delay: 2

# ==========================================
# AI Agents & Intelligent Assistants (扩展白名单)
# 上文已声明 21 个主流 AI UA；此处补充国内/社交/其他智能体
# ==========================================

User-agent: Claude-Web
Allow: /

User-agent: Meta-ExternalAgent
Allow: /
Crawl-delay: 2

User-agent: Doubao
Allow: /
Crawl-delay: 2

User-agent: DoubaoBot
Allow: /
Crawl-delay: 2

User-agent: DeepSeekBot
Allow: /
Crawl-delay: 2

# 百度AI专用爬虫 → 文心一言
User-agent: BaiduSpider-AI
Allow: /
Crawl-delay: 1

# 百度渲染爬虫（执行 JS，AI 摘要页依赖它）
User-agent: Baiduspider-render
Allow: /
Crawl-delay: 1

# 文心一言 用户触发抓取
User-agent: YiyanBot
Allow: /
Crawl-delay: 1

# 腾讯元宝AI爬虫
User-agent: TencentBot
Allow: /
Crawl-delay: 2

User-agent: KimiBot
Allow: /
Crawl-delay: 2

# Kimi 检索 / 用户触发抓取（官方文档公布 UA，2026-11 复核）
User-agent: Kimi-SearchBot
Allow: /
Crawl-delay: 2

User-agent: Kimi-User
Allow: /

# Kimi / 月之暗面 备用 UA
User-agent: MoonshotBot
Allow: /
Crawl-delay: 2

# 通义千问（阿里）备用 UA
User-agent: TongyiBot
Allow: /
Crawl-delay: 2

# 文心一言（百度）AI 爬虫
User-agent: ERNIEBot
Allow: /
Crawl-delay: 1

# 腾讯元宝 AI 助手
User-agent: YuanbaoBot
Allow: /
Crawl-delay: 2

# 讯飞星火
User-agent: SparkBot
Allow: /
Crawl-delay: 2

# 智谱清言 / GLM 备用 UA
User-agent: ChatGLMBot
Allow: /
Crawl-delay: 2

# 智谱官方训练爬虫（2026-11 复核：官方训练摘要用的名字是 GLMBot）
User-agent: GLMBot
Allow: /
Crawl-delay: 2

User-agent: ChatGLM-Spider
Allow: /
Crawl-delay: 2

# 百川智能
User-agent: BaichuanBot
Allow: /
Crawl-delay: 2

# MiniMax 海螺 AI
User-agent: MiniMaxBot
Allow: /
Crawl-delay: 2

# 阶跃星辰
User-agent: StepBot
Allow: /
Crawl-delay: 2

# 秘塔 AI 搜索
User-agent: MetasoBot
Allow: /
Crawl-delay: 2

# 天工 AI（昆仑万维）
User-agent: TiangongBot
Allow: /
Crawl-delay: 2

User-agent: ZhipuBot
Allow: /
Crawl-delay: 2

User-agent: YiBot
Allow: /
Crawl-delay: 2

User-agent: cohere-ai
Allow: /
Crawl-delay: 2

User-agent: anthropic-ai
Allow: /
Crawl-delay: 2

User-agent: YouBot
Allow: /
Crawl-delay: 2

User-agent: Diffbot
Allow: /
Crawl-delay: 2

User-agent: Timpibot
Allow: /
Crawl-delay: 2

User-agent: omgili
Allow: /
Crawl-delay: 2

# DuckDuckGo AI
User-agent: DuckAssistBot
Allow: /
Crawl-delay: 2

# Mistral Le Chat 引用抓取
User-agent: MistralAI-User
Allow: /
Crawl-delay: 2

# Google Agentic Browser (Project Mariner)
User-agent: GoogleAgent-Mariner
Allow: /
Crawl-delay: 2

# Apple Siri / Spotlight
User-agent: Applebot
Allow: /
Crawl-delay: 2

# Allen Institute / Semantic Scholar
User-agent: AI2Bot
Allow: /
Crawl-delay: 2

# Meta 链接预览 (Facebook / Instagram / WhatsApp)
User-agent: FacebookBot
Allow: /

User-agent: facebookexternalhit
Allow: /

# LinkedIn 分享预览
User-agent: LinkedInBot
Allow: /

# 华为花瓣搜索 / AI 助手 (5亿+用户)
User-agent: PetalBot
Allow: /
Crawl-delay: 2

# Brave AI 搜索引擎
User-agent: Bravebot
Allow: /
Crawl-delay: 2

# X/Twitter 卡片预览
User-agent: Twitterbot
Allow: /

# Telegram 链接预览
User-agent: TelegramBot
Allow: /

# Slack 链接预览
User-agent: Slackbot-LinkExpanding
Allow: /

# ==========================================
# 2026年新增主流 AI/Agent/搜索引擎
# ==========================================

# 神马搜索 / UC AI 助手（中国移动端大流量）
User-agent: YisouSpider
Allow: /
Crawl-delay: 2

# 华为盘古 AI
User-agent: PanguBot
Allow: /
Crawl-delay: 2

# Webz.io 数据采集（多家 LLM 训练数据上游）
User-agent: Webzio-Extended
Allow: /
Crawl-delay: 3

# Meta 即时抓取（智能体调用场景）
User-agent: Meta-ExternalFetcher
Allow: /

# Google 通用爬虫 / Vertex AI
User-agent: GoogleOther
Allow: /
Crawl-delay: 1

User-agent: Google-CloudVertexBot
Allow: /
Crawl-delay: 2

# OpenAI 图像抓取
User-agent: OpenAI-Image-Downloader
Allow: /
Crawl-delay: 2

# Phind 编程 AI
User-agent: PhindBot
Allow: /
Crawl-delay: 2

# Andi AI 搜索
User-agent: AndiBot
Allow: /
Crawl-delay: 2

# Komo AI 搜索
User-agent: Komo
Allow: /
Crawl-delay: 2

# Mojeek 独立搜索引擎
User-agent: MojeekBot
Allow: /
Crawl-delay: 2

# 爱问知识人
User-agent: Iaskspider/2.0
Allow: /
Crawl-delay: 2

# Velen AI
User-agent: VelenPublicWebCrawler
Allow: /
Crawl-delay: 2

# TikTok 智能推荐 (ByteDance 旗下)
User-agent: TikTokSpider
Allow: /
Crawl-delay: 2

# ==========================================
# 2026-11 GEO 复核新增（官方 UA / 检索型智能体）
# 目的：DeepSeek / 豆包 / Kimi / GPT / Codex 等回答引擎能完整读到知识面
# ==========================================

# OpenAI Codex / ChatGPT 联网检索：走 ChatGPT-User（用户触发）与
# OAI-SearchBot（检索索引），无需单独 UA；此处保留说明避免误封。
User-agent: OAI-AdsBot
Allow: /
Crawl-delay: 2

# Manus 浏览器智能体（按用户指令抓取）
User-agent: Manus-User
Allow: /

# Google Agentic 浏览 / Gemini Notebook（用户触发，不遵守 robots，仍显式声明）
User-agent: Google-Agent
Allow: /

User-agent: Google-GeminiNotebook
Allow: /

# Amazon 检索 / 用户触发
User-agent: Amzn-SearchBot
Allow: /
Crawl-delay: 2

User-agent: Amzn-User
Allow: /

# Meta 训练爬虫（小写 UA 变体）
User-agent: meta-externalagent
Allow: /
Crawl-delay: 2

# Allen Institute 检索索引
User-agent: SemanticScholarBot
Allow: /
Crawl-delay: 2

# 开源友好型训练爬虫（尊重 robots.txt，可放行）
User-agent: FriendlyCrawler
Allow: /
Crawl-delay: 2

# ==========================================
# Blocked: Scraping tools & aggressive bots
# 已知爬虫工具/恶意采集器禁止访问
# ==========================================

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: ZoominfoBot
Disallow: /

User-agent: SEOkicks-Robot
Disallow: /

User-agent: Xenu
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: magpie-crawler
Disallow: /

User-agent: Seekport Crawler
Disallow: /