{
  "schema_version": "1.0",
  "updated_at": "2026-10-07",
  "source_url": "https://rimonoa.jp/labs/ai-crawler/user-agent-dictionary",
  "methodology_url": "https://rimonoa.jp/labs/ai-crawler#methodology",
  "usage_info_url": "https://rimonoa.jp/reprint-policy#dataset-citation",
  "monitoring": {
    "cadence": "daily",
    "source_policy": "official vendor documentation plus multiple public discovery registries",
    "auto_publish_catalog": false,
    "review_gate": "persistent human-reviewed pull request; runtime catalog changes remain separate",
    "failure_mode": "fail closed when any configured source cannot be fetched"
  },
  "matching": {
    "field": "HTTP User-Agent",
    "strategy": "exclude robots_only_control_tokens first, then case-insensitive substring match against match_token",
    "exclusion_order": "robots_only_control_tokens before entries",
    "warning": "A User-Agent match is a claimed identity, not proof that the request originated from the named operator."
  },
  "entries": [
    {
      "match_token": "OAI-SearchBot",
      "bot_name": "OAI-SearchBot",
      "provider": "openai",
      "provider_label": "OpenAI",
      "crawler_kind": "ai",
      "raw_purpose": "search",
      "purpose_category": "search_index",
      "legacy_purposes": [],
      "description_ja": "ChatGPT Searchの検索結果にWebページを表示するための検索クローラー。",
      "robots_behavior": "respects",
      "official_docs_url": "https://developers.openai.com/api/docs/bots",
      "verification_url": "https://openai.com/searchbot.json",
      "source_confidence": "official",
      "verified_on": "2026-10-07",
      "note": null
    },
    {
      "match_token": "GPTBot",
      "bot_name": "GPTBot",
      "provider": "openai",
      "provider_label": "OpenAI",
      "crawler_kind": "ai",
      "raw_purpose": "training",
      "purpose_category": "training",
      "legacy_purposes": [],
      "description_ja": "OpenAIの生成AI基盤モデル改善に利用される可能性があるコンテンツを収集するクローラー。",
      "robots_behavior": "respects",
      "official_docs_url": "https://developers.openai.com/api/docs/bots",
      "verification_url": "https://openai.com/gptbot.json",
      "source_confidence": "official",
      "verified_on": "2026-10-07",
      "note": null
    },
    {
      "match_token": "ChatGPT-User",
      "bot_name": "ChatGPT-User",
      "provider": "openai",
      "provider_label": "OpenAI",
      "crawler_kind": "ai",
      "raw_purpose": "user_fetch",
      "purpose_category": "user_fetch",
      "legacy_purposes": [],
      "description_ja": "ChatGPTやCustom GPTで、ユーザー操作を起点にWebページを取得するUser-Agent。",
      "robots_behavior": "user_triggered_may_ignore",
      "official_docs_url": "https://developers.openai.com/api/docs/bots",
      "verification_url": "https://openai.com/chatgpt-user.json",
      "source_confidence": "official",
      "verified_on": "2026-10-07",
      "note": null
    },
    {
      "match_token": "Claude-SearchBot",
      "bot_name": "Claude-SearchBot",
      "provider": "anthropic",
      "provider_label": "Anthropic",
      "crawler_kind": "ai",
      "raw_purpose": "search",
      "purpose_category": "search_index",
      "legacy_purposes": [],
      "description_ja": "Claudeの検索体験でWebコンテンツを発見・参照するための検索クローラー。",
      "robots_behavior": "respects",
      "official_docs_url": "https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "verification_url": "https://claude.com/crawling/bots.json",
      "source_confidence": "official",
      "verified_on": "2026-10-07",
      "note": null
    },
    {
      "match_token": "ClaudeBot",
      "bot_name": "ClaudeBot",
      "provider": "anthropic",
      "provider_label": "Anthropic",
      "crawler_kind": "ai",
      "raw_purpose": "training",
      "purpose_category": "training",
      "legacy_purposes": [],
      "description_ja": "Anthropicのモデル改善に利用される可能性がある公開Webコンテンツを収集するクローラー。",
      "robots_behavior": "respects",
      "official_docs_url": "https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "verification_url": "https://claude.com/crawling/bots.json",
      "source_confidence": "official",
      "verified_on": "2026-10-07",
      "note": null
    },
    {
      "match_token": "Claude-User",
      "bot_name": "Claude-User",
      "provider": "anthropic",
      "provider_label": "Anthropic",
      "crawler_kind": "ai",
      "raw_purpose": "user_fetch",
      "purpose_category": "user_fetch",
      "legacy_purposes": [],
      "description_ja": "Claudeユーザーの操作を起点に、その場でWebページを取得するUser-Agent。",
      "robots_behavior": "respects",
      "official_docs_url": "https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "verification_url": "https://claude.com/crawling/bots.json",
      "source_confidence": "official",
      "verified_on": "2026-10-07",
      "note": null
    },
    {
      "match_token": "PerplexityBot",
      "bot_name": "PerplexityBot",
      "provider": "perplexity",
      "provider_label": "Perplexity",
      "crawler_kind": "ai",
      "raw_purpose": "search",
      "purpose_category": "search_index",
      "legacy_purposes": [],
      "description_ja": "Perplexityの検索結果でWebサイトを発見・リンクするためのクローラー。基盤モデル学習用ではないと公式説明されています。",
      "robots_behavior": "respects",
      "official_docs_url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
      "verification_url": "https://www.perplexity.com/perplexitybot.json",
      "source_confidence": "official",
      "verified_on": "2026-10-07",
      "note": null
    },
    {
      "match_token": "Perplexity-User",
      "bot_name": "Perplexity-User",
      "provider": "perplexity",
      "provider_label": "Perplexity",
      "crawler_kind": "ai",
      "raw_purpose": "user_fetch",
      "purpose_category": "user_fetch",
      "legacy_purposes": [],
      "description_ja": "Perplexityユーザーの操作を起点にWebページを取得するUser-Agent。",
      "robots_behavior": "user_triggered_may_ignore",
      "official_docs_url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
      "verification_url": "https://www.perplexity.com/perplexity-user.json",
      "source_confidence": "official",
      "verified_on": "2026-10-07",
      "note": null
    },
    {
      "match_token": "MistralAI-Training",
      "bot_name": "MistralAI-Training",
      "provider": "mistral",
      "provider_label": "Mistral AI",
      "crawler_kind": "ai",
      "raw_purpose": "training",
      "purpose_category": "training",
      "legacy_purposes": [],
      "description_ja": "Mistralの生成AIモデル学習用データセットを構築するためにWebコンテンツを収集するクローラー。",
      "robots_behavior": "respects",
      "official_docs_url": "https://docs.mistral.ai/robots",
      "verification_url": null,
      "source_confidence": "official",
      "verified_on": "2026-10-07",
      "note": null
    },
    {
      "match_token": "MistralAI-Index",
      "bot_name": "MistralAI-Index",
      "provider": "mistral",
      "provider_label": "Mistral AI",
      "crawler_kind": "ai",
      "raw_purpose": "search",
      "purpose_category": "search_index",
      "legacy_purposes": [],
      "description_ja": "Mistral Search / Vibeの回答用インデックスを構築するクローラー。公式説明では生成AI学習には使用しません。",
      "robots_behavior": "respects",
      "official_docs_url": "https://docs.mistral.ai/robots",
      "verification_url": null,
      "source_confidence": "official",
      "verified_on": "2026-10-07",
      "note": null
    },
    {
      "match_token": "MistralAI-User",
      "bot_name": "MistralAI-User",
      "provider": "mistral",
      "provider_label": "Mistral AI",
      "crawler_kind": "ai",
      "raw_purpose": "user_fetch",
      "purpose_category": "user_fetch",
      "legacy_purposes": [],
      "description_ja": "Vibeユーザーの質問を起点に、その場でWebページを取得するUser-Agent。自動巡回や生成AI学習には使わないと公式説明されています。",
      "robots_behavior": "respects",
      "official_docs_url": "https://docs.mistral.ai/robots",
      "verification_url": null,
      "source_confidence": "official",
      "verified_on": "2026-10-07",
      "note": null
    },
    {
      "match_token": "Meta-WebIndexer",
      "bot_name": "Meta-WebIndexer",
      "provider": "meta",
      "provider_label": "Meta",
      "crawler_kind": "ai",
      "raw_purpose": "search_index",
      "purpose_category": "search_index",
      "legacy_purposes": [],
      "description_ja": "Meta AIの検索品質や回答中のリンク・引用を改善するためのWebインデクサー。",
      "robots_behavior": "respects",
      "official_docs_url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/",
      "verification_url": null,
      "source_confidence": "official",
      "verified_on": "2026-10-07",
      "note": null
    },
    {
      "match_token": "Meta-ExternalFetcher",
      "bot_name": "Meta-ExternalFetcher",
      "provider": "meta",
      "provider_label": "Meta",
      "crawler_kind": "ai",
      "raw_purpose": "user_fetch",
      "purpose_category": "user_fetch",
      "legacy_purposes": [],
      "description_ja": "ユーザーのリンク操作やMeta AIのエージェント機能などを起点に外部ページを取得するUser-Agent。",
      "robots_behavior": "user_triggered_may_ignore",
      "official_docs_url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/",
      "verification_url": null,
      "source_confidence": "official",
      "verified_on": "2026-10-07",
      "note": null
    },
    {
      "match_token": "Meta-ExternalAgent",
      "bot_name": "Meta-ExternalAgent",
      "provider": "meta",
      "provider_label": "Meta",
      "crawler_kind": "ai",
      "raw_purpose": "training",
      "purpose_category": "training",
      "legacy_purposes": [
        "agent"
      ],
      "description_ja": "Metaの生成AIモデル改善やAI製品のWebコンテンツ取得に使われる外部クローラー。",
      "robots_behavior": "respects",
      "official_docs_url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/",
      "verification_url": null,
      "source_confidence": "official",
      "verified_on": "2026-10-07",
      "note": "用途はモデル改善だけでなく製品機能向けの取得も含むため、アクセス単体から実際の学習採用を断定できません。"
    },
    {
      "match_token": "Google-CloudVertexBot",
      "bot_name": "Google-CloudVertexBot",
      "provider": "google",
      "provider_label": "Google",
      "crawler_kind": "ai",
      "raw_purpose": "agent",
      "purpose_category": "other",
      "legacy_purposes": [],
      "description_ja": "サイト所有者がVertex AI Agentsを構築するときに要求されるWebクロール。",
      "robots_behavior": "respects",
      "official_docs_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
      "verification_url": null,
      "source_confidence": "official",
      "verified_on": "2026-10-07",
      "note": null
    },
    {
      "match_token": "Amazonbot",
      "bot_name": "Amazonbot",
      "provider": "amazon",
      "provider_label": "Amazon",
      "crawler_kind": "ai",
      "raw_purpose": "agent",
      "purpose_category": "other",
      "legacy_purposes": [],
      "description_ja": "Amazonの製品・サービス改善に使われる汎用クローラー。取得データがAIモデル学習に利用される可能性も公式に示されています。",
      "robots_behavior": "respects",
      "official_docs_url": "https://developer.amazon.com/amazonbot",
      "verification_url": "https://developer.amazon.com/amazonbot/ip-addresses/",
      "source_confidence": "official",
      "verified_on": "2026-10-07",
      "note": "用途が複数あるため、リモノアではTrainingへ断定せずOtherに分類します。"
    },
    {
      "match_token": "Amzn-SearchBot",
      "bot_name": "Amzn-SearchBot",
      "provider": "amazon",
      "provider_label": "Amazon",
      "crawler_kind": "search",
      "raw_purpose": "search",
      "purpose_category": "search_index",
      "legacy_purposes": [],
      "description_ja": "Amazonの検索体験を改善するための検索クローラー。生成AIモデル学習には使わないと公式説明されているため、リモノアではAI生アクセスではなく検索比較側に分類します。",
      "robots_behavior": "respects",
      "official_docs_url": "https://developer.amazon.com/amazonbot",
      "verification_url": "https://developer.amazon.com/amazonbot/searchbot-ip-addresses/",
      "source_confidence": "official",
      "verified_on": "2026-10-07",
      "note": null
    },
    {
      "match_token": "Amzn-User",
      "bot_name": "Amzn-User",
      "provider": "amazon",
      "provider_label": "Amazon",
      "crawler_kind": "ai",
      "raw_purpose": "user_fetch",
      "purpose_category": "user_fetch",
      "legacy_purposes": [],
      "description_ja": "Alexaなどのユーザー操作を起点にWebページを取得するUser-Agent。生成AIモデル学習には使わないと公式説明されています。",
      "robots_behavior": "user_triggered_may_ignore",
      "official_docs_url": "https://developer.amazon.com/amazonbot",
      "verification_url": "https://developer.amazon.com/amazonbot/live-ip-addresses/",
      "source_confidence": "official",
      "verified_on": "2026-10-07",
      "note": null
    },
    {
      "match_token": "DuckAssistBot",
      "bot_name": "DuckAssistBot",
      "provider": "duckduckgo",
      "provider_label": "DuckDuckGo",
      "crawler_kind": "ai",
      "raw_purpose": "search",
      "purpose_category": "search_index",
      "legacy_purposes": [],
      "description_ja": "DuckDuckGoのAI支援回答で、引用付きの最新情報を提供するためのクローラー。AIモデル学習には使わないと公式説明されています。",
      "robots_behavior": "respects",
      "official_docs_url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot/",
      "verification_url": "https://duckduckgo.com/duckassistbot.json",
      "source_confidence": "official",
      "verified_on": "2026-10-07",
      "note": null
    },
    {
      "match_token": "Bytespider",
      "bot_name": "Bytespider",
      "provider": "bytedance",
      "provider_label": "ByteDance",
      "crawler_kind": "ai",
      "raw_purpose": "training",
      "purpose_category": "other",
      "legacy_purposes": [],
      "description_ja": "ByteDance系として広く識別されるクローラー。リモノアでは既存観測との連続性のため検知を継続します。",
      "robots_behavior": "unknown",
      "official_docs_url": null,
      "verification_url": null,
      "source_confidence": "primary_source_unconfirmed",
      "verified_on": "2026-10-07",
      "note": "2026-10-07時点で用途を裏づける公開一次ソースを確認できていないため、raw purposeは既存互換でtrainingを維持しつつ、公開集計ではOtherとして扱います。"
    },
    {
      "match_token": "Googlebot",
      "bot_name": "Googlebot",
      "provider": "google",
      "provider_label": "Google",
      "crawler_kind": "search",
      "raw_purpose": "search_index",
      "purpose_category": "search_index",
      "legacy_purposes": [],
      "description_ja": "Google Searchのインデックス作成に使われる標準検索クローラー。",
      "robots_behavior": "respects",
      "official_docs_url": "https://developers.google.com/search/docs/crawling-indexing/googlebot",
      "verification_url": "https://developers.google.com/search/docs/crawling-indexing/verifying-googlebot",
      "source_confidence": "official",
      "verified_on": "2026-10-07",
      "note": null
    },
    {
      "match_token": "bingbot",
      "bot_name": "bingbot",
      "provider": "microsoft",
      "provider_label": "Microsoft Bing",
      "crawler_kind": "search",
      "raw_purpose": "search_index",
      "purpose_category": "search_index",
      "legacy_purposes": [],
      "description_ja": "Bing Searchのインデックス作成に使われる標準検索クローラー。",
      "robots_behavior": "respects",
      "official_docs_url": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0",
      "verification_url": "https://www.bing.com/webmasters/help/Verify-Bingbot-2195837f",
      "source_confidence": "official",
      "verified_on": "2026-10-07",
      "note": null
    },
    {
      "match_token": "Applebot",
      "bot_name": "Applebot",
      "provider": "apple",
      "provider_label": "Apple",
      "crawler_kind": "search",
      "raw_purpose": "search_index",
      "purpose_category": "search_index",
      "legacy_purposes": [],
      "description_ja": "Spotlight・Siri・SafariなどAppleの検索機能で使われるWebクローラー。",
      "robots_behavior": "respects",
      "official_docs_url": "https://support.apple.com/119829",
      "verification_url": null,
      "source_confidence": "official",
      "verified_on": "2026-10-07",
      "note": "AppleはApplebotでクロールしたデータを基盤モデル学習に利用する場合があると説明しており、学習利用はApplebot-Extendedで制御できます。"
    },
    {
      "match_token": "DuckDuckBot",
      "bot_name": "DuckDuckBot",
      "provider": "duckduckgo",
      "provider_label": "DuckDuckGo",
      "crawler_kind": "search",
      "raw_purpose": "search_index",
      "purpose_category": "search_index",
      "legacy_purposes": [],
      "description_ja": "DuckDuckGoの通常検索結果を改善するためのWebクローラー。",
      "robots_behavior": "respects",
      "official_docs_url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot/",
      "verification_url": "https://duckduckgo.com/duckduckbot.json",
      "source_confidence": "official",
      "verified_on": "2026-10-07",
      "note": null
    }
  ],
  "robots_only_control_tokens": [
    {
      "token": "Google-Extended",
      "provider": "google",
      "provider_label": "Google",
      "official_docs_url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
      "description_ja": "Gemini等の生成AI用途を制御するrobots.txtトークン。独立したHTTP User-Agentではないため、アクセスログ検知対象にはできません。"
    },
    {
      "token": "Applebot-Extended",
      "provider": "apple",
      "provider_label": "Apple",
      "official_docs_url": "https://support.apple.com/119829",
      "description_ja": "Appleの生成AI学習利用を制御するrobots.txtトークン。独立したHTTP User-Agentではありません。"
    }
  ]
}