From 5e98a29be87dd3e2169197bf8d306c4ebd714d38 Mon Sep 17 00:00:00 2001 From: yusukebe <10682+yusukebe@users.noreply.github.com> Date: Fri, 13 Nov 2026 06:47:36 +0000 Subject: [PATCH] chore(ua-blocker): update robots.json from upstream --- .changeset/auto-sync-robots.md | 5 ++++ packages/ua-blocker/src/data/robots.json | 38 ++++++++++++++++++++---- packages/ua-blocker/src/generated.ts | 12 +++++--- 3 files changed, 46 insertions(+), 9 deletions(-) create mode 100644 .changeset/auto-sync-robots.md diff --git a/.changeset/auto-sync-robots.md b/.changeset/auto-sync-robots.md new file mode 100644 index 000000000..daf4c74d5 --- /dev/null +++ b/.changeset/auto-sync-robots.md @@ -0,0 +1,5 @@ +--- +'@hono/ua-blocker': patch +--- + +chore(ua-blocker): sync `robots.json` with upstream diff --git a/packages/ua-blocker/src/data/robots.json b/packages/ua-blocker/src/data/robots.json index eb31c4c3a..05db57542 100644 --- a/packages/ua-blocker/src/data/robots.json +++ b/packages/ua-blocker/src/data/robots.json @@ -46,7 +46,7 @@ "respect": "Unclear at this time.", "function": "AI Assistants", "frequency": "Unclear at this time.", - "description": "Ai2Bot-DeepResearchEval is operated by Ai2, a non-profit AI research institute. It's used to collect and scan resources used in deep research queries performed by Ai2's o… More info can be found at https://knownagents.com/agents/ai2bot-deepresearcheval" + "description": "AI2Bot-DeepResearchEval retrieves resources used to evaluate deep-research queries performed by Ai2's open-source models. More info can be found at https://knownagents.com/agents/ai2bot-deepresearcheval" }, "Ai2Bot-Dolma": { "operator": "[Ai2](https://allenai.org/crawler)", @@ -453,7 +453,7 @@ "respect": "Yes", "function": "AI Assistants", "frequency": "Only when prompted by a user.", - "description": "Diffbot-User is used by requests originating on behalf of a human user browsing a URL using Diffbot software, in response to their input. Documented by Diffbot at https://docs.diffbot.com/docs/does-crawl-respect-robotstxt" + "description": "Diffbot-User fetches URLs on behalf of human users interacting with Diffbot software. More info can be found at https://knownagents.com/agents/diffbot-user" }, "Dot": { "operator": "Unclear at this time.", @@ -616,12 +616,19 @@ "frequency": "Unclear at this time.", "description": "Gemini CLI is an AI coding agent by Google that can query and edit large codebases, generate apps from images or PDFs, and automate complex workflows directly from the te… More info can be found at https://knownagents.com/agents/google-gemini-cli" }, + "Google-GeminiNotebook": { + "operator": "Unclear at this time.", + "respect": "Unclear at this time.", + "function": "AI Assistants", + "frequency": "Unclear at this time.", + "description": "Google-GeminiNotebook fetches individual URLs that users add as sources in Gemini Notebook so Google can analyze and summarize their content. More info can be found at https://knownagents.com/agents/google-gemininotebook" + }, "Google-NotebookLM": { "operator": "Unclear at this time.", "respect": "Unclear at this time.", "function": "AI Assistants", "frequency": "Unclear at this time.", - "description": "Google-NotebookLM is an AI-powered research and note-taking assistant that helps users synthesize information from uploaded sources like documents, transcripts, or web co… More info can be found at https://knownagents.com/agents/google-notebooklm" + "description": "Google-NotebookLM is Google's former Gemini Notebook fetcher identifier that continues to retrieve source URLs users add to their notebooks. More info can be found at https://knownagents.com/agents/google-notebooklm" }, "GoogleAgent-Mariner": { "operator": "Google", @@ -686,6 +693,13 @@ "frequency": "Unclear at this time.", "description": "Used to collect publicly available web content to support the training of Grok AI models." }, + "Haiku-SearchBot": { + "operator": "Unclear at this time.", + "respect": "Unclear at this time.", + "function": "AI Assistants", + "frequency": "Unclear at this time.", + "description": "Haiku-SearchBot retrieves webpages in response to user requests through Haiku's legal AI services. More info can be found at https://knownagents.com/agents/haiku-searchbot" + }, "HarkBot": { "operator": "Unclear at this time.", "respect": "Unclear at this time.", @@ -861,6 +875,13 @@ "frequency": "Defined per-user.", "description": "Lightpanda is a custom-built headless browser designed for AI and automation." }, + "Liner-User": { + "operator": "Unclear at this time.", + "respect": "Unclear at this time.", + "function": "AI Assistants", + "frequency": "Unclear at this time.", + "description": "Liner-User fetches web content for Liner to answer user questions with cited information. More info can be found at https://knownagents.com/agents/liner-user" + }, "LinerBot": { "operator": "Unclear at this time.", "respect": "Unclear at this time.", @@ -924,6 +945,13 @@ "frequency": "Unhinged, more than 1 per second.", "description": "As per their documentation, \"The Meta-WebIndexer crawler navigates the web to improve Meta AI search result quality for users. In doing so, Meta analyzes online content to enhance the relevance and accuracy of Meta AI. Allowing Meta-WebIndexer in your robots.txt file helps us cite and link to your content in Meta AI's responses.\"" }, + "MiniMaxAgent": { + "operator": "Unclear at this time.", + "respect": "Unclear at this time.", + "function": "AI Agents", + "frequency": "Unclear at this time.", + "description": "MiniMaxAgent browses and uses online information to complete complex user-directed tasks in MiniMax Agent. More info can be found at https://knownagents.com/agents/minimax-agent" + }, "MistralAI-Index": { "operator": "Mistral AI", "respect": "Yes", @@ -1020,7 +1048,7 @@ "respect": "Unclear at this time.", "function": "AI Agents", "frequency": "Unclear at this time.", - "description": "Nova Act is an AI agent created by Amazon that can use a web browser. It can intelligently navigate and interact with websites to complete multi-step tasks on behalf of a… More info can be found at https://knownagents.com/agents/novaact" + "description": "NovaAct is Amazon's browser-using agent for navigating websites and completing multi-step tasks on behalf of users. More info can be found at https://knownagents.com/agents/novaact" }, "OAI-AdsBot": { "operator": "[OpenAI](https://openai.com)", @@ -1104,7 +1132,7 @@ "respect": "[No](https://docs.perplexity.ai/guides/bots)", "function": "AI Assistants", "frequency": "Only when prompted by a user.", - "description": "Perplexity-User supports user actions within Perplexity. When users ask Perplexity a question, it might visit a web page to help provide an accurate answer and include a … More info can be found at https://knownagents.com/agents/perplexity-user" + "description": "Perplexity-User retrieves webpages in response to user requests so Perplexity can answer questions using current information from those pages. More info can be found at https://knownagents.com/agents/perplexity-user" }, "PerplexityBot": { "operator": "[Perplexity](https://www.perplexity.ai/)", diff --git a/packages/ua-blocker/src/generated.ts b/packages/ua-blocker/src/generated.ts index c7806cc60..f745fe599 100644 --- a/packages/ua-blocker/src/generated.ts +++ b/packages/ua-blocker/src/generated.ts @@ -88,6 +88,7 @@ User-agent: Google-CloudVertexBot User-agent: Google-Extended User-agent: Google-Firebase User-agent: Google-Gemini-CLI +User-agent: Google-GeminiNotebook User-agent: Google-NotebookLM User-agent: GoogleAgent-Mariner User-agent: GoogleAgent-URLContext @@ -98,6 +99,7 @@ User-agent: GPTBot User-agent: Grok Bot User-agent: GrokAppAndroid User-agent: GrokBot +User-agent: Haiku-SearchBot User-agent: HarkBot User-agent: HenkBot User-agent: Hunyuan @@ -123,6 +125,7 @@ User-agent: laion-huggingface-processor User-agent: LAIONDownloader User-agent: LCC User-agent: Lightpanda +User-agent: Liner-User User-agent: LinerBot User-agent: Linguee Bot User-agent: LinkupBot @@ -132,6 +135,7 @@ User-agent: Meta-ExternalAgent User-agent: meta-externalfetcher User-agent: Meta-ExternalFetcher User-agent: meta-webindexer +User-agent: MiniMaxAgent User-agent: MistralAI-Index User-agent: MistralAI-Training User-agent: MistralAI-User @@ -207,10 +211,10 @@ User-agent: ZanistaBot Disallow: / `; // prettier-ignore -export const ALL_BOTS = ["__usage_ai-training", "AddSearchBot", "AgentDataBot", "AgentTimes", "AGI Agent", "AI2Bot", "AI2Bot-DeepResearchEval", "Ai2Bot-Dolma", "aiHitBot", "AIWebIndex", "amazon-kendra", "amazon-QBusiness", "Amazonbot", "AmazonBuyForMe", "Amzn-SearchBot", "Amzn-User", "Andibot", "Anomura", "anthropic-ai", "ApifyBot", "ApifyWebsiteContentCrawler", "Applebot", "Applebot-Extended", "Aranet-SearchBot", "atlassian-bot", "Awario", "AzureAI-SearchBot", "bedrockbot", "bigsur.ai", "BixelBot", "Bravebot", "Brightbot", "Brightbot 1.0", "BuddyBot", "Bytespider", "CartAI Agentic Commerce", "CCBot", "Channel3Bot", "ChatGLM-Spider", "ChatGPT Agent", "ChatGPT-User", "Claude-Agent", "Claude-Code", "Claude-SearchBot", "Claude-User", "Claude-Web", "ClaudeBot", "CledaraBot", "Cloudflare-AutoRAG", "CloudflareBrowserRenderingCrawler", "CloudVertexBot", "Code", "Codex", "cohere-ai", "cohere-training-data-crawler", "Cotoyogi", "CragCrawler", "Crawl4AI", "Crawlspace", "Cursor", "Datenbank Crawler", "DeepSeekBot", "Devin", "Diffbot", "Diffbot-User", "Dot", "DoubaoBot", "DuckAssistBot", "Echobot Bot", "EchoboxBot", "ElevenlabsBot", "ElevenlabsCrawler", "ERNIEBot", "ExaBot", "ExaSearchBot", "ExaUser", "FacebookBot", "facebookexternalhit", "Factset_spyderbot", "FirecrawlAgent", "FriendlyCrawler", "GeistHaus-PageFetcher", "Gemini-Deep-Research", "Google-Agent", "Google-CloudVertexBot", "Google-Extended", "Google-Firebase", "Google-Gemini-CLI", "Google-NotebookLM", "GoogleAgent-Mariner", "GoogleAgent-URLContext", "GoogleOther", "GoogleOther-Image", "GoogleOther-Video", "GPTBot", "Grok Bot", "GrokAppAndroid", "GrokBot", "HarkBot", "HenkBot", "Hunyuan", "iAskBot", "iaskspider", "iaskspider/2.0", "ICC-Crawler", "ImagesiftBot", "imageSpider", "img2dataset", "Instinct", "ISSCyberRiskCrawler", "kagi-fetcher", "Kangaroo Bot", "KeenableBot", "Kimi-Agent", "Kimi-SearchBot", "Kimi-User", "KimiBot", "KlaviyoAIBot", "KunatoCrawler", "laion-huggingface-processor", "LAIONDownloader", "LCC", "Lightpanda", "LinerBot", "Linguee Bot", "LinkupBot", "Manus-User", "meta-externalagent", "Meta-ExternalAgent", "meta-externalfetcher", "Meta-ExternalFetcher", "meta-webindexer", "MistralAI-Index", "MistralAI-Training", "MistralAI-User", "MistralAI-User/1.0", "MoonshotBot", "Mozilla-Tabstack", "Muse", "MyCentralAIScraperBot", "NagetBot", "nbot", "netEstate Imprint Crawler", "newsai", "NotebookLM", "NovaAct", "OAI-AdsBot", "OAI-SearchBot", "omgili", "omgilibot", "OpenAI", "opencode", "Operator", "PanguBot", "Panscient", "panscient.com", "Payhawk Invoice Fetching Agent", "Perplexity-User", "PerplexityBot", "PetalBot", "PhindBot", "Poggio-Citations", "Poseidon Research Crawler", "qodercli", "QualifiedBot", "Querit-SearchBot", "QueritBot", "QuillBot", "quillbot.com", "QwenBot", "Reflectionbot", "SBIntuitionsBot", "Scrapy", "ScryBot", "SemrushBot-OCOB", "SemrushBot-SWA", "Shap-User", "ShapBot", "Sidetrade indexer bot", "Spider", "TavilyBot", "Terra Cotta", "TerraCotta", "Thinkbot", "TikTokSpider", "Timpibot", "TongyiBot", "Trae", "TwinAgent", "UseAI", "VelenPublicWebCrawler", "WARDBot", "Webzio-Extended", "webzio-extended", "wpbot", "WRTNBot", "xAI-Grok", "YaK", "YandexAdditional", "YandexAdditionalBot", "YiBot", "YiyanBot", "YouBot", "ZanistaBot"]; +export const ALL_BOTS = ["__usage_ai-training", "AddSearchBot", "AgentDataBot", "AgentTimes", "AGI Agent", "AI2Bot", "AI2Bot-DeepResearchEval", "Ai2Bot-Dolma", "aiHitBot", "AIWebIndex", "amazon-kendra", "amazon-QBusiness", "Amazonbot", "AmazonBuyForMe", "Amzn-SearchBot", "Amzn-User", "Andibot", "Anomura", "anthropic-ai", "ApifyBot", "ApifyWebsiteContentCrawler", "Applebot", "Applebot-Extended", "Aranet-SearchBot", "atlassian-bot", "Awario", "AzureAI-SearchBot", "bedrockbot", "bigsur.ai", "BixelBot", "Bravebot", "Brightbot", "Brightbot 1.0", "BuddyBot", "Bytespider", "CartAI Agentic Commerce", "CCBot", "Channel3Bot", "ChatGLM-Spider", "ChatGPT Agent", "ChatGPT-User", "Claude-Agent", "Claude-Code", "Claude-SearchBot", "Claude-User", "Claude-Web", "ClaudeBot", "CledaraBot", "Cloudflare-AutoRAG", "CloudflareBrowserRenderingCrawler", "CloudVertexBot", "Code", "Codex", "cohere-ai", "cohere-training-data-crawler", "Cotoyogi", "CragCrawler", "Crawl4AI", "Crawlspace", "Cursor", "Datenbank Crawler", "DeepSeekBot", "Devin", "Diffbot", "Diffbot-User", "Dot", "DoubaoBot", "DuckAssistBot", "Echobot Bot", "EchoboxBot", "ElevenlabsBot", "ElevenlabsCrawler", "ERNIEBot", "ExaBot", "ExaSearchBot", "ExaUser", "FacebookBot", "facebookexternalhit", "Factset_spyderbot", "FirecrawlAgent", "FriendlyCrawler", "GeistHaus-PageFetcher", "Gemini-Deep-Research", "Google-Agent", "Google-CloudVertexBot", "Google-Extended", "Google-Firebase", "Google-Gemini-CLI", "Google-GeminiNotebook", "Google-NotebookLM", "GoogleAgent-Mariner", "GoogleAgent-URLContext", "GoogleOther", "GoogleOther-Image", "GoogleOther-Video", "GPTBot", "Grok Bot", "GrokAppAndroid", "GrokBot", "Haiku-SearchBot", "HarkBot", "HenkBot", "Hunyuan", "iAskBot", "iaskspider", "iaskspider/2.0", "ICC-Crawler", "ImagesiftBot", "imageSpider", "img2dataset", "Instinct", "ISSCyberRiskCrawler", "kagi-fetcher", "Kangaroo Bot", "KeenableBot", "Kimi-Agent", "Kimi-SearchBot", "Kimi-User", "KimiBot", "KlaviyoAIBot", "KunatoCrawler", "laion-huggingface-processor", "LAIONDownloader", "LCC", "Lightpanda", "Liner-User", "LinerBot", "Linguee Bot", "LinkupBot", "Manus-User", "meta-externalagent", "Meta-ExternalAgent", "meta-externalfetcher", "Meta-ExternalFetcher", "meta-webindexer", "MiniMaxAgent", "MistralAI-Index", "MistralAI-Training", "MistralAI-User", "MistralAI-User/1.0", "MoonshotBot", "Mozilla-Tabstack", "Muse", "MyCentralAIScraperBot", "NagetBot", "nbot", "netEstate Imprint Crawler", "newsai", "NotebookLM", "NovaAct", "OAI-AdsBot", "OAI-SearchBot", "omgili", "omgilibot", "OpenAI", "opencode", "Operator", "PanguBot", "Panscient", "panscient.com", "Payhawk Invoice Fetching Agent", "Perplexity-User", "PerplexityBot", "PetalBot", "PhindBot", "Poggio-Citations", "Poseidon Research Crawler", "qodercli", "QualifiedBot", "Querit-SearchBot", "QueritBot", "QuillBot", "quillbot.com", "QwenBot", "Reflectionbot", "SBIntuitionsBot", "Scrapy", "ScryBot", "SemrushBot-OCOB", "SemrushBot-SWA", "Shap-User", "ShapBot", "Sidetrade indexer bot", "Spider", "TavilyBot", "Terra Cotta", "TerraCotta", "Thinkbot", "TikTokSpider", "Timpibot", "TongyiBot", "Trae", "TwinAgent", "UseAI", "VelenPublicWebCrawler", "WARDBot", "Webzio-Extended", "webzio-extended", "wpbot", "WRTNBot", "xAI-Grok", "YaK", "YandexAdditional", "YandexAdditionalBot", "YiBot", "YiyanBot", "YouBot", "ZanistaBot"]; // prettier-ignore -export const NON_RESPECTING_BOTS = ["AddSearchBot", "AgentDataBot", "AgentTimes", "AGI Agent", "AI2Bot-DeepResearchEval", "amazon-QBusiness", "AmazonBuyForMe", "Amzn-SearchBot", "Amzn-User", "Andibot", "anthropic-ai", "ApifyBot", "ApifyWebsiteContentCrawler", "Aranet-SearchBot", "Awario", "AzureAI-SearchBot", "bigsur.ai", "BixelBot", "Brightbot", "Brightbot 1.0", "BuddyBot", "Bytespider", "CartAI Agentic Commerce", "Channel3Bot", "ChatGLM-Spider", "Claude-Agent", "Claude-Code", "Claude-Web", "CledaraBot", "CloudflareBrowserRenderingCrawler", "CloudVertexBot", "Code", "Codex", "cohere-ai", "cohere-training-data-crawler", "CragCrawler", "Crawl4AI", "Cursor", "Datenbank Crawler", "DeepSeekBot", "Diffbot", "Dot", "DoubaoBot", "Echobot Bot", "EchoboxBot", "ERNIEBot", "ExaBot", "ExaSearchBot", "ExaUser", "facebookexternalhit", "Factset_spyderbot", "GeistHaus-PageFetcher", "Gemini-Deep-Research", "Google-Firebase", "Google-Gemini-CLI", "Google-NotebookLM", "GoogleAgent-Mariner", "GoogleAgent-URLContext", "Grok Bot", "GrokAppAndroid", "GrokBot", "HarkBot", "HenkBot", "Hunyuan", "iAskBot", "iaskspider", "iaskspider/2.0", "imageSpider", "img2dataset", "Instinct", "ISSCyberRiskCrawler", "kagi-fetcher", "Kangaroo Bot", "KeenableBot", "Kimi-Agent", "Kimi-User", "KunatoCrawler", "laion-huggingface-processor", "LAIONDownloader", "LCC", "Lightpanda", "LinerBot", "Linguee Bot", "LinkupBot", "Manus-User", "meta-externalfetcher", "Meta-ExternalFetcher", "meta-webindexer", "MistralAI-User", "MoonshotBot", "Muse", "MyCentralAIScraperBot", "NagetBot", "nbot", "netEstate Imprint Crawler", "newsai", "NotebookLM", "NovaAct", "OAI-AdsBot", "opencode", "Operator", "PanguBot", "Payhawk Invoice Fetching Agent", "Perplexity-User", "PhindBot", "Poggio-Citations", "Poseidon Research Crawler", "qodercli", "QualifiedBot", "Querit-SearchBot", "QueritBot", "QuillBot", "quillbot.com", "QwenBot", "Reflectionbot", "Scrapy", "ScryBot", "Shap-User", "Sidetrade indexer bot", "Spider", "TavilyBot", "Terra Cotta", "Thinkbot", "TikTokSpider", "Timpibot", "TongyiBot", "Trae", "TwinAgent", "UseAI", "WARDBot", "Webzio-Extended", "webzio-extended", "wpbot", "WRTNBot", "xAI-Grok", "YaK", "YiBot", "YiyanBot", "ZanistaBot"]; +export const NON_RESPECTING_BOTS = ["AddSearchBot", "AgentDataBot", "AgentTimes", "AGI Agent", "AI2Bot-DeepResearchEval", "amazon-QBusiness", "AmazonBuyForMe", "Amzn-SearchBot", "Amzn-User", "Andibot", "anthropic-ai", "ApifyBot", "ApifyWebsiteContentCrawler", "Aranet-SearchBot", "Awario", "AzureAI-SearchBot", "bigsur.ai", "BixelBot", "Brightbot", "Brightbot 1.0", "BuddyBot", "Bytespider", "CartAI Agentic Commerce", "Channel3Bot", "ChatGLM-Spider", "Claude-Agent", "Claude-Code", "Claude-Web", "CledaraBot", "CloudflareBrowserRenderingCrawler", "CloudVertexBot", "Code", "Codex", "cohere-ai", "cohere-training-data-crawler", "CragCrawler", "Crawl4AI", "Cursor", "Datenbank Crawler", "DeepSeekBot", "Diffbot", "Dot", "DoubaoBot", "Echobot Bot", "EchoboxBot", "ERNIEBot", "ExaBot", "ExaSearchBot", "ExaUser", "facebookexternalhit", "Factset_spyderbot", "GeistHaus-PageFetcher", "Gemini-Deep-Research", "Google-Firebase", "Google-Gemini-CLI", "Google-GeminiNotebook", "Google-NotebookLM", "GoogleAgent-Mariner", "GoogleAgent-URLContext", "Grok Bot", "GrokAppAndroid", "GrokBot", "Haiku-SearchBot", "HarkBot", "HenkBot", "Hunyuan", "iAskBot", "iaskspider", "iaskspider/2.0", "imageSpider", "img2dataset", "Instinct", "ISSCyberRiskCrawler", "kagi-fetcher", "Kangaroo Bot", "KeenableBot", "Kimi-Agent", "Kimi-User", "KunatoCrawler", "laion-huggingface-processor", "LAIONDownloader", "LCC", "Lightpanda", "Liner-User", "LinerBot", "Linguee Bot", "LinkupBot", "Manus-User", "meta-externalfetcher", "Meta-ExternalFetcher", "meta-webindexer", "MiniMaxAgent", "MistralAI-User", "MoonshotBot", "Muse", "MyCentralAIScraperBot", "NagetBot", "nbot", "netEstate Imprint Crawler", "newsai", "NotebookLM", "NovaAct", "OAI-AdsBot", "opencode", "Operator", "PanguBot", "Payhawk Invoice Fetching Agent", "Perplexity-User", "PhindBot", "Poggio-Citations", "Poseidon Research Crawler", "qodercli", "QualifiedBot", "Querit-SearchBot", "QueritBot", "QuillBot", "quillbot.com", "QwenBot", "Reflectionbot", "Scrapy", "ScryBot", "Shap-User", "Sidetrade indexer bot", "Spider", "TavilyBot", "Terra Cotta", "Thinkbot", "TikTokSpider", "Timpibot", "TongyiBot", "Trae", "TwinAgent", "UseAI", "WARDBot", "Webzio-Extended", "webzio-extended", "wpbot", "WRTNBot", "xAI-Grok", "YaK", "YiBot", "YiyanBot", "ZanistaBot"]; // prettier-ignore -export const ALL_BOTS_REGEX = /(__USAGE_AI-TRAINING|ADDSEARCHBOT|AGENTDATABOT|AGENTTIMES|AGI AGENT|AI2BOT|AI2BOT-DEEPRESEARCHEVAL|AI2BOT-DOLMA|AIHITBOT|AIWEBINDEX|AMAZON-KENDRA|AMAZON-QBUSINESS|AMAZONBOT|AMAZONBUYFORME|AMZN-SEARCHBOT|AMZN-USER|ANDIBOT|ANOMURA|ANTHROPIC-AI|APIFYBOT|APIFYWEBSITECONTENTCRAWLER|APPLEBOT|APPLEBOT-EXTENDED|ARANET-SEARCHBOT|ATLASSIAN-BOT|AWARIO|AZUREAI-SEARCHBOT|BEDROCKBOT|BIGSUR\.AI|BIXELBOT|BRAVEBOT|BRIGHTBOT|BRIGHTBOT 1\.0|BUDDYBOT|BYTESPIDER|CARTAI AGENTIC COMMERCE|CCBOT|CHANNEL3BOT|CHATGLM-SPIDER|CHATGPT AGENT|CHATGPT-USER|CLAUDE-AGENT|CLAUDE-CODE|CLAUDE-SEARCHBOT|CLAUDE-USER|CLAUDE-WEB|CLAUDEBOT|CLEDARABOT|CLOUDFLARE-AUTORAG|CLOUDFLAREBROWSERRENDERINGCRAWLER|CLOUDVERTEXBOT|CODE|CODEX|COHERE-AI|COHERE-TRAINING-DATA-CRAWLER|COTOYOGI|CRAGCRAWLER|CRAWL4AI|CRAWLSPACE|CURSOR|DATENBANK CRAWLER|DEEPSEEKBOT|DEVIN|DIFFBOT|DIFFBOT-USER|DOT|DOUBAOBOT|DUCKASSISTBOT|ECHOBOT BOT|ECHOBOXBOT|ELEVENLABSBOT|ELEVENLABSCRAWLER|ERNIEBOT|EXABOT|EXASEARCHBOT|EXAUSER|FACEBOOKBOT|FACEBOOKEXTERNALHIT|FACTSET_SPYDERBOT|FIRECRAWLAGENT|FRIENDLYCRAWLER|GEISTHAUS-PAGEFETCHER|GEMINI-DEEP-RESEARCH|GOOGLE-AGENT|GOOGLE-CLOUDVERTEXBOT|GOOGLE-EXTENDED|GOOGLE-FIREBASE|GOOGLE-GEMINI-CLI|GOOGLE-NOTEBOOKLM|GOOGLEAGENT-MARINER|GOOGLEAGENT-URLCONTEXT|GOOGLEOTHER|GOOGLEOTHER-IMAGE|GOOGLEOTHER-VIDEO|GPTBOT|GROK BOT|GROKAPPANDROID|GROKBOT|HARKBOT|HENKBOT|HUNYUAN|IASKBOT|IASKSPIDER|IASKSPIDER\/2\.0|ICC-CRAWLER|IMAGESIFTBOT|IMAGESPIDER|IMG2DATASET|INSTINCT|ISSCYBERRISKCRAWLER|KAGI-FETCHER|KANGAROO BOT|KEENABLEBOT|KIMI-AGENT|KIMI-SEARCHBOT|KIMI-USER|KIMIBOT|KLAVIYOAIBOT|KUNATOCRAWLER|LAION-HUGGINGFACE-PROCESSOR|LAIONDOWNLOADER|LCC|LIGHTPANDA|LINERBOT|LINGUEE BOT|LINKUPBOT|MANUS-USER|META-EXTERNALAGENT|META-EXTERNALAGENT|META-EXTERNALFETCHER|META-EXTERNALFETCHER|META-WEBINDEXER|MISTRALAI-INDEX|MISTRALAI-TRAINING|MISTRALAI-USER|MISTRALAI-USER\/1\.0|MOONSHOTBOT|MOZILLA-TABSTACK|MUSE|MYCENTRALAISCRAPERBOT|NAGETBOT|NBOT|NETESTATE IMPRINT CRAWLER|NEWSAI|NOTEBOOKLM|NOVAACT|OAI-ADSBOT|OAI-SEARCHBOT|OMGILI|OMGILIBOT|OPENAI|OPENCODE|OPERATOR|PANGUBOT|PANSCIENT|PANSCIENT\.COM|PAYHAWK INVOICE FETCHING AGENT|PERPLEXITY-USER|PERPLEXITYBOT|PETALBOT|PHINDBOT|POGGIO-CITATIONS|POSEIDON RESEARCH CRAWLER|QODERCLI|QUALIFIEDBOT|QUERIT-SEARCHBOT|QUERITBOT|QUILLBOT|QUILLBOT\.COM|QWENBOT|REFLECTIONBOT|SBINTUITIONSBOT|SCRAPY|SCRYBOT|SEMRUSHBOT-OCOB|SEMRUSHBOT-SWA|SHAP-USER|SHAPBOT|SIDETRADE INDEXER BOT|SPIDER|TAVILYBOT|TERRA COTTA|TERRACOTTA|THINKBOT|TIKTOKSPIDER|TIMPIBOT|TONGYIBOT|TRAE|TWINAGENT|USEAI|VELENPUBLICWEBCRAWLER|WARDBOT|WEBZIO-EXTENDED|WEBZIO-EXTENDED|WPBOT|WRTNBOT|XAI-GROK|YAK|YANDEXADDITIONAL|YANDEXADDITIONALBOT|YIBOT|YIYANBOT|YOUBOT|ZANISTABOT)/; +export const ALL_BOTS_REGEX = /(__USAGE_AI-TRAINING|ADDSEARCHBOT|AGENTDATABOT|AGENTTIMES|AGI AGENT|AI2BOT|AI2BOT-DEEPRESEARCHEVAL|AI2BOT-DOLMA|AIHITBOT|AIWEBINDEX|AMAZON-KENDRA|AMAZON-QBUSINESS|AMAZONBOT|AMAZONBUYFORME|AMZN-SEARCHBOT|AMZN-USER|ANDIBOT|ANOMURA|ANTHROPIC-AI|APIFYBOT|APIFYWEBSITECONTENTCRAWLER|APPLEBOT|APPLEBOT-EXTENDED|ARANET-SEARCHBOT|ATLASSIAN-BOT|AWARIO|AZUREAI-SEARCHBOT|BEDROCKBOT|BIGSUR\.AI|BIXELBOT|BRAVEBOT|BRIGHTBOT|BRIGHTBOT 1\.0|BUDDYBOT|BYTESPIDER|CARTAI AGENTIC COMMERCE|CCBOT|CHANNEL3BOT|CHATGLM-SPIDER|CHATGPT AGENT|CHATGPT-USER|CLAUDE-AGENT|CLAUDE-CODE|CLAUDE-SEARCHBOT|CLAUDE-USER|CLAUDE-WEB|CLAUDEBOT|CLEDARABOT|CLOUDFLARE-AUTORAG|CLOUDFLAREBROWSERRENDERINGCRAWLER|CLOUDVERTEXBOT|CODE|CODEX|COHERE-AI|COHERE-TRAINING-DATA-CRAWLER|COTOYOGI|CRAGCRAWLER|CRAWL4AI|CRAWLSPACE|CURSOR|DATENBANK CRAWLER|DEEPSEEKBOT|DEVIN|DIFFBOT|DIFFBOT-USER|DOT|DOUBAOBOT|DUCKASSISTBOT|ECHOBOT BOT|ECHOBOXBOT|ELEVENLABSBOT|ELEVENLABSCRAWLER|ERNIEBOT|EXABOT|EXASEARCHBOT|EXAUSER|FACEBOOKBOT|FACEBOOKEXTERNALHIT|FACTSET_SPYDERBOT|FIRECRAWLAGENT|FRIENDLYCRAWLER|GEISTHAUS-PAGEFETCHER|GEMINI-DEEP-RESEARCH|GOOGLE-AGENT|GOOGLE-CLOUDVERTEXBOT|GOOGLE-EXTENDED|GOOGLE-FIREBASE|GOOGLE-GEMINI-CLI|GOOGLE-GEMININOTEBOOK|GOOGLE-NOTEBOOKLM|GOOGLEAGENT-MARINER|GOOGLEAGENT-URLCONTEXT|GOOGLEOTHER|GOOGLEOTHER-IMAGE|GOOGLEOTHER-VIDEO|GPTBOT|GROK BOT|GROKAPPANDROID|GROKBOT|HAIKU-SEARCHBOT|HARKBOT|HENKBOT|HUNYUAN|IASKBOT|IASKSPIDER|IASKSPIDER\/2\.0|ICC-CRAWLER|IMAGESIFTBOT|IMAGESPIDER|IMG2DATASET|INSTINCT|ISSCYBERRISKCRAWLER|KAGI-FETCHER|KANGAROO BOT|KEENABLEBOT|KIMI-AGENT|KIMI-SEARCHBOT|KIMI-USER|KIMIBOT|KLAVIYOAIBOT|KUNATOCRAWLER|LAION-HUGGINGFACE-PROCESSOR|LAIONDOWNLOADER|LCC|LIGHTPANDA|LINER-USER|LINERBOT|LINGUEE BOT|LINKUPBOT|MANUS-USER|META-EXTERNALAGENT|META-EXTERNALAGENT|META-EXTERNALFETCHER|META-EXTERNALFETCHER|META-WEBINDEXER|MINIMAXAGENT|MISTRALAI-INDEX|MISTRALAI-TRAINING|MISTRALAI-USER|MISTRALAI-USER\/1\.0|MOONSHOTBOT|MOZILLA-TABSTACK|MUSE|MYCENTRALAISCRAPERBOT|NAGETBOT|NBOT|NETESTATE IMPRINT CRAWLER|NEWSAI|NOTEBOOKLM|NOVAACT|OAI-ADSBOT|OAI-SEARCHBOT|OMGILI|OMGILIBOT|OPENAI|OPENCODE|OPERATOR|PANGUBOT|PANSCIENT|PANSCIENT\.COM|PAYHAWK INVOICE FETCHING AGENT|PERPLEXITY-USER|PERPLEXITYBOT|PETALBOT|PHINDBOT|POGGIO-CITATIONS|POSEIDON RESEARCH CRAWLER|QODERCLI|QUALIFIEDBOT|QUERIT-SEARCHBOT|QUERITBOT|QUILLBOT|QUILLBOT\.COM|QWENBOT|REFLECTIONBOT|SBINTUITIONSBOT|SCRAPY|SCRYBOT|SEMRUSHBOT-OCOB|SEMRUSHBOT-SWA|SHAP-USER|SHAPBOT|SIDETRADE INDEXER BOT|SPIDER|TAVILYBOT|TERRA COTTA|TERRACOTTA|THINKBOT|TIKTOKSPIDER|TIMPIBOT|TONGYIBOT|TRAE|TWINAGENT|USEAI|VELENPUBLICWEBCRAWLER|WARDBOT|WEBZIO-EXTENDED|WEBZIO-EXTENDED|WPBOT|WRTNBOT|XAI-GROK|YAK|YANDEXADDITIONAL|YANDEXADDITIONALBOT|YIBOT|YIYANBOT|YOUBOT|ZANISTABOT)/; // prettier-ignore -export const NON_RESPECTING_BOTS_REGEX = /(ADDSEARCHBOT|AGENTDATABOT|AGENTTIMES|AGI AGENT|AI2BOT-DEEPRESEARCHEVAL|AMAZON-QBUSINESS|AMAZONBUYFORME|AMZN-SEARCHBOT|AMZN-USER|ANDIBOT|ANTHROPIC-AI|APIFYBOT|APIFYWEBSITECONTENTCRAWLER|ARANET-SEARCHBOT|AWARIO|AZUREAI-SEARCHBOT|BIGSUR\.AI|BIXELBOT|BRIGHTBOT|BRIGHTBOT 1\.0|BUDDYBOT|BYTESPIDER|CARTAI AGENTIC COMMERCE|CHANNEL3BOT|CHATGLM-SPIDER|CLAUDE-AGENT|CLAUDE-CODE|CLAUDE-WEB|CLEDARABOT|CLOUDFLAREBROWSERRENDERINGCRAWLER|CLOUDVERTEXBOT|CODE|CODEX|COHERE-AI|COHERE-TRAINING-DATA-CRAWLER|CRAGCRAWLER|CRAWL4AI|CURSOR|DATENBANK CRAWLER|DEEPSEEKBOT|DIFFBOT|DOT|DOUBAOBOT|ECHOBOT BOT|ECHOBOXBOT|ERNIEBOT|EXABOT|EXASEARCHBOT|EXAUSER|FACEBOOKEXTERNALHIT|FACTSET_SPYDERBOT|GEISTHAUS-PAGEFETCHER|GEMINI-DEEP-RESEARCH|GOOGLE-FIREBASE|GOOGLE-GEMINI-CLI|GOOGLE-NOTEBOOKLM|GOOGLEAGENT-MARINER|GOOGLEAGENT-URLCONTEXT|GROK BOT|GROKAPPANDROID|GROKBOT|HARKBOT|HENKBOT|HUNYUAN|IASKBOT|IASKSPIDER|IASKSPIDER\/2\.0|IMAGESPIDER|IMG2DATASET|INSTINCT|ISSCYBERRISKCRAWLER|KAGI-FETCHER|KANGAROO BOT|KEENABLEBOT|KIMI-AGENT|KIMI-USER|KUNATOCRAWLER|LAION-HUGGINGFACE-PROCESSOR|LAIONDOWNLOADER|LCC|LIGHTPANDA|LINERBOT|LINGUEE BOT|LINKUPBOT|MANUS-USER|META-EXTERNALFETCHER|META-EXTERNALFETCHER|META-WEBINDEXER|MISTRALAI-USER|MOONSHOTBOT|MUSE|MYCENTRALAISCRAPERBOT|NAGETBOT|NBOT|NETESTATE IMPRINT CRAWLER|NEWSAI|NOTEBOOKLM|NOVAACT|OAI-ADSBOT|OPENCODE|OPERATOR|PANGUBOT|PAYHAWK INVOICE FETCHING AGENT|PERPLEXITY-USER|PHINDBOT|POGGIO-CITATIONS|POSEIDON RESEARCH CRAWLER|QODERCLI|QUALIFIEDBOT|QUERIT-SEARCHBOT|QUERITBOT|QUILLBOT|QUILLBOT\.COM|QWENBOT|REFLECTIONBOT|SCRAPY|SCRYBOT|SHAP-USER|SIDETRADE INDEXER BOT|SPIDER|TAVILYBOT|TERRA COTTA|THINKBOT|TIKTOKSPIDER|TIMPIBOT|TONGYIBOT|TRAE|TWINAGENT|USEAI|WARDBOT|WEBZIO-EXTENDED|WEBZIO-EXTENDED|WPBOT|WRTNBOT|XAI-GROK|YAK|YIBOT|YIYANBOT|ZANISTABOT)/; +export const NON_RESPECTING_BOTS_REGEX = /(ADDSEARCHBOT|AGENTDATABOT|AGENTTIMES|AGI AGENT|AI2BOT-DEEPRESEARCHEVAL|AMAZON-QBUSINESS|AMAZONBUYFORME|AMZN-SEARCHBOT|AMZN-USER|ANDIBOT|ANTHROPIC-AI|APIFYBOT|APIFYWEBSITECONTENTCRAWLER|ARANET-SEARCHBOT|AWARIO|AZUREAI-SEARCHBOT|BIGSUR\.AI|BIXELBOT|BRIGHTBOT|BRIGHTBOT 1\.0|BUDDYBOT|BYTESPIDER|CARTAI AGENTIC COMMERCE|CHANNEL3BOT|CHATGLM-SPIDER|CLAUDE-AGENT|CLAUDE-CODE|CLAUDE-WEB|CLEDARABOT|CLOUDFLAREBROWSERRENDERINGCRAWLER|CLOUDVERTEXBOT|CODE|CODEX|COHERE-AI|COHERE-TRAINING-DATA-CRAWLER|CRAGCRAWLER|CRAWL4AI|CURSOR|DATENBANK CRAWLER|DEEPSEEKBOT|DIFFBOT|DOT|DOUBAOBOT|ECHOBOT BOT|ECHOBOXBOT|ERNIEBOT|EXABOT|EXASEARCHBOT|EXAUSER|FACEBOOKEXTERNALHIT|FACTSET_SPYDERBOT|GEISTHAUS-PAGEFETCHER|GEMINI-DEEP-RESEARCH|GOOGLE-FIREBASE|GOOGLE-GEMINI-CLI|GOOGLE-GEMININOTEBOOK|GOOGLE-NOTEBOOKLM|GOOGLEAGENT-MARINER|GOOGLEAGENT-URLCONTEXT|GROK BOT|GROKAPPANDROID|GROKBOT|HAIKU-SEARCHBOT|HARKBOT|HENKBOT|HUNYUAN|IASKBOT|IASKSPIDER|IASKSPIDER\/2\.0|IMAGESPIDER|IMG2DATASET|INSTINCT|ISSCYBERRISKCRAWLER|KAGI-FETCHER|KANGAROO BOT|KEENABLEBOT|KIMI-AGENT|KIMI-USER|KUNATOCRAWLER|LAION-HUGGINGFACE-PROCESSOR|LAIONDOWNLOADER|LCC|LIGHTPANDA|LINER-USER|LINERBOT|LINGUEE BOT|LINKUPBOT|MANUS-USER|META-EXTERNALFETCHER|META-EXTERNALFETCHER|META-WEBINDEXER|MINIMAXAGENT|MISTRALAI-USER|MOONSHOTBOT|MUSE|MYCENTRALAISCRAPERBOT|NAGETBOT|NBOT|NETESTATE IMPRINT CRAWLER|NEWSAI|NOTEBOOKLM|NOVAACT|OAI-ADSBOT|OPENCODE|OPERATOR|PANGUBOT|PAYHAWK INVOICE FETCHING AGENT|PERPLEXITY-USER|PHINDBOT|POGGIO-CITATIONS|POSEIDON RESEARCH CRAWLER|QODERCLI|QUALIFIEDBOT|QUERIT-SEARCHBOT|QUERITBOT|QUILLBOT|QUILLBOT\.COM|QWENBOT|REFLECTIONBOT|SCRAPY|SCRYBOT|SHAP-USER|SIDETRADE INDEXER BOT|SPIDER|TAVILYBOT|TERRA COTTA|THINKBOT|TIKTOKSPIDER|TIMPIBOT|TONGYIBOT|TRAE|TWINAGENT|USEAI|WARDBOT|WEBZIO-EXTENDED|WEBZIO-EXTENDED|WPBOT|WRTNBOT|XAI-GROK|YAK|YIBOT|YIYANBOT|ZANISTABOT)/;