diff --git a/.changeset/auto-sync-robots.md b/.changeset/auto-sync-robots.md new file mode 100644 index 000000000..daf4c74d5 --- /dev/null +++ b/.changeset/auto-sync-robots.md @@ -0,0 +1,5 @@ +--- +'@hono/ua-blocker': patch +--- + +chore(ua-blocker): sync `robots.json` with upstream diff --git a/packages/ua-blocker/src/data/robots.json b/packages/ua-blocker/src/data/robots.json index a12d49f4e..29fdf32a5 100644 --- a/packages/ua-blocker/src/data/robots.json +++ b/packages/ua-blocker/src/data/robots.json @@ -27,19 +27,19 @@ "frequency": "No information provided.", "description": "Scrapes data for AI systems." }, - "Amazonbot": { + "amazon-kendra-": { "operator": "Amazon", "respect": "Yes", - "function": "Service improvement and enabling answers for Alexa users.", + "function": "Collects data for AI natural language search", "frequency": "No information provided.", - "description": "Includes references to crawled website when surfacing answers via Alexa; does not clearly outline other uses." + "description": "Amazon Kendra is a highly accurate intelligent search service that enables your users to search unstructured data using natural language. It returns specific answers to questions, giving users an experience that's close to interacting with a human expert. It is highly scalable and capable of meeting performance demands, tightly integrated with other AWS services such as Amazon S3 and Amazon Lex, and offers enterprise-grade security." }, - "amazon-kendra-": { + "Amazonbot": { "operator": "Amazon", "respect": "Yes", - "function": "Collects data for AI natural language search", + "function": "Service improvement and enabling answers for Alexa users.", "frequency": "No information provided.", - "description": "Amazon Kendra is a highly accurate intelligent search service that enables your users to search unstructured data using natural language. It returns specific answers to questions, giving users an experience that's close to interacting with a human expert. It is highly scalable and capable of meeting performance demands, tightly integrated with other AWS services such as Amazon S3 and Amazon Lex, and offers enterprise-grade security." + "description": "Includes references to crawled website when surfacing answers via Alexa; does not clearly outline other uses." }, "Andibot": { "operator": "[Andi](https://andisearch.com/)", @@ -419,6 +419,13 @@ "frequency": "Unclear at this time.", "description": "LinerBot is the web crawler used by Liner AI assistant to gather information from academic sources and websites to provide accurate answers with line-by-line source citations for research and scholarly work. More info can be found at https://darkvisitors.com/agents/agents/linerbot" }, + "Linguee Bot": { + "operator": "[Linguee](https://www.linguee.com)", + "respect": "No", + "function": "AI powered translation service", + "frequency": "Unclear at this time.", + "description": "Linguee Bot is a web crawler used by Linguee to gather training data for its AI powered translation service." + }, "meta-externalagent": { "operator": "[Meta](https://developers.facebook.com/docs/sharing/webmasters/web-crawlers)", "respect": "Yes", diff --git a/packages/ua-blocker/src/generated.ts b/packages/ua-blocker/src/generated.ts index f6b016148..4977390f0 100644 --- a/packages/ua-blocker/src/generated.ts +++ b/packages/ua-blocker/src/generated.ts @@ -4,8 +4,8 @@ export const ROBOTS_TXT = `User-agent: AddSearchBot User-agent: AI2Bot User-agent: Ai2Bot-Dolma User-agent: aiHitBot -User-agent: Amazonbot User-agent: amazon-kendra- +User-agent: Amazonbot User-agent: Andibot User-agent: Anomura User-agent: anthropic-ai @@ -60,6 +60,7 @@ User-agent: img2dataset User-agent: ISSCyberRiskCrawler User-agent: Kangaroo Bot User-agent: LinerBot +User-agent: Linguee Bot User-agent: meta-externalagent User-agent: Meta-ExternalAgent User-agent: meta-externalfetcher @@ -107,10 +108,10 @@ User-agent: YouBot Disallow: / `; // prettier-ignore -export const ALL_BOTS = ["AddSearchBot", "AI2Bot", "Ai2Bot-Dolma", "aiHitBot", "Amazonbot", "amazon-kendra-", "Andibot", "Anomura", "anthropic-ai", "Applebot", "Applebot-Extended", "Awario", "bedrockbot", "bigsur.ai", "Bravebot", "Brightbot 1.0", "Bytespider", "CCBot", "ChatGPT Agent", "ChatGPT-User", "Claude-SearchBot", "Claude-User", "Claude-Web", "ClaudeBot", "Cloudflare-AutoRAG", "CloudVertexBot", "cohere-ai", "cohere-training-data-crawler", "Cotoyogi", "Crawlspace", "Datenbank Crawler", "DeepSeekBot", "Devin", "Diffbot", "DuckAssistBot", "Echobot Bot", "EchoboxBot", "FacebookBot", "facebookexternalhit", "Factset_spyderbot", "FirecrawlAgent", "FriendlyCrawler", "Gemini-Deep-Research", "Google-CloudVertexBot", "Google-Extended", "Google-Firebase", "Google-NotebookLM", "GoogleAgent-Mariner", "GoogleOther", "GoogleOther-Image", "GoogleOther-Video", "GPTBot", "iaskspider/2.0", "IbouBot", "ICC-Crawler", "ImagesiftBot", "img2dataset", "ISSCyberRiskCrawler", "Kangaroo Bot", "LinerBot", "meta-externalagent", "Meta-ExternalAgent", "meta-externalfetcher", "Meta-ExternalFetcher", "meta-webindexer", "MistralAI-User", "MistralAI-User/1.0", "MyCentralAIScraperBot", "netEstate Imprint Crawler", "NovaAct", "OAI-SearchBot", "omgili", "omgilibot", "OpenAI", "Operator", "PanguBot", "Panscient", "panscient.com", "Perplexity-User", "PerplexityBot", "PetalBot", "PhindBot", "Poseidon Research Crawler", "QualifiedBot", "QuillBot", "quillbot.com", "SBIntuitionsBot", "Scrapy", "SemrushBot-OCOB", "SemrushBot-SWA", "ShapBot", "Sidetrade indexer bot", "TerraCotta", "Thinkbot", "TikTokSpider", "Timpibot", "VelenPublicWebCrawler", "WARDBot", "Webzio-Extended", "wpbot", "YaK", "YandexAdditional", "YandexAdditionalBot", "YouBot"]; +export const ALL_BOTS = ["AddSearchBot", "AI2Bot", "Ai2Bot-Dolma", "aiHitBot", "amazon-kendra-", "Amazonbot", "Andibot", "Anomura", "anthropic-ai", "Applebot", "Applebot-Extended", "Awario", "bedrockbot", "bigsur.ai", "Bravebot", "Brightbot 1.0", "Bytespider", "CCBot", "ChatGPT Agent", "ChatGPT-User", "Claude-SearchBot", "Claude-User", "Claude-Web", "ClaudeBot", "Cloudflare-AutoRAG", "CloudVertexBot", "cohere-ai", "cohere-training-data-crawler", "Cotoyogi", "Crawlspace", "Datenbank Crawler", "DeepSeekBot", "Devin", "Diffbot", "DuckAssistBot", "Echobot Bot", "EchoboxBot", "FacebookBot", "facebookexternalhit", "Factset_spyderbot", "FirecrawlAgent", "FriendlyCrawler", "Gemini-Deep-Research", "Google-CloudVertexBot", "Google-Extended", "Google-Firebase", "Google-NotebookLM", "GoogleAgent-Mariner", "GoogleOther", "GoogleOther-Image", "GoogleOther-Video", "GPTBot", "iaskspider/2.0", "IbouBot", "ICC-Crawler", "ImagesiftBot", "img2dataset", "ISSCyberRiskCrawler", "Kangaroo Bot", "LinerBot", "Linguee Bot", "meta-externalagent", "Meta-ExternalAgent", "meta-externalfetcher", "Meta-ExternalFetcher", "meta-webindexer", "MistralAI-User", "MistralAI-User/1.0", "MyCentralAIScraperBot", "netEstate Imprint Crawler", "NovaAct", "OAI-SearchBot", "omgili", "omgilibot", "OpenAI", "Operator", "PanguBot", "Panscient", "panscient.com", "Perplexity-User", "PerplexityBot", "PetalBot", "PhindBot", "Poseidon Research Crawler", "QualifiedBot", "QuillBot", "quillbot.com", "SBIntuitionsBot", "Scrapy", "SemrushBot-OCOB", "SemrushBot-SWA", "ShapBot", "Sidetrade indexer bot", "TerraCotta", "Thinkbot", "TikTokSpider", "Timpibot", "VelenPublicWebCrawler", "WARDBot", "Webzio-Extended", "wpbot", "YaK", "YandexAdditional", "YandexAdditionalBot", "YouBot"]; // prettier-ignore -export const NON_RESPECTING_BOTS = ["AddSearchBot", "Andibot", "anthropic-ai", "Applebot", "Awario", "bigsur.ai", "Brightbot 1.0", "Bytespider", "Claude-Web", "CloudVertexBot", "cohere-ai", "cohere-training-data-crawler", "Datenbank Crawler", "DeepSeekBot", "Diffbot", "DuckAssistBot", "Echobot Bot", "EchoboxBot", "facebookexternalhit", "Factset_spyderbot", "Gemini-Deep-Research", "Google-Firebase", "Google-NotebookLM", "GoogleAgent-Mariner", "iaskspider/2.0", "img2dataset", "ISSCyberRiskCrawler", "Kangaroo Bot", "LinerBot", "Meta-ExternalAgent", "meta-externalfetcher", "Meta-ExternalFetcher", "meta-webindexer", "MistralAI-User", "MyCentralAIScraperBot", "netEstate Imprint Crawler", "NovaAct", "Operator", "PanguBot", "Perplexity-User", "PhindBot", "Poseidon Research Crawler", "QualifiedBot", "QuillBot", "quillbot.com", "Scrapy", "Sidetrade indexer bot", "Thinkbot", "TikTokSpider", "Timpibot", "WARDBot", "Webzio-Extended", "wpbot", "YaK"]; +export const NON_RESPECTING_BOTS = ["AddSearchBot", "Andibot", "anthropic-ai", "Applebot", "Awario", "bigsur.ai", "Brightbot 1.0", "Bytespider", "Claude-Web", "CloudVertexBot", "cohere-ai", "cohere-training-data-crawler", "Datenbank Crawler", "DeepSeekBot", "Diffbot", "DuckAssistBot", "Echobot Bot", "EchoboxBot", "facebookexternalhit", "Factset_spyderbot", "Gemini-Deep-Research", "Google-Firebase", "Google-NotebookLM", "GoogleAgent-Mariner", "iaskspider/2.0", "img2dataset", "ISSCyberRiskCrawler", "Kangaroo Bot", "LinerBot", "Linguee Bot", "Meta-ExternalAgent", "meta-externalfetcher", "Meta-ExternalFetcher", "meta-webindexer", "MistralAI-User", "MyCentralAIScraperBot", "netEstate Imprint Crawler", "NovaAct", "Operator", "PanguBot", "Perplexity-User", "PhindBot", "Poseidon Research Crawler", "QualifiedBot", "QuillBot", "quillbot.com", "Scrapy", "Sidetrade indexer bot", "Thinkbot", "TikTokSpider", "Timpibot", "WARDBot", "Webzio-Extended", "wpbot", "YaK"]; // prettier-ignore -export const ALL_BOTS_REGEX = /(ADDSEARCHBOT|AI2BOT|AI2BOT-DOLMA|AIHITBOT|AMAZONBOT|AMAZON-KENDRA-|ANDIBOT|ANOMURA|ANTHROPIC-AI|APPLEBOT|APPLEBOT-EXTENDED|AWARIO|BEDROCKBOT|BIGSUR.AI|BRAVEBOT|BRIGHTBOT 1.0|BYTESPIDER|CCBOT|CHATGPT AGENT|CHATGPT-USER|CLAUDE-SEARCHBOT|CLAUDE-USER|CLAUDE-WEB|CLAUDEBOT|CLOUDFLARE-AUTORAG|CLOUDVERTEXBOT|COHERE-AI|COHERE-TRAINING-DATA-CRAWLER|COTOYOGI|CRAWLSPACE|DATENBANK CRAWLER|DEEPSEEKBOT|DEVIN|DIFFBOT|DUCKASSISTBOT|ECHOBOT BOT|ECHOBOXBOT|FACEBOOKBOT|FACEBOOKEXTERNALHIT|FACTSET_SPYDERBOT|FIRECRAWLAGENT|FRIENDLYCRAWLER|GEMINI-DEEP-RESEARCH|GOOGLE-CLOUDVERTEXBOT|GOOGLE-EXTENDED|GOOGLE-FIREBASE|GOOGLE-NOTEBOOKLM|GOOGLEAGENT-MARINER|GOOGLEOTHER|GOOGLEOTHER-IMAGE|GOOGLEOTHER-VIDEO|GPTBOT|IASKSPIDER\/2.0|IBOUBOT|ICC-CRAWLER|IMAGESIFTBOT|IMG2DATASET|ISSCYBERRISKCRAWLER|KANGAROO BOT|LINERBOT|META-EXTERNALAGENT|META-EXTERNALAGENT|META-EXTERNALFETCHER|META-EXTERNALFETCHER|META-WEBINDEXER|MISTRALAI-USER|MISTRALAI-USER\/1.0|MYCENTRALAISCRAPERBOT|NETESTATE IMPRINT CRAWLER|NOVAACT|OAI-SEARCHBOT|OMGILI|OMGILIBOT|OPENAI|OPERATOR|PANGUBOT|PANSCIENT|PANSCIENT.COM|PERPLEXITY-USER|PERPLEXITYBOT|PETALBOT|PHINDBOT|POSEIDON RESEARCH CRAWLER|QUALIFIEDBOT|QUILLBOT|QUILLBOT.COM|SBINTUITIONSBOT|SCRAPY|SEMRUSHBOT-OCOB|SEMRUSHBOT-SWA|SHAPBOT|SIDETRADE INDEXER BOT|TERRACOTTA|THINKBOT|TIKTOKSPIDER|TIMPIBOT|VELENPUBLICWEBCRAWLER|WARDBOT|WEBZIO-EXTENDED|WPBOT|YAK|YANDEXADDITIONAL|YANDEXADDITIONALBOT|YOUBOT)/; +export const ALL_BOTS_REGEX = /(ADDSEARCHBOT|AI2BOT|AI2BOT-DOLMA|AIHITBOT|AMAZON-KENDRA-|AMAZONBOT|ANDIBOT|ANOMURA|ANTHROPIC-AI|APPLEBOT|APPLEBOT-EXTENDED|AWARIO|BEDROCKBOT|BIGSUR.AI|BRAVEBOT|BRIGHTBOT 1.0|BYTESPIDER|CCBOT|CHATGPT AGENT|CHATGPT-USER|CLAUDE-SEARCHBOT|CLAUDE-USER|CLAUDE-WEB|CLAUDEBOT|CLOUDFLARE-AUTORAG|CLOUDVERTEXBOT|COHERE-AI|COHERE-TRAINING-DATA-CRAWLER|COTOYOGI|CRAWLSPACE|DATENBANK CRAWLER|DEEPSEEKBOT|DEVIN|DIFFBOT|DUCKASSISTBOT|ECHOBOT BOT|ECHOBOXBOT|FACEBOOKBOT|FACEBOOKEXTERNALHIT|FACTSET_SPYDERBOT|FIRECRAWLAGENT|FRIENDLYCRAWLER|GEMINI-DEEP-RESEARCH|GOOGLE-CLOUDVERTEXBOT|GOOGLE-EXTENDED|GOOGLE-FIREBASE|GOOGLE-NOTEBOOKLM|GOOGLEAGENT-MARINER|GOOGLEOTHER|GOOGLEOTHER-IMAGE|GOOGLEOTHER-VIDEO|GPTBOT|IASKSPIDER\/2.0|IBOUBOT|ICC-CRAWLER|IMAGESIFTBOT|IMG2DATASET|ISSCYBERRISKCRAWLER|KANGAROO BOT|LINERBOT|LINGUEE BOT|META-EXTERNALAGENT|META-EXTERNALAGENT|META-EXTERNALFETCHER|META-EXTERNALFETCHER|META-WEBINDEXER|MISTRALAI-USER|MISTRALAI-USER\/1.0|MYCENTRALAISCRAPERBOT|NETESTATE IMPRINT CRAWLER|NOVAACT|OAI-SEARCHBOT|OMGILI|OMGILIBOT|OPENAI|OPERATOR|PANGUBOT|PANSCIENT|PANSCIENT.COM|PERPLEXITY-USER|PERPLEXITYBOT|PETALBOT|PHINDBOT|POSEIDON RESEARCH CRAWLER|QUALIFIEDBOT|QUILLBOT|QUILLBOT.COM|SBINTUITIONSBOT|SCRAPY|SEMRUSHBOT-OCOB|SEMRUSHBOT-SWA|SHAPBOT|SIDETRADE INDEXER BOT|TERRACOTTA|THINKBOT|TIKTOKSPIDER|TIMPIBOT|VELENPUBLICWEBCRAWLER|WARDBOT|WEBZIO-EXTENDED|WPBOT|YAK|YANDEXADDITIONAL|YANDEXADDITIONALBOT|YOUBOT)/; // prettier-ignore -export const NON_RESPECTING_BOTS_REGEX = /(ADDSEARCHBOT|ANDIBOT|ANTHROPIC-AI|APPLEBOT|AWARIO|BIGSUR.AI|BRIGHTBOT 1.0|BYTESPIDER|CLAUDE-WEB|CLOUDVERTEXBOT|COHERE-AI|COHERE-TRAINING-DATA-CRAWLER|DATENBANK CRAWLER|DEEPSEEKBOT|DIFFBOT|DUCKASSISTBOT|ECHOBOT BOT|ECHOBOXBOT|FACEBOOKEXTERNALHIT|FACTSET_SPYDERBOT|GEMINI-DEEP-RESEARCH|GOOGLE-FIREBASE|GOOGLE-NOTEBOOKLM|GOOGLEAGENT-MARINER|IASKSPIDER\/2.0|IMG2DATASET|ISSCYBERRISKCRAWLER|KANGAROO BOT|LINERBOT|META-EXTERNALAGENT|META-EXTERNALFETCHER|META-EXTERNALFETCHER|META-WEBINDEXER|MISTRALAI-USER|MYCENTRALAISCRAPERBOT|NETESTATE IMPRINT CRAWLER|NOVAACT|OPERATOR|PANGUBOT|PERPLEXITY-USER|PHINDBOT|POSEIDON RESEARCH CRAWLER|QUALIFIEDBOT|QUILLBOT|QUILLBOT.COM|SCRAPY|SIDETRADE INDEXER BOT|THINKBOT|TIKTOKSPIDER|TIMPIBOT|WARDBOT|WEBZIO-EXTENDED|WPBOT|YAK)/; +export const NON_RESPECTING_BOTS_REGEX = /(ADDSEARCHBOT|ANDIBOT|ANTHROPIC-AI|APPLEBOT|AWARIO|BIGSUR.AI|BRIGHTBOT 1.0|BYTESPIDER|CLAUDE-WEB|CLOUDVERTEXBOT|COHERE-AI|COHERE-TRAINING-DATA-CRAWLER|DATENBANK CRAWLER|DEEPSEEKBOT|DIFFBOT|DUCKASSISTBOT|ECHOBOT BOT|ECHOBOXBOT|FACEBOOKEXTERNALHIT|FACTSET_SPYDERBOT|GEMINI-DEEP-RESEARCH|GOOGLE-FIREBASE|GOOGLE-NOTEBOOKLM|GOOGLEAGENT-MARINER|IASKSPIDER\/2.0|IMG2DATASET|ISSCYBERRISKCRAWLER|KANGAROO BOT|LINERBOT|LINGUEE BOT|META-EXTERNALAGENT|META-EXTERNALFETCHER|META-EXTERNALFETCHER|META-WEBINDEXER|MISTRALAI-USER|MYCENTRALAISCRAPERBOT|NETESTATE IMPRINT CRAWLER|NOVAACT|OPERATOR|PANGUBOT|PERPLEXITY-USER|PHINDBOT|POSEIDON RESEARCH CRAWLER|QUALIFIEDBOT|QUILLBOT|QUILLBOT.COM|SCRAPY|SIDETRADE INDEXER BOT|THINKBOT|TIKTOKSPIDER|TIMPIBOT|WARDBOT|WEBZIO-EXTENDED|WPBOT|YAK)/;