{ "schema": 1, "version": "2026-10-28", "generated_at": "2026-10-28T17:15:47Z", "license": "CC-BY-4.0", "attribution": "Posterna operator directory (https://github.com/posternahq/posterna), CC BY 4.0. Community records derived from ai.robots.txt (https://github.com/ai-robots-txt/ai.robots.txt), MIT.", "operators": [ { "slug": "01-ai", "name": "01.ai", "website": "https://01.ai/", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "YiBot", "coverage": "community", "user_agent_tokens": [ "YiBot" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "YiBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"This bot crawls the web to gather data and content for indexation and AI model training.\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "addsearch", "name": "AddSearch", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt lists the operator as unclear; it is named in the agent's own description there.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "AddSearchBot", "coverage": "community", "user_agent_tokens": [ "AddSearchBot" ], "aliases": [], "purpose": "search", "purpose_quote": null, "search_engine": false, "robots_token": "AddSearchBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Its operator is named in its ai.robots.txt description. Function: \"AI Search Crawlers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "agentdata", "name": "AgentData", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt lists the operator as unclear; it is named in the agent's own description there.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "AgentDataBot", "coverage": "community", "user_agent_tokens": [ "AgentDataBot" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "AgentDataBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Its operator is named in its ai.robots.txt description. Function: \"AI Data Providers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "agi", "name": "AGI", "website": null, "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "AGI Agent", "coverage": "community", "user_agent_tokens": [ "AGI Agent" ], "aliases": [], "purpose": "agent", "purpose_quote": null, "search_engine": false, "robots_token": "AGI Agent", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Agents\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "ai2", "name": "Ai2", "website": null, "docs_url": "https://allenai.org/crawler", "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "AI2Bot", "coverage": "community", "user_agent_tokens": [ "AI2Bot" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "AI2Bot", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Content is used to train open language models.\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "AI2Bot-DeepResearchEval", "coverage": "community", "user_agent_tokens": [ "AI2Bot-DeepResearchEval" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": null, "search_engine": false, "robots_token": "AI2Bot-DeepResearchEval", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Assistants\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "Ai2Bot-Dolma", "coverage": "community", "user_agent_tokens": [ "Ai2Bot-Dolma" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "Ai2Bot-Dolma", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Content is used to train open language models.\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "aihit", "name": "aiHit", "website": null, "docs_url": "https://www.aihitdata.com/about", "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "aiHitBot", "coverage": "community", "user_agent_tokens": [ "aiHitBot" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "aiHitBot", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"A massive, artificial intelligence/machine learning, automated system.\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "alibaba", "name": "Alibaba", "website": null, "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "QwenBot", "coverage": "community", "user_agent_tokens": [ "QwenBot" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "QwenBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Collects public web content for Alibaba's Qwen (Tongyi Qianwen) large language models.\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "TongyiBot", "coverage": "community", "user_agent_tokens": [ "TongyiBot" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": null, "search_engine": false, "robots_token": "TongyiBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Assistants\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "amazon", "name": "Amazon", "website": "https://www.amazon.com/", "docs_url": "https://developer.amazon.com/docs/amazonbot/amazonbot.html", "coverage": "researched", "note": null, "verification": { "signing": { "status": "none_found", "directory_url": "https://kzejmtkesloc1.keydirectory.signer.ap-northeast-1.on.aws/.well-known/http-message-signatures-directory", "kid_convention": null, "purpose_field": null, "covers": [ "Amazon Bedrock AgentCore Browser" ], "note": "AWS documents that AgentCore Browser signs each request when Web Bot Auth is enabled and that a Signature-Agent header \"Points to the public key directory for verification\", but AWS names no directory host. The hosts come from a third party (mod_pagespeed probe of 19 October 2026, which took them from Cloudflare Radar's AgentCore Browser entries), one per region: kzejmtkesloc1.keydirectory.signer.ap-northeast-1.on.aws, wdfzm130yrb91.keydirectory.signer.ap-south-1.on.aws, slf696jfe4gp0.keydirectory.signer.ap-southeast-1.on.aws, meh0x7tptr6o1.keydirectory.signer.ap-southeast-2.on.aws, o42g509c7dlj6.keydirectory.signer.eu-central-1.on.aws, c3drvlj8gw240.keydirectory.signer.eu-west-1.on.aws, xhah6q48pbxb4.keydirectory.signer.us-east-1.on.aws, ogtj5xdh5udp4.keydirectory.signer.us-east-2.on.aws and bxtrz00tv0lm1.keydirectory.signer.us-west-2.on.aws, each at /.well-known/http-message-signatures-directory. The third party saw 200 with one Ed25519 key at each on 19 October 2026. On 2026-10-26 all nine returned 404 (application/json, body {\"error\":\"JWK not found\"}). The directory schema holds one URL, so only the first host is recorded here. These keys identify AWS customers' AgentCore browsers, not Amazonbot, Amzn-SearchBot or Amzn-User. The key directory URL returned HTTP 404.", "sources": [ { "url": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/browser-web-bot-auth.html", "title": "Reducing CAPTCHAs with Web Bot Auth (Amazon Bedrock AgentCore)", "fetched_at": "2026-10-28T17:14:27.840Z" }, { "url": "https://modpagespeed.com/blog/which-ai-crawlers-sign-their-requests/", "title": "55 of 150 AI crawler operators publish Web Bot Auth keys (third party, mod_pagespeed blog)", "fetched_at": "2026-10-28T17:14:32.840Z" }, { "url": "https://modpagespeed.com/data/web-bot-auth-2026-10-19/results.tsv", "title": "Web Bot Auth key directory probe results, 19 October 2026 (third party)", "fetched_at": "2026-10-28T17:14:34.046Z" } ] }, "ip_lists": [ { "agent": "amazonbot", "url": "https://developer.amazon.com/amazonbot/ip-addresses/", "format": "html-pre+ipranges-json", "prefix_count": 1292, "fetched_at": "2026-10-28T17:13:57.623Z", "sha256": "83b8eac231a9a11eb10b7b14b79ccf807c05c9686d6924825f6567bb82b909d3", "prefixes": [ "3.81.245.78/32", "3.81.253.213/32", "3.85.116.107/32", "3.86.129.126/32", "3.88.96.254/32", "3.88.245.236/32", "3.89.170.186/32", "3.89.176.255/32", "3.90.73.206/32", "3.91.138.117/32", "3.91.140.8/32", "3.92.37.4/32", "3.92.253.216/32", "3.93.98.99/32", "3.93.211.16/32", "3.93.222.50/32", "3.93.253.174/32", "3.94.40.182/32", "3.94.46.239/32", "3.94.75.111/32", "3.94.156.104/32", "3.94.157.25/32", "3.94.199.128/32", "3.208.33.245/32", "3.208.47.73/32", "3.208.129.204/32", "3.208.146.193/32", "3.208.156.9/32", "3.208.224.218/32", "3.209.2.17/32", "3.209.156.113/32", "3.209.174.110/32", "3.209.187.64/32", "3.209.221.199/32", "3.210.29.96/32", "3.210.114.189/32", "3.210.115.187/32", "3.210.123.121/32", "3.210.152.134/32", "3.210.223.61/32", "3.211.16.169/32", "3.211.90.118/32", "3.211.105.134/32", "3.211.181.86/32", "3.211.184.174/32", "3.211.196.5/32", "3.212.18.59/32", "3.212.32.46/32", "3.212.86.97/32", "3.212.136.194/32", "3.212.144.73/32", "3.212.205.90/32", "3.212.219.113/32", "3.212.237.74/32", "3.213.46.222/32", "3.213.80.185/32", "3.213.85.234/32", "3.213.106.226/32", "3.213.142.248/32", "3.213.213.161/32", "3.214.176.44/32", "3.214.218.132/32", "3.215.14.122/32", "3.215.59.93/32", "3.215.156.193/32", "3.215.221.125/32", "3.216.13.10/32", "3.216.86.144/32", "3.216.199.69/32", "3.216.227.216/32", "3.216.250.224/32", "3.217.47.0/32", "3.217.82.254/32", "3.217.171.106/32", "3.217.190.160/32", "3.217.211.204/32", "3.218.20.222/32", "3.218.35.239/32", "3.218.103.254/32", "3.218.147.175/32", "3.218.202.170/32", "3.219.80.71/32", "3.219.81.66/32", "3.219.118.96/32", "3.219.145.31/32", "3.220.55.106/32", "3.220.70.171/32", "3.220.138.53/32", "3.220.148.166/32", "3.220.175.119/32", "3.220.202.135/32", "3.221.48.218/32", "3.221.50.71/32", "3.221.156.96/32", "3.221.222.168/32", "3.221.232.188/32", "3.221.244.28/32", "3.222.85.38/32", "3.222.134.229/32", "3.222.144.151/32", "3.222.186.80/32", "3.222.190.107/32", "3.222.230.205/32", "3.223.38.249/32", "3.223.61.36/32", "3.223.107.167/32", "3.223.122.95/32", "3.223.134.5/32", "3.223.181.32/32", "3.223.229.145/32", "3.223.245.78/32", "3.224.104.67/32", "3.224.157.127/32", "3.224.191.210/32", "3.224.205.25/32", "3.224.215.150/32", "3.224.220.101/32", "3.225.9.97/32", "3.225.12.172/32", "3.225.45.252/32", "3.225.50.29/32", "3.226.34.98/32", "3.226.101.249/32", "3.226.106.93/32", "3.226.115.242/32", "3.226.160.157/32", "3.226.184.56/32", "3.227.62.11/32", "3.227.96.103/32", "3.227.180.70/32", "3.228.111.48/32", "3.228.140.40/32", "3.228.150.65/32", "3.229.2.217/32", "3.229.62.67/32", "3.229.65.105/32", "3.229.95.193/32", "3.229.128.107/32", "3.229.164.203/32", "3.229.181.245/32", "3.230.14.171/32", "3.230.43.143/32", "3.230.54.110/32", "3.230.69.161/32", "3.230.161.52/32", "3.230.224.6/32", "3.231.72.210/32", "3.231.180.166/32", "3.231.192.213/32", "3.231.193.38/32", "3.231.193.169/32", "3.232.39.98/32", "3.232.82.72/32", "3.232.84.110/32", "3.232.102.111/32", "3.232.145.99/32", "3.232.202.209/32", "3.232.208.252/32", "3.233.110.127/32", "3.234.87.64/32", "3.234.94.166/32", "3.235.215.92/32", "3.239.194.129/32", "13.216.71.17/32", "13.216.200.208/32", "13.216.205.205/32", "13.217.73.235/32", "13.219.57.118/32", "13.223.39.7/32", "13.223.113.242/32", "13.223.134.105/32", "13.223.215.220/32", "18.204.83.102/32", "18.204.89.56/32", "18.204.152.114/32", "18.204.239.30/32", "18.205.91.101/32", "18.205.127.11/32", "18.205.173.53/32", "18.205.213.231/32", "18.206.47.187/32", "18.206.80.86/32", "18.207.79.144/32", "18.207.89.138/32", "18.208.11.93/32", "18.208.15.146/32", "18.208.113.192/32", "18.209.82.211/32", "18.209.137.234/32", "18.209.198.85/32", "18.209.201.119/32", "18.210.56.100/32", "18.210.58.238/32", "18.210.77.56/32", "18.210.120.208/32", "18.210.147.97/32", "18.211.32.162/32", "18.211.39.188/32", "18.211.76.162/32", "18.211.148.239/32", "18.211.161.73/32", "18.211.191.113/32", "18.211.217.198/32", "18.211.242.14/32", "18.213.27.222/32", "18.213.70.100/32", "18.213.86.182/32", "18.213.87.246/32", "18.213.102.186/32", "18.213.123.90/32", "18.213.133.156/32", "18.213.188.154/32", "18.213.191.165/32", "18.213.240.226/32", "18.214.7.8/32", "18.214.17.122/32", "18.214.78.166/32", "18.214.106.155/32", "18.214.124.6/32", "18.214.138.148/32", "18.214.186.220/32", "18.214.195.55/32", "18.214.238.178/32", "18.214.251.19/32", "18.215.24.66/32", "18.215.31.42/32", "18.215.49.176/32", "18.215.77.19/32", "18.215.112.101/32", "18.215.119.68/32", "18.232.11.247/32", "18.232.12.157/32", "18.232.36.1/32", "18.232.191.232/32", "18.232.204.128/32", "18.233.22.168/32", "18.233.24.238/32", "18.235.81.246/32", "18.235.158.19/32", "23.20.146.223/32", "23.20.149.71/32", "23.20.178.124/32", "23.21.119.232/32", "23.21.148.226/32", "23.21.175.228/32", "23.21.179.27/32", "23.21.179.35/32", "23.21.179.120/32", "23.21.204.95/32", "23.21.205.230/32", "23.21.207.129/32", "23.21.210.40/32", "23.21.225.190/32", "23.21.227.240/32", "23.21.228.180/32", "23.21.250.48/32", "23.22.19.198/32", "23.22.20.228/32", "23.22.35.162/32", "23.22.59.87/32", "23.22.105.143/32", "23.22.141.240/32", "23.22.146.41/32", "23.22.148.181/32", "23.22.230.70/32", "23.23.14.241/32", "23.23.99.55/32", "23.23.103.31/32", "23.23.104.107/32", "23.23.137.202/32", "23.23.169.144/32", "23.23.180.225/32", "23.23.212.212/32", "23.23.213.182/32", "23.23.214.190/32", "23.23.224.159/32", "32.192.16.179/32", "32.192.112.25/32", "32.192.146.164/32", "32.192.158.119/32", "32.192.170.30/32", "32.193.7.60/32", "32.193.117.94/32", "32.193.200.204/32", "32.194.54.130/32", "32.194.70.253/32", "32.194.84.196/32", "32.194.189.87/32", "32.194.241.222/32", "32.195.25.4/32", "32.195.30.1/32", "32.195.79.67/32", "32.195.82.105/32", "32.195.85.141/32", "32.195.96.205/32", "32.195.127.227/32", "32.195.171.51/32", "32.195.185.225/32", "32.196.5.45/32", "32.196.13.177/32", "32.196.78.138/32", "32.196.99.18/32", "32.196.127.32/32", "32.196.172.181/32", "32.196.219.86/32", "32.196.220.178/32", "32.197.0.57/32", "32.197.29.81/32", "32.197.74.42/32", "32.197.108.117/32", "32.197.176.179/32", "32.198.237.6/32", "32.198.246.135/32", "32.199.124.50/32", "34.192.55.180/32", "34.192.67.98/32", "34.192.125.239/32", "34.192.126.103/32", "34.192.215.69/32", "34.192.221.205/32", "34.193.2.57/32", "34.193.23.131/32", "34.193.220.170/32", "34.193.228.159/32", "34.194.14.255/32", "34.194.95.99/32", "34.194.100.135/32", "34.194.165.45/32", "34.194.209.245/32", "34.194.225.116/32", "34.194.226.74/32", "34.194.233.48/32", "34.194.243.75/32", "34.195.60.66/32", "34.195.144.83/32", "34.195.170.146/32", "34.195.185.14/32", "34.195.191.37/32", "34.195.248.30/32", "34.196.6.199/32", "34.196.6.240/32", "34.196.30.12/32", "34.196.49.100/32", "34.196.114.170/32", "34.196.190.12/32", "34.196.237.236/32", "34.197.28.78/32", "34.197.96.67/32", "34.197.179.252/32", "34.198.44.69/32", "34.199.54.89/32", "34.199.72.6/32", "34.199.167.166/32", "34.199.252.22/32", "34.200.36.216/32", "34.200.86.246/32", "34.200.133.183/32", "34.200.169.0/32", "34.201.181.54/32", "34.202.16.142/32", "34.202.88.37/32", "34.202.103.152/32", "34.202.112.188/32", "34.202.153.229/32", "34.202.164.232/32", "34.202.182.197/32", "34.202.210.13/32", "34.203.105.116/32", "34.203.111.15/32", "34.203.118.32/32", "34.203.118.85/32", "34.203.172.148/32", "34.203.201.219/32", "34.204.150.196/32", "34.204.233.120/32", "34.205.10.151/32", "34.205.117.127/32", "34.205.122.5/32", "34.205.123.23/32", "34.205.163.103/32", "34.205.170.13/32", "34.205.175.100/32", "34.205.199.8/32", "34.205.234.123/32", "34.206.193.60/32", "34.206.205.131/32", "34.206.212.24/32", "34.206.229.66/32", "34.206.249.188/32", "34.224.9.144/32", "34.224.132.215/32", "34.224.201.217/32", "34.224.224.155/32", "34.224.232.175/32", "34.225.24.180/32", "34.225.79.179/32", "34.225.87.80/32", "34.225.138.57/32", "34.225.243.131/32", "34.226.56.197/32", "34.226.89.140/32", "34.227.156.153/32", "34.227.233.168/32", "34.227.234.246/32", "34.228.105.150/32", "34.230.124.21/32", "34.230.161.23/32", "34.230.175.224/32", "34.230.208.4/32", "34.230.245.58/32", "34.231.11.219/32", "34.231.45.47/32", "34.231.77.232/32", "34.231.118.144/32", "34.231.130.193/32", "34.231.156.59/32", "34.231.181.240/32", "34.231.219.196/32", "34.232.103.129/32", "34.232.115.106/32", "34.232.173.22/32", "34.232.249.216/32", "34.233.111.76/32", "34.233.114.237/32", "34.233.164.201/32", "34.233.171.240/32", "34.233.219.155/32", "34.234.10.40/32", "34.234.60.110/32", "34.234.121.216/32", "34.234.190.75/32", "34.234.197.175/32", "34.234.200.207/32", "34.234.206.30/32", "34.234.235.23/32", "34.235.75.8/32", "34.235.173.180/32", "34.235.178.44/32", "34.235.239.240/32", "34.236.26.60/32", "34.236.41.241/32", "34.236.82.22/32", "34.236.89.150/32", "34.236.91.13/32", "34.236.135.14/32", "34.236.185.101/32", "34.236.189.117/32", "34.236.252.251/32", "34.237.89.221/32", "34.238.45.183/32", "34.238.144.226/32", "34.239.85.139/32", "34.239.197.197/32", "34.239.229.196/32", "35.153.42.147/32", "35.153.86.40/32", "35.153.86.200/32", "35.153.155.149/32", "35.153.218.211/32", "35.168.73.117/32", "35.168.151.95/32", "35.168.238.50/32", "35.169.89.12/32", "35.169.102.85/32", "35.169.119.108/32", "35.169.209.75/32", "35.169.240.53/32", "35.170.40.230/32", "35.170.103.34/32", "35.170.116.99/32", "35.170.195.233/32", "35.170.205.140/32", "35.170.215.163/32", "35.171.117.160/32", "35.171.122.241/32", "35.171.141.42/32", "35.171.173.150/32", "35.172.91.0/32", "35.172.125.172/32", "35.173.18.61/32", "35.173.38.202/32", "35.173.60.40/32", "35.173.88.80/32", "35.173.97.68/32", "35.174.141.243/32", "35.174.150.32/32", "35.174.185.174/32", "35.174.187.247/32", "35.174.253.85/32", "35.175.85.189/32", "44.193.102.198/32", "44.193.115.232/32", "44.194.6.175/32", "44.194.64.226/32", "44.194.108.197/32", "44.194.134.53/32", "44.194.139.149/32", "44.195.50.71/32", "44.195.132.139/32", "44.195.142.56/32", "44.195.145.102/32", "44.195.163.77/32", "44.195.225.95/32", "44.196.2.129/32", "44.196.35.207/32", "44.196.118.6/32", "44.196.161.7/32", "44.197.65.237/32", "44.197.76.210/32", "44.198.40.214/32", "44.198.105.10/32", "44.198.154.81/32", "44.198.223.49/32", "44.198.231.31/32", "44.199.175.190/32", "44.205.43.166/32", "44.205.65.19/32", "44.205.74.196/32", "44.205.120.22/32", "44.205.156.100/32", "44.205.180.155/32", "44.205.192.249/32", "44.206.65.8/32", "44.206.93.215/32", "44.206.213.217/32", "44.207.29.72/32", "44.207.69.106/32", "44.207.155.89/32", "44.207.207.36/32", "44.207.209.230/32", "44.207.241.206/32", "44.207.252.58/32", "44.208.89.66/32", "44.208.131.254/32", "44.208.193.63/32", "44.208.212.174/32", "44.208.220.225/32", "44.208.223.68/32", "44.208.245.240/32", "44.209.24.209/32", "44.209.35.147/32", "44.209.80.115/32", "44.209.89.189/32", "44.209.108.13/32", "44.209.187.99/32", "44.209.254.112/32", "44.210.60.235/32", "44.210.199.190/32", "44.210.204.255/32", "44.210.213.220/32", "44.210.226.29/32", "44.211.3.154/32", "44.211.10.233/32", "44.212.106.171/32", "44.212.131.50/32", "44.212.145.46/32", "44.212.232.231/32", "44.212.242.70/32", "44.213.36.21/32", "44.213.159.215/32", "44.213.202.136/32", "44.213.202.160/32", "44.214.19.8/32", "44.214.224.199/32", "44.214.227.26/32", "44.214.237.175/32", "44.215.61.66/32", "44.215.168.29/32", "44.215.181.176/32", "44.215.210.112/32", "44.215.227.72/32", "44.215.231.15/32", "44.215.235.20/32", "44.215.239.33/32", "44.216.105.106/32", "44.216.172.204/32", "44.216.237.125/32", "44.217.12.90/32", "44.217.44.195/32", "44.217.174.180/32", "44.217.177.142/32", "44.217.255.167/32", "44.218.6.93/32", "44.218.79.142/32", "44.218.120.180/32", "44.218.170.184/32", "44.219.46.88/32", "44.219.74.122/32", "44.219.243.132/32", "44.220.2.97/32", "44.220.21.130/32", "44.221.19.65/32", "44.221.37.41/32", "44.221.105.234/32", "44.221.106.162/32", "44.221.138.137/32", "44.221.180.179/32", "44.221.227.90/32", "44.222.9.101/32", "44.223.115.10/32", "44.223.116.149/32", "44.223.116.180/32", "44.223.193.255/32", "44.223.227.26/32", "44.223.232.55/32", "50.16.43.55/32", "50.16.72.185/32", "50.16.216.166/32", "50.16.247.74/32", "50.16.248.61/32", "50.17.115.150/32", "50.17.193.48/32", "50.17.228.207/32", "50.17.246.254/32", "50.19.46.134/32", "50.19.64.10/32", "50.19.78.178/32", "50.19.79.213/32", "50.19.102.70/32", "50.19.106.86/32", "50.19.167.76/32", "50.19.221.48/32", "50.19.243.82/32", "50.19.251.21/32", "52.0.41.164/32", "52.0.63.151/32", "52.0.81.156/32", "52.0.101.108/32", "52.0.105.244/32", "52.0.108.238/32", "52.0.178.92/32", "52.0.218.219/32", "52.1.75.30/32", "52.1.84.252/32", "52.1.100.136/32", "52.1.106.130/32", "52.1.157.90/32", "52.2.4.213/32", "52.2.11.133/32", "52.2.39.252/32", "52.2.47.27/32", "52.2.58.41/32", "52.2.63.204/32", "52.2.83.227/32", "52.2.191.202/32", "52.3.26.180/32", "52.3.74.68/32", "52.3.78.144/32", "52.3.102.51/32", "52.3.104.214/32", "52.3.127.170/32", "52.3.130.157/32", "52.3.155.146/32", "52.3.156.186/32", "52.3.188.146/32", "52.4.76.156/32", "52.4.128.0/32", "52.4.143.63/32", "52.4.201.109/32", "52.4.213.199/32", "52.4.229.9/32", "52.4.238.8/32", "52.5.13.52/32", "52.5.165.11/32", "52.5.175.224/32", "52.5.221.152/32", "52.5.232.250/32", "52.5.233.62/32", "52.5.242.243/32", "52.6.5.24/32", "52.6.89.179/32", "52.6.97.88/32", "52.6.173.78/32", "52.6.216.39/32", "52.6.232.201/32", "52.6.255.193/32", "52.7.13.44/32", "52.7.13.143/32", "52.7.33.248/32", "52.7.78.54/32", "52.7.101.58/32", "52.7.232.91/32", "52.20.8.32/32", "52.20.88.157/32", "52.20.98.185/32", "52.20.137.159/32", "52.20.137.229/32", "52.20.157.145/32", "52.20.159.75/32", "52.20.203.24/32", "52.20.205.113/32", "52.20.240.226/32", "52.21.26.34/32", "52.21.46.201/32", "52.21.50.209/32", "52.21.62.139/32", "52.21.85.213/32", "52.21.109.236/32", "52.21.184.211/32", "52.21.187.65/32", "52.21.223.189/32", "52.22.2.95/32", "52.22.64.232/32", "52.22.87.224/32", "52.23.91.14/32", "52.23.112.144/32", "52.44.148.203/32", "52.44.157.127/32", "52.44.174.136/32", "52.44.229.124/32", "52.45.15.233/32", "52.45.29.57/32", "52.45.77.169/32", "52.45.92.83/32", "52.45.184.225/32", "52.45.190.80/32", "52.45.194.165/32", "52.54.15.103/32", "52.54.45.70/32", "52.54.54.197/32", "52.54.67.48/32", "52.54.95.127/32", "52.54.157.23/32", "52.54.249.218/32", "52.55.108.228/32", "52.55.208.70/32", "52.70.123.241/32", "52.70.138.176/32", "52.70.166.208/32", "52.70.169.120/32", "52.70.170.85/32", "52.70.184.86/32", "52.70.209.13/32", "52.70.240.171/32", "52.70.249.201/32", "52.70.251.201/32", "52.71.46.142/32", "52.71.123.230/32", "52.71.139.1/32", "52.71.203.206/32", "52.71.216.196/32", "52.71.218.25/32", "52.72.165.106/32", "52.72.240.198/32", "52.73.6.26/32", "52.73.52.182/32", "52.73.59.127/32", "52.73.132.106/32", "52.73.142.41/32", "52.86.84.160/32", "52.86.125.110/32", "52.86.135.254/32", "52.87.216.1/32", "52.90.14.235/32", "52.91.123.6/32", "52.200.39.39/32", "52.200.54.136/32", "52.200.58.199/32", "52.200.72.138/32", "52.200.93.170/32", "52.200.115.159/32", "52.200.142.199/32", "52.200.146.144/32", "52.200.245.197/32", "52.200.251.20/32", "52.201.0.89/32", "52.201.9.130/32", "52.201.35.30/32", "52.201.64.62/32", "52.201.155.215/32", "52.201.158.126/32", "52.201.173.132/32", "52.202.52.82/32", "52.202.229.247/32", "52.202.233.37/32", "52.202.248.123/32", "52.203.65.83/32", "52.203.68.145/32", "52.203.92.216/32", "52.203.110.191/32", "52.203.152.231/32", "52.203.175.146/32", "52.203.203.38/32", "52.203.237.170/32", "52.204.15.176/32", "52.204.37.237/32", "52.204.71.8/32", "52.204.81.148/32", "52.204.89.12/32", "52.204.174.139/32", "52.204.217.5/32", "52.204.253.129/32", "52.205.113.104/32", "52.205.141.124/32", "52.205.222.214/32", "52.205.246.252/32", "52.206.4.37/32", "52.206.52.236/32", "52.206.59.204/32", "52.206.64.59/32", "52.206.147.195/32", "52.206.204.50/32", "52.207.28.157/32", "52.207.47.227/32", "54.80.1.215/32", "54.80.73.122/32", "54.80.120.78/32", "54.80.121.108/32", "54.80.136.167/32", "54.80.185.200/32", "54.81.11.133/32", "54.81.14.82/32", "54.81.45.126/32", "54.81.134.131/32", "54.81.191.5/32", "54.81.238.39/32", "54.82.109.15/32", "54.83.11.233/32", "54.83.23.103/32", "54.83.56.1/32", "54.83.180.239/32", "54.83.199.241/32", "54.83.240.58/32", "54.84.88.22/32", "54.84.93.8/32", "54.84.102.81/32", "54.84.147.79/32", "54.84.161.62/32", "54.84.169.196/32", "54.84.205.220/32", "54.84.250.51/32", "54.85.7.119/32", "54.85.109.140/32", "54.85.126.86/32", "54.85.223.45/32", "54.85.241.125/32", "54.86.59.155/32", "54.86.197.92/32", "54.86.234.119/32", "54.86.239.192/32", "54.87.4.126/32", "54.87.62.248/32", "54.87.95.7/32", "54.87.236.2/32", "54.88.84.219/32", "54.88.188.59/32", "54.88.247.44/32", "54.89.90.224/32", "54.90.8.255/32", "54.90.21.233/32", "54.90.26.89/32", "54.90.244.132/32", "54.91.122.193/32", "54.92.171.106/32", "54.144.22.233/32", "54.144.42.14/32", "54.144.44.182/32", "54.144.185.255/32", "54.144.247.210/32", "54.145.82.217/32", "54.146.67.173/32", "54.146.84.169/32", "54.146.102.185/32", "54.146.119.126/32", "54.146.167.72/32", "54.146.185.99/32", "54.147.80.137/32", "54.147.136.146/32", "54.147.182.90/32", "54.147.238.89/32", "54.152.21.44/32", "54.152.30.242/32", "54.152.163.42/32", "54.152.194.33/32", "54.156.55.147/32", "54.156.124.2/32", "54.156.208.198/32", "54.156.234.172/32", "54.156.248.117/32", "54.157.84.74/32", "54.157.99.244/32", "54.157.174.108/32", "54.158.43.170/32", "54.158.139.192/32", "54.158.155.182/32", "54.159.18.27/32", "54.159.98.248/32", "54.159.140.186/32", "54.159.223.204/32", "54.159.245.22/32", "54.160.47.239/32", "54.160.118.78/32", "54.160.136.135/32", "54.161.158.222/32", "54.161.238.223/32", "54.161.253.173/32", "54.162.58.185/32", "54.162.69.192/32", "54.162.70.124/32", "54.163.33.176/32", "54.163.136.244/32", "54.163.169.168/32", "54.163.213.42/32", "54.164.106.236/32", "54.164.208.167/32", "54.165.11.5/32", "54.165.99.177/32", "54.165.161.174/32", "54.166.104.83/32", "54.166.126.132/32", "54.166.181.246/32", "54.167.14.199/32", "54.167.32.123/32", "54.167.81.78/32", "54.172.35.126/32", "54.172.157.35/32", "54.172.235.198/32", "54.173.224.14/32", "54.174.244.224/32", "54.175.220.249/32", "54.197.52.174/32", "54.197.82.195/32", "54.197.102.71/32", "54.197.114.76/32", "54.197.178.107/32", "54.198.33.233/32", "54.198.34.175/32", "54.204.12.115/32", "54.204.35.170/32", "54.204.62.163/32", "54.204.120.7/32", "54.204.178.26/32", "54.205.137.46/32", "54.205.150.0/32", "54.205.205.14/32", "54.208.10.176/32", "54.208.120.99/32", "54.208.131.189/32", "54.209.6.230/32", "54.209.100.30/32", "54.210.51.58/32", "54.210.77.145/32", "54.210.152.179/32", "54.210.155.69/32", "54.211.13.212/32", "54.211.195.14/32", "54.221.203.24/32", "54.221.211.160/32", "54.221.239.88/32", "54.221.245.154/32", "54.221.254.95/32", "54.224.80.42/32", "54.224.242.33/32", "54.225.81.20/32", "54.225.98.148/32", "54.225.148.123/32", "54.225.181.161/32", "54.225.199.17/32", "54.225.201.2/32", "54.226.95.110/32", "54.226.117.47/32", "54.226.201.45/32", "54.227.251.182/32", "54.234.122.80/32", "54.235.81.195/32", "54.235.125.129/32", "54.235.158.162/32", "54.235.172.96/32", "54.235.172.108/32", "54.235.191.179/32", "54.235.199.187/32", "54.236.90.11/32", "54.236.95.142/32", "54.236.100.102/32", "54.236.100.106/32", "54.236.102.157/32", "54.236.227.238/32", "54.237.219.210/32", "54.243.63.52/32", "54.243.204.71/32", "54.243.211.244/32", "67.202.25.8/32", "72.44.42.254/32", "75.101.156.93/32", "98.80.17.126/32", "98.80.60.233/32", "98.80.89.155/32", "98.80.130.239/32", "98.82.38.120/32", "98.82.39.241/32", "98.82.40.168/32", "98.82.42.197/32", "98.82.59.253/32", "98.82.63.147/32", "98.82.66.172/32", "98.82.67.109/32", "98.82.70.65/32", "98.82.88.135/32", "98.82.107.102/32", "98.82.214.73/32", "98.82.224.29/32", "98.82.226.145/32", "98.83.8.142/32", "98.83.10.183/32", "98.83.59.126/32", "98.83.72.38/32", "98.83.94.113/32", "98.83.101.133/32", "98.83.176.66/32", "98.83.177.42/32", "98.83.178.66/32", "98.83.225.16/32", "98.83.225.157/32", "98.83.226.125/32", "98.83.250.208/32", "98.84.60.17/32", "98.84.70.201/32", "98.84.131.195/32", "98.84.184.80/32", "98.84.192.142/32", "98.84.200.43/32", "98.84.206.170/32", "98.85.17.171/32", "98.85.32.206/32", "98.85.36.128/32", "98.85.170.76/32", "98.86.65.222/32", "98.86.80.124/32", "98.86.132.251/32", "98.86.203.217/32", "98.86.212.51/32", "98.87.77.225/32", "98.87.133.23/32", "98.87.141.21/32", "98.87.182.7/32", "98.87.207.126/32", "98.87.213.181/32", "98.87.238.5/32", "98.87.250.62/32", "98.88.57.244/32", "98.88.187.143/32", "98.89.99.238/32", "98.89.106.231/32", "98.89.197.159/32", "98.89.209.185/32", "98.90.98.103/32", "98.90.110.79/32", "98.90.125.157/32", "98.90.148.129/32", "98.90.215.62/32", "98.90.249.120/32", "98.91.44.166/32", "98.91.72.186/32", "98.91.94.136/32", "98.91.104.214/32", "98.91.110.187/32", "98.91.111.24/32", "98.91.143.225/32", "98.94.138.179/32", "98.94.142.186/32", "98.95.124.8/32", "100.24.90.135/32", "100.24.134.117/32", "100.24.149.244/32", "100.24.167.60/32", "100.25.9.234/32", "100.25.96.137/32", "100.25.103.91/32", "100.25.120.246/32", "100.26.7.250/32", "100.27.71.33/32", "100.27.79.68/32", "100.27.149.198/32", "100.27.153.9/32", "100.27.161.155/32", "100.28.44.58/32", "100.28.49.152/32", "100.28.57.133/32", "100.28.118.16/32", "100.28.133.214/32", "100.28.204.82/32", "100.28.245.118/32", "100.29.34.97/32", "100.29.48.32/32", "100.29.63.24/32", "100.29.70.220/32", "100.29.74.61/32", "100.29.107.38/32", "100.29.116.208/32", "100.29.126.60/32", "100.29.128.75/32", "100.29.148.60/32", "100.29.155.89/32", "100.29.160.53/32", "100.29.164.142/32", "100.29.164.178/32", "100.30.8.247/32", "100.30.43.47/32", "100.30.51.229/32", "100.30.88.88/32", "100.30.157.22/32", "100.31.32.118/32", "100.31.229.251/32", "100.48.38.64/32", "100.48.40.16/32", "100.48.126.46/32", "100.48.145.68/32", "100.48.254.147/32", "100.49.9.181/32", "100.49.11.239/32", "100.49.26.210/32", "100.49.87.166/32", "100.50.57.146/32", "100.50.82.187/32", "100.50.88.53/32", "100.50.92.12/32", "100.50.101.121/32", "100.50.195.113/32", "100.50.204.204/32", "100.51.28.129/32", "100.51.30.134/32", "100.51.131.3/32", "100.51.145.133/32", "100.51.173.237/32", "100.51.246.38/32", "100.51.251.51/32", "100.52.10.132/32", "100.52.56.144/32", "100.52.80.224/32", "100.52.115.57/32", "100.52.138.120/32", "100.52.138.205/32", "100.55.106.152/32", "100.55.188.3/32", "100.55.234.220/32", "100.56.51.40/32", "100.56.99.138/32", "100.56.112.244/32", "100.56.127.196/32", "100.56.142.210/32", "100.56.151.105/32", "100.56.195.235/32", "100.57.46.166/32", "100.57.60.108/32", "100.57.69.236/32", "100.57.70.187/32", "100.57.145.59/32", "100.57.194.146/32", "100.57.211.199/32", "100.57.216.250/32", "100.57.246.13/32", "100.58.4.37/32", "100.58.45.198/32", "100.59.145.242/32", "100.59.209.89/32", "100.59.227.61/32", "100.60.1.224/32", "100.60.51.118/32", "100.60.153.145/32", "100.60.159.110/32", "100.60.164.156/32", "100.60.181.98/32", "100.60.212.69/32", "100.60.218.80/32", "100.60.224.239/32", "100.60.230.7/32", "100.60.230.43/32", "100.61.20.229/32", "100.61.24.214/32", "100.61.64.186/32", "100.61.110.40/32", "100.61.251.108/32", "100.61.252.112/32", "100.62.132.114/32", "100.62.191.85/32", "100.62.205.149/32", "100.62.208.199/32", "100.62.224.15/32", "100.62.247.207/32", "100.63.9.237/32", "100.63.38.186/32", "100.63.167.112/32", "100.63.217.53/32", "100.63.226.160/32", "100.63.227.156/32", "100.63.236.174/32", "100.63.239.85/32", "107.20.25.33/32", "107.20.181.148/32", "107.20.224.184/32", "107.20.255.194/32", "107.21.4.76/32", "107.21.49.20/32", "107.21.171.94/32", "107.22.54.180/32", "107.22.208.39/32", "107.22.219.12/32", "107.23.62.75/32", "107.23.208.98/32", "174.129.20.137/32", "174.129.115.115/32", "174.129.131.108/32", "174.129.189.60/32", "174.129.242.195/32", "184.72.84.154/32", "184.72.95.195/32", "184.72.140.245/32", "184.73.35.182/32", "184.73.47.24/32", "184.73.68.20/32", "184.73.98.121/32", "184.73.126.164/32", "184.73.167.217/32", "184.73.175.194/32", "184.73.195.18/32", "184.73.216.192/32", "184.73.232.51/32", "184.73.239.35/32", "184.192.10.131/32", "184.192.13.104/32", "184.192.16.41/32", "184.192.17.38/32", "184.192.22.162/32", "184.192.32.200/32", "184.192.37.104/32", "184.192.45.148/32", "184.192.57.50/32", "184.192.58.249/32", "184.192.60.239/32", "184.192.65.160/32", "184.192.71.112/32", "184.192.75.146/32", "184.192.76.56/32", "184.192.87.53/32", "184.192.89.251/32", "184.192.90.88/32", "184.192.91.223/32", "184.192.111.161/32", "184.192.114.128/32", "184.192.119.173/32", "184.192.123.97/32", "184.192.124.134/32", "184.192.127.115/32", "184.192.132.218/32", "184.192.133.10/32", "184.192.138.135/32", "184.192.139.185/32", "184.192.142.177/32", "184.192.142.248/32", "184.192.145.41/32", "184.192.150.219/32", "184.192.153.80/32", "184.192.160.199/32", "184.192.161.161/32", "184.192.163.135/32", "184.192.166.248/32", "184.192.167.118/32", "184.192.172.179/32", "184.192.174.76/32", "184.192.175.55/32", "184.192.176.204/32", "184.192.180.115/32", "184.192.184.46/32", "184.192.189.128/32", "184.192.199.116/32", "184.192.199.181/32", "184.192.200.93/32", "184.192.200.176/32", "184.192.200.235/32", "184.192.202.136/32", "184.192.203.238/32", "184.192.210.213/32", "184.192.212.143/32", "184.192.212.212/32", "184.192.218.172/32", "184.192.228.95/32", "184.192.232.81/32", "184.192.232.184/32", "184.192.233.81/32", "184.192.233.216/32", "184.192.234.12/32", "184.192.234.21/32", "184.192.234.129/32", "184.192.234.200/32", "184.192.234.218/32", "184.192.234.235/32", "184.192.235.64/32", "184.192.235.146/32", "184.192.235.180/32", "184.192.236.147/32", "184.192.236.194/32", "184.192.236.219/32" ], "note": null, "sources": [ { "url": "https://developer.amazon.com/docs/amazonbot/amazonbot.html", "title": "Amazon Web Search crawlers | Amazonbot", "fetched_at": "2026-10-28T17:14:02.251Z" } ] }, { "agent": "amzn-searchbot", "url": "https://developer.amazon.com/amazonbot/searchbot-ip-addresses/", "format": "html-pre+ipranges-json", "prefix_count": 816, "fetched_at": "2026-10-28T17:13:59.048Z", "sha256": "94d6892c7287075a1bf192cf31115bdb78947482f82fbf16a0d08754edf7d104", "prefixes": [ "3.82.67.224/32", "3.82.90.168/32", "3.83.75.212/32", "3.85.141.163/32", "3.86.74.89/32", "3.89.17.222/32", "3.89.155.158/32", "3.90.92.205/32", "3.90.117.221/32", "3.91.142.95/32", "3.91.164.253/32", "3.92.140.162/32", "3.93.116.95/32", "3.93.120.17/32", "3.94.185.94/32", "3.208.37.94/32", "3.208.113.39/32", "3.209.57.4/32", "3.209.71.33/32", "3.209.162.193/32", "3.210.17.17/32", "3.210.117.45/32", "3.210.238.203/32", "3.210.239.195/32", "3.211.100.186/32", "3.211.143.75/32", "3.211.158.39/32", "3.211.227.132/32", "3.212.138.251/32", "3.212.155.47/32", "3.213.103.242/32", "3.213.110.122/32", "3.213.117.114/32", "3.213.253.252/32", "3.214.51.149/32", "3.214.131.136/32", "3.214.235.2/32", "3.214.252.9/32", "3.215.155.189/32", "3.215.205.141/32", "3.216.26.183/32", "3.216.221.144/32", "3.216.244.57/32", "3.216.253.252/32", "3.217.20.26/32", "3.217.34.58/32", "3.217.34.183/32", "3.217.63.43/32", "3.217.66.30/32", "3.217.161.64/32", "3.218.2.114/32", "3.218.37.5/32", "3.219.6.124/32", "3.219.192.9/32", "3.219.192.12/32", "3.219.223.95/32", "3.220.71.23/32", "3.221.35.158/32", "3.221.56.120/32", "3.221.61.12/32", "3.221.116.68/32", "3.221.121.46/32", "3.221.167.25/32", "3.221.254.28/32", "3.222.16.22/32", "3.222.19.80/32", "3.222.127.245/32", "3.222.196.181/32", "3.222.252.213/32", "3.223.42.33/32", "3.223.100.234/32", "3.223.139.87/32", "3.223.155.74/32", "3.223.242.201/32", "3.224.45.108/32", "3.224.73.194/32", "3.224.123.189/32", "3.224.159.71/32", "3.224.205.7/32", "3.224.216.148/32", "3.225.111.192/32", "3.225.136.217/32", "3.225.200.77/32", "3.225.209.122/32", "3.226.3.62/32", "3.226.50.226/32", "3.226.139.230/32", "3.226.196.238/32", "3.226.229.153/32", "3.227.142.177/32", "3.227.227.6/32", "3.228.117.65/32", "3.228.151.92/32", "3.228.206.225/32", "3.229.67.114/32", "3.229.72.127/32", "3.229.219.29/32", "3.230.59.35/32", "3.230.179.91/32", "3.230.230.154/32", "3.230.250.35/32", "3.230.253.54/32", "3.231.178.162/32", "3.231.188.44/32", "3.232.18.244/32", "3.232.204.221/32", "3.232.253.211/32", "3.233.175.168/32", "3.233.189.187/32", "3.234.13.59/32", "3.234.28.73/32", "3.234.152.213/32", "13.216.24.101/32", "13.216.102.104/32", "13.216.187.148/32", "13.216.196.168/32", "13.219.54.187/32", "13.219.152.101/32", "13.219.170.211/32", "13.219.196.61/32", "13.222.91.251/32", "13.223.109.92/32", "13.223.163.204/32", "13.223.222.73/32", "18.204.69.251/32", "18.204.90.3/32", "18.205.13.0/32", "18.205.44.190/32", "18.205.140.254/32", "18.205.180.59/32", "18.206.50.226/32", "18.206.66.34/32", "18.206.69.92/32", "18.206.202.183/32", "18.207.88.6/32", "18.208.43.113/32", "18.208.95.0/32", "18.209.135.224/32", "18.209.142.234/32", "18.209.178.202/32", "18.210.45.73/32", "18.210.115.131/32", "18.210.225.20/32", "18.211.141.154/32", "18.213.84.33/32", "18.213.133.4/32", "18.213.249.250/32", "18.214.84.4/32", "18.214.124.101/32", "18.214.125.232/32", "18.214.186.114/32", "18.214.206.100/32", "18.215.42.159/32", "18.215.99.115/32", "18.215.135.45/32", "18.215.215.162/32", "18.233.44.129/32", "18.233.214.108/32", "18.233.235.66/32", "18.234.249.23/32", "18.235.234.69/32", "23.20.194.146/32", "23.21.30.253/32", "23.21.72.195/32", "23.21.115.175/32", "23.21.214.230/32", "23.21.236.157/32", "23.21.254.26/32", "23.22.4.24/32", "23.22.245.96/32", "23.22.246.84/32", "23.23.97.185/32", "23.23.138.175/32", "23.23.169.169/32", "23.23.188.58/32", "23.23.201.125/32", "23.23.214.73/32", "32.193.207.248/32", "32.193.230.132/32", "32.194.42.66/32", "32.196.238.12/32", "32.197.85.247/32", "32.197.106.56/32", "32.198.161.214/32", "32.199.114.136/32", "32.199.225.217/32", "34.192.94.60/32", "34.192.213.30/32", "34.192.224.176/32", "34.193.28.138/32", "34.193.137.18/32", "34.193.176.214/32", "34.194.81.161/32", "34.194.96.61/32", "34.194.165.188/32", "34.194.196.150/32", "34.194.236.194/32", "34.194.237.29/32", "34.195.79.160/32", "34.195.90.205/32", "34.195.194.181/32", "34.196.22.75/32", "34.196.127.79/32", "34.196.221.51/32", "34.197.25.140/32", "34.197.27.62/32", "34.197.154.32/32", "34.197.240.80/32", "34.197.246.201/32", "34.198.33.203/32", "34.198.41.184/32", "34.198.117.140/32", "34.198.223.196/32", "34.199.0.168/32", "34.199.2.135/32", "34.199.111.130/32", "34.199.134.92/32", "34.200.55.56/32", "34.200.59.71/32", "34.200.178.28/32", "34.200.190.72/32", "34.201.89.57/32", "34.201.229.234/32", "34.201.230.145/32", "34.203.29.230/32", "34.203.33.121/32", "34.204.31.92/32", "34.204.82.126/32", "34.204.145.175/32", "34.204.234.171/32", "34.206.13.250/32", "34.206.127.169/32", "34.206.160.62/32", "34.206.189.108/32", "34.207.31.244/32", "34.207.190.20/32", "34.224.5.130/32", "34.224.43.52/32", "34.224.104.19/32", "34.224.135.86/32", "34.224.242.232/32", "34.225.204.15/32", "34.225.220.51/32", "34.225.241.157/32", "34.226.77.125/32", "34.226.171.86/32", "34.226.176.13/32", "34.227.89.68/32", "34.227.210.84/32", "34.227.211.218/32", "34.228.0.97/32", "34.228.124.71/32", "34.229.27.119/32", "34.229.55.190/32", "34.230.225.128/32", "34.231.91.210/32", "34.231.191.101/32", "34.231.198.5/32", "34.232.233.214/32", "34.233.138.154/32", "34.233.177.9/32", "34.233.191.178/32", "34.233.249.54/32", "34.234.21.24/32", "34.234.177.179/32", "34.234.182.15/32", "34.234.221.209/32", "34.235.6.168/32", "34.235.19.146/32", "34.235.221.79/32", "34.235.227.102/32", "34.236.65.219/32", "34.236.66.23/32", "34.236.163.131/32", "34.237.22.79/32", "34.237.37.16/32", "34.238.12.204/32", "34.238.63.49/32", "34.238.66.168/32", "34.238.75.183/32", "34.238.132.4/32", "34.239.77.160/32", "34.239.87.171/32", "34.239.90.1/32", "34.239.253.240/32", "35.153.187.89/32", "35.168.48.89/32", "35.168.75.176/32", "35.168.147.87/32", "35.168.181.24/32", "35.169.1.17/32", "35.169.2.39/32", "35.169.2.149/32", "35.169.74.227/32", "35.169.106.154/32", "35.169.110.137/32", "35.170.104.4/32", "35.170.105.219/32", "35.170.123.50/32", "35.170.123.152/32", "35.171.37.0/32", "35.171.77.201/32", "35.171.190.227/32", "35.171.199.44/32", "35.171.200.29/32", "35.172.15.28/32", "35.172.125.46/32", "35.172.188.13/32", "35.173.101.225/32", "35.173.123.161/32", "35.174.28.56/32", "35.175.65.253/32", "35.175.132.11/32", "44.193.34.148/32", "44.193.96.92/32", "44.194.47.12/32", "44.194.84.22/32", "44.194.124.103/32", "44.194.165.181/32", "44.195.71.32/32", "44.195.217.176/32", "44.196.9.78/32", "44.196.83.48/32", "44.196.116.215/32", "44.196.118.139/32", "44.196.235.241/32", "44.197.4.190/32", "44.197.254.236/32", "44.198.66.156/32", "44.198.230.203/32", "44.199.62.58/32", "44.199.112.216/32", "44.199.145.199/32", "44.205.173.56/32", "44.205.208.84/32", "44.206.81.21/32", "44.206.127.11/32", "44.206.217.175/32", "44.207.64.241/32", "44.208.55.64/32", "44.208.90.52/32", "44.208.239.245/32", "44.209.121.79/32", "44.209.167.138/32", "44.209.243.164/32", "44.210.22.203/32", "44.210.41.97/32", "44.210.169.22/32", "44.210.170.50/32", "44.210.179.109/32", "44.210.189.34/32", "44.210.225.188/32", "44.211.6.49/32", "44.212.202.62/32", "44.212.207.23/32", "44.212.241.192/32", "44.212.251.138/32", "44.213.56.230/32", "44.213.206.151/32", "44.213.211.166/32", "44.213.228.74/32", "44.213.241.45/32", "44.215.74.155/32", "44.215.99.191/32", "44.215.159.243/32", "44.215.171.38/32", "44.216.105.237/32", "44.216.173.42/32", "44.216.177.139/32", "44.216.239.225/32", "44.217.43.241/32", "44.217.64.212/32", "44.217.156.115/32", "44.217.164.115/32", "44.217.204.19/32", "44.217.221.211/32", "44.217.223.236/32", "44.217.237.69/32", "44.217.251.254/32", "44.218.56.203/32", "44.218.61.226/32", "44.218.248.140/32", "44.218.253.125/32", "44.219.149.9/32", "44.219.225.200/32", "44.219.225.212/32", "44.221.129.95/32", "44.221.203.99/32", "44.221.241.220/32", "44.221.244.76/32", "44.223.253.120/32", "50.17.88.23/32", "50.19.98.34/32", "50.19.173.230/32", "52.0.114.157/32", "52.0.207.54/32", "52.0.236.109/32", "52.1.97.122/32", "52.1.233.104/32", "52.2.191.37/32", "52.3.17.185/32", "52.3.30.234/32", "52.3.62.173/32", "52.3.91.235/32", "52.3.193.39/32", "52.3.251.73/32", "52.4.189.224/32", "52.4.251.246/32", "52.5.12.245/32", "52.5.20.105/32", "52.5.45.215/32", "52.5.113.11/32", "52.6.24.217/32", "52.6.54.116/32", "52.6.57.17/32", "52.6.171.10/32", "52.6.216.48/32", "52.6.231.0/32", "52.7.48.229/32", "52.7.126.210/32", "52.7.171.101/32", "52.7.225.39/32", "52.7.227.80/32", "52.7.249.12/32", "52.20.27.34/32", "52.20.85.108/32", "52.20.124.247/32", "52.20.158.171/32", "52.20.234.254/32", "52.21.93.114/32", "52.21.99.253/32", "52.21.133.215/32", "52.22.52.60/32", "52.22.75.143/32", "52.22.233.217/32", "52.23.69.1/32", "52.44.115.134/32", "52.44.199.164/32", "52.45.83.81/32", "52.45.194.99/32", "52.45.211.110/32", "52.54.130.150/32", "52.55.107.99/32", "52.70.16.42/32", "52.70.37.172/32", "52.70.95.48/32", "52.70.226.93/32", "52.71.16.223/32", "52.71.72.39/32", "52.71.144.112/32", "52.71.159.156/32", "52.71.205.86/32", "52.71.243.5/32", "52.72.227.157/32", "52.73.35.133/32", "52.87.75.190/32", "52.87.216.218/32", "52.200.64.94/32", "52.200.125.80/32", "52.200.198.83/32", "52.200.202.121/32", "52.200.244.204/32", "52.201.26.95/32", "52.201.77.2/32", "52.201.114.98/32", "52.201.149.75/32", "52.202.15.97/32", "52.202.30.225/32", "52.202.160.12/32", "52.202.182.148/32", "52.202.234.14/32", "52.203.139.107/32", "52.203.235.174/32", "52.204.1.121/32", "52.204.203.171/32", "52.205.16.194/32", "52.206.29.234/32", "52.206.59.229/32", "52.206.116.153/32", "52.206.129.89/32", "52.206.188.45/32", "52.207.6.18/32", "52.207.43.63/32", "54.80.143.10/32", "54.81.2.245/32", "54.81.15.167/32", "54.81.32.142/32", "54.81.126.112/32", "54.82.2.214/32", "54.83.18.137/32", "54.83.58.194/32", "54.83.126.221/32", "54.83.172.18/32", "54.84.71.94/32", "54.84.83.113/32", "54.84.160.219/32", "54.84.185.230/32", "54.85.38.148/32", "54.85.95.196/32", "54.85.178.213/32", "54.87.216.234/32", "54.87.255.90/32", "54.88.60.135/32", "54.88.236.97/32", "54.89.16.186/32", "54.89.45.9/32", "54.89.69.250/32", "54.89.70.93/32", "54.90.18.3/32", "54.90.28.43/32", "54.90.227.223/32", "54.91.249.2/32", "54.92.148.34/32", "54.92.203.235/32", "54.145.34.70/32", "54.145.161.111/32", "54.146.71.85/32", "54.147.14.240/32", "54.147.66.130/32", "54.147.119.34/32", "54.147.229.64/32", "54.157.89.253/32", "54.157.138.252/32", "54.157.208.183/32", "54.158.133.188/32", "54.158.154.192/32", "54.159.237.70/32", "54.160.45.52/32", "54.161.25.53/32", "54.161.157.185/32", "54.162.54.187/32", "54.162.153.212/32", "54.163.22.228/32", "54.163.82.179/32", "54.163.123.236/32", "54.165.95.216/32", "54.165.171.73/32", "54.165.202.67/32", "54.165.205.16/32", "54.166.24.13/32", "54.166.70.223/32", "54.167.5.166/32", "54.167.94.112/32", "54.167.133.42/32", "54.167.188.181/32", "54.172.50.210/32", "54.172.92.66/32", "54.172.176.34/32", "54.173.156.58/32", "54.173.160.86/32", "54.173.161.122/32", "54.174.127.227/32", "54.174.161.242/32", "54.175.167.28/32", "54.196.187.118/32", "54.204.10.196/32", "54.204.14.125/32", "54.204.105.49/32", "54.204.159.166/32", "54.204.189.191/32", "54.205.6.162/32", "54.205.7.250/32", "54.205.137.236/32", "54.208.186.113/32", "54.208.229.84/32", "54.209.33.199/32", "54.209.59.28/32", "54.209.67.252/32", "54.209.139.3/32", "54.210.16.158/32", "54.210.47.231/32", "54.210.51.122/32", "54.221.233.6/32", "54.224.159.224/32", "54.225.210.250/32", "54.225.211.168/32", "54.225.220.226/32", "54.226.54.148/32", "54.227.95.204/32", "54.227.122.71/32", "54.227.250.126/32", "54.234.109.91/32", "54.234.156.138/32", "54.234.227.173/32", "54.235.183.179/32", "54.235.187.99/32", "54.236.77.176/32", "54.236.90.239/32", "54.236.146.97/32", "54.236.180.98/32", "54.236.184.238/32", "54.236.251.98/32", "54.237.229.245/32", "54.242.155.65/32", "54.243.78.23/32", "54.243.119.155/32", "54.243.128.188/32", "54.243.172.250/32", "54.243.185.76/32", "54.243.211.63/32", "72.44.45.175/32", "75.101.144.251/32", "98.80.36.221/32", "98.80.83.102/32", "98.82.61.199/32", "98.82.68.7/32", "98.82.78.131/32", "98.82.83.103/32", "98.82.193.30/32", "98.82.238.39/32", "98.83.194.21/32", "98.83.194.176/32", "98.84.79.84/32", "98.84.86.216/32", "98.84.185.30/32", "98.84.242.117/32", "98.85.153.239/32", "98.85.191.237/32", "98.85.194.38/32", "98.86.33.14/32", "98.86.38.110/32", "98.86.69.65/32", "98.86.83.164/32", "98.86.93.251/32", "98.86.101.20/32", "98.86.133.40/32", "98.86.217.46/32", "98.87.182.1/32", "98.87.203.235/32", "98.87.218.210/32", "98.88.3.219/32", "98.88.4.13/32", "98.88.4.160/32", "98.88.43.210/32", "98.88.67.17/32", "98.88.114.29/32", "98.88.146.174/32", "98.88.180.222/32", "98.88.186.79/32", "98.88.219.59/32", "98.88.224.25/32", "98.88.234.184/32", "98.88.240.41/32", "98.89.74.239/32", "98.89.89.208/32", "98.89.110.216/32", "98.89.176.131/32", "98.89.191.68/32", "98.89.198.186/32", "98.89.238.137/32", "98.89.254.223/32", "98.89.255.185/32", "98.90.1.201/32", "98.90.53.3/32", "98.90.53.72/32", "98.90.93.167/32", "98.90.129.48/32", "98.90.132.132/32", "98.90.144.220/32", "98.90.155.195/32", "98.90.166.177/32", "98.90.171.206/32", "98.90.212.161/32", "98.91.3.186/32", "98.91.9.192/32", "98.91.81.246/32", "98.91.91.18/32", "98.91.104.211/32", "98.91.120.100/32", "98.91.142.137/32", "98.91.147.104/32", "98.91.150.214/32", "98.91.160.54/32", "98.91.171.8/32", "98.94.116.10/32", "98.94.126.135/32", "98.94.137.118/32", "98.94.149.8/32", "98.94.158.230/32", "98.94.164.245/32", "98.94.174.49/32", "98.94.174.228/32", "98.94.180.226/32", "98.94.182.242/32", "98.94.188.12/32", "98.94.193.28/32", "98.94.196.171/32", "98.94.202.176/32", "98.94.203.213/32", "98.94.208.188/32", "98.94.209.71/32", "98.94.216.122/32", "98.94.225.199/32", "98.94.225.207/32", "98.94.228.253/32", "98.94.237.222/32", "98.94.240.137/32", "98.94.246.215/32", "98.94.252.84/32", "98.94.253.137/32", "98.95.1.231/32", "98.95.7.71/32", "98.95.20.69/32", "98.95.23.214/32", "98.95.23.228/32", "98.95.34.178/32", "98.95.37.200/32", "98.95.38.161/32", "98.95.39.39/32", "98.95.44.247/32", "98.95.49.54/32", "98.95.55.232/32", "98.95.56.98/32", "98.95.56.240/32", "98.95.62.182/32", "98.95.65.140/32", "98.95.74.218/32", "98.95.89.44/32", "98.95.91.59/32", "98.95.100.253/32", "98.95.104.207/32", "98.95.106.159/32", "98.95.110.94/32", "98.95.120.52/32", "98.95.126.63/32", "98.95.129.148/32", "98.95.130.0/32", "98.95.145.11/32", "98.95.148.56/32", "98.95.150.125/32", "98.95.156.127/32", "98.95.159.241/32", "98.95.161.76/32", "98.95.161.103/32", "98.95.162.121/32", "98.95.174.52/32", "98.95.175.92/32", "100.24.190.202/32", "100.25.17.231/32", "100.25.30.152/32", "100.25.127.167/32", "100.25.153.201/32", "100.25.199.161/32", "100.26.122.24/32", "100.27.85.243/32", "100.28.152.127/32", "100.28.177.247/32", "100.29.59.194/32", "100.29.157.182/32", "100.31.22.225/32", "100.31.34.200/32", "100.48.0.143/32", "100.48.137.73/32", "100.48.162.26/32", "100.49.196.199/32", "100.50.44.58/32", "100.50.76.106/32", "100.50.108.84/32", "100.50.153.151/32", "100.51.22.85/32", "100.51.81.206/32", "100.51.156.79/32", "100.52.70.176/32", "100.55.111.152/32", "100.55.191.83/32", "100.56.25.168/32", "100.57.68.185/32", "100.58.26.90/32", "100.58.39.170/32", "100.58.52.102/32", "100.58.57.149/32", "100.58.58.209/32", "100.59.72.83/32", "100.59.254.242/32", "100.60.14.30/32", "100.60.123.70/32", "100.60.141.252/32", "100.60.170.171/32", "100.60.204.184/32", "100.60.204.250/32", "100.61.92.1/32", "100.62.13.116/32", "100.62.157.123/32", "100.62.179.212/32", "100.62.186.188/32", "107.20.21.185/32", "107.20.124.65/32", "107.20.132.237/32", "107.20.164.28/32", "107.20.195.185/32", "107.21.9.54/32", "107.21.62.197/32", "107.22.6.189/32", "107.22.145.162/32", "107.22.199.65/32", "107.23.25.176/32", "107.23.41.220/32", "174.129.228.153/32", "184.72.137.83/32", "184.73.10.164/32", "184.73.154.145/32", "184.192.3.139/32", "184.192.68.254/32", "184.192.125.80/32", "184.192.132.1/32", "184.192.160.224/32", "184.192.172.115/32", "184.192.201.21/32", "184.192.207.91/32", "184.192.211.188/32", "184.192.212.181/32", "184.192.213.86/32", "184.192.213.200/32", "184.192.214.49/32", "184.192.214.62/32", "184.192.215.194/32" ], "note": null, "sources": [ { "url": "https://developer.amazon.com/docs/amazonbot/amazonbot.html", "title": "Amazon Web Search crawlers | Amazonbot", "fetched_at": "2026-10-28T17:14:02.251Z" } ] }, { "agent": "amzn-user", "url": "https://developer.amazon.com/amazonbot/live-ip-addresses/", "format": "html-pre+ipranges-json", "prefix_count": 1023, "fetched_at": "2026-10-28T17:13:59.948Z", "sha256": "9e016910ca9a2cb64088a17d8f9a4df634b6cc864ee53254e31378daf5b40dd3", "prefixes": [ "3.81.194.188/32", "3.81.253.151/32", "3.82.29.30/32", "3.82.135.179/32", "3.84.64.201/32", "3.85.182.19/32", "3.86.137.101/32", "3.87.104.37/32", "3.88.92.27/32", "3.89.14.22/32", "3.89.14.224/32", "3.89.213.23/32", "3.91.124.128/32", "3.92.14.207/32", "3.93.132.173/32", "3.94.34.209/32", "3.94.154.72/32", "3.94.216.37/32", "3.94.225.17/32", "3.94.226.92/32", "3.95.79.208/32", "3.95.81.91/32", "3.95.96.198/32", "3.208.46.236/32", "3.208.167.128/32", "3.208.247.204/32", "3.209.130.7/32", "3.210.93.112/32", "3.211.22.7/32", "3.211.99.154/32", "3.211.144.3/32", "3.211.145.5/32", "3.211.153.227/32", "3.211.177.14/32", "3.211.189.40/32", "3.211.192.237/32", "3.211.199.193/32", "3.212.20.54/32", "3.212.148.16/32", "3.212.151.142/32", "3.212.155.80/32", "3.212.155.239/32", "3.212.203.251/32", "3.212.205.155/32", "3.213.23.95/32", "3.213.74.26/32", "3.213.80.230/32", "3.213.119.107/32", "3.213.132.166/32", "3.213.183.18/32", "3.213.192.229/32", "3.213.205.133/32", "3.213.223.246/32", "3.214.107.231/32", "3.214.110.59/32", "3.214.154.139/32", "3.214.173.19/32", "3.214.189.85/32", "3.214.213.94/32", "3.215.8.42/32", "3.215.85.254/32", "3.215.193.161/32", "3.215.254.248/32", "3.216.16.254/32", "3.216.24.167/32", "3.216.83.21/32", "3.217.47.198/32", "3.217.54.78/32", "3.217.93.234/32", "3.217.124.227/32", "3.217.155.115/32", "3.217.166.57/32", "3.217.186.68/32", "3.218.11.113/32", "3.218.35.149/32", "3.218.68.30/32", "3.218.125.55/32", "3.218.140.139/32", "3.218.177.210/32", "3.218.184.146/32", "3.218.191.159/32", "3.218.226.239/32", "3.218.255.41/32", "3.219.10.249/32", "3.219.27.100/32", "3.219.52.199/32", "3.219.98.171/32", "3.219.103.72/32", "3.219.249.170/32", "3.220.2.22/32", "3.220.89.3/32", "3.220.134.7/32", "3.221.105.67/32", "3.221.110.107/32", "3.221.204.121/32", "3.221.221.12/32", "3.221.252.69/32", "3.222.22.183/32", "3.222.57.253/32", "3.222.193.181/32", "3.222.195.11/32", "3.223.37.137/32", "3.223.130.34/32", "3.223.132.246/32", "3.223.148.78/32", "3.223.173.19/32", "3.223.212.88/32", "3.223.245.225/32", "3.224.3.11/32", "3.224.66.185/32", "3.224.95.40/32", "3.225.27.26/32", "3.225.112.119/32", "3.225.130.6/32", "3.225.214.36/32", "3.225.250.64/32", "3.225.254.186/32", "3.226.55.191/32", "3.226.85.33/32", "3.226.118.11/32", "3.226.168.167/32", "3.226.198.163/32", "3.226.230.216/32", "3.227.129.67/32", "3.227.179.202/32", "3.227.195.183/32", "3.228.43.75/32", "3.228.90.152/32", "3.228.145.87/32", "3.228.187.171/32", "3.229.69.123/32", "3.229.88.194/32", "3.229.141.83/32", "3.229.147.55/32", "3.229.204.29/32", "3.229.206.235/32", "3.229.254.1/32", "3.230.15.20/32", "3.230.29.144/32", "3.230.47.56/32", "3.230.52.168/32", "3.230.90.160/32", "3.230.99.92/32", "3.230.138.76/32", "3.230.149.251/32", "3.231.88.131/32", "3.231.179.13/32", "3.232.19.149/32", "3.232.20.91/32", "3.232.137.33/32", "3.232.180.42/32", "3.232.197.101/32", "3.232.221.166/32", "3.232.235.15/32", "3.232.240.88/32", "3.233.93.65/32", "3.233.103.22/32", "3.233.121.160/32", "3.233.183.168/32", "3.233.218.90/32", "3.234.8.148/32", "3.234.47.132/32", "3.234.77.32/32", "3.234.161.147/32", "3.234.185.44/32", "3.235.12.133/32", "3.235.125.9/32", "13.216.27.246/32", "13.216.113.150/32", "13.216.136.62/32", "13.216.143.182/32", "13.216.152.158/32", "13.216.162.32/32", "13.216.169.197/32", "13.217.158.72/32", "13.218.255.40/32", "13.219.0.206/32", "13.219.15.124/32", "13.219.48.105/32", "13.219.122.158/32", "13.219.125.104/32", "13.219.138.92/32", "13.219.173.86/32", "13.219.211.133/32", "13.222.48.80/32", "13.222.68.90/32", "13.223.10.22/32", "13.223.12.55/32", "13.223.32.32/32", "13.223.33.67/32", "13.223.112.99/32", "13.223.123.7/32", "13.223.125.191/32", "13.223.127.163/32", "13.223.135.149/32", "13.223.141.183/32", "13.223.143.171/32", "13.223.208.86/32", "13.223.212.171/32", "13.223.219.110/32", "13.223.222.177/32", "13.223.237.145/32", "18.204.77.144/32", "18.204.129.185/32", "18.204.147.58/32", "18.204.174.34/32", "18.204.200.202/32", "18.205.94.193/32", "18.205.121.5/32", "18.205.170.69/32", "18.205.179.96/32", "18.205.207.200/32", "18.206.10.25/32", "18.206.31.0/32", "18.206.49.169/32", "18.206.63.178/32", "18.206.75.17/32", "18.206.80.142/32", "18.206.102.242/32", "18.206.103.117/32", "18.207.68.55/32", "18.207.71.20/32", "18.207.79.53/32", "18.208.49.162/32", "18.208.68.221/32", "18.208.96.116/32", "18.209.149.152/32", "18.209.155.76/32", "18.209.183.128/32", "18.209.189.61/32", "18.210.58.248/32", "18.210.79.138/32", "18.210.88.179/32", "18.210.135.253/32", "18.210.143.14/32", "18.210.177.112/32", "18.210.211.84/32", "18.211.100.116/32", "18.211.172.15/32", "18.211.184.127/32", "18.211.228.10/32", "18.213.18.100/32", "18.213.102.71/32", "18.213.108.226/32", "18.213.174.38/32", "18.213.225.24/32", "18.214.32.251/32", "18.214.97.194/32", "18.214.155.227/32", "18.214.160.25/32", "18.214.180.93/32", "18.214.202.10/32", "18.214.208.13/32", "18.214.236.252/32", "18.215.19.140/32", "18.215.37.82/32", "18.215.81.159/32", "18.215.133.216/32", "18.232.221.67/32", "18.233.44.115/32", "18.233.52.165/32", "18.233.242.106/32", "18.233.247.66/32", "18.233.254.6/32", "18.235.117.231/32", "18.235.176.68/32", "23.20.14.43/32", "23.20.14.164/32", "23.20.157.74/32", "23.20.190.101/32", "23.21.74.3/32", "23.21.78.102/32", "23.21.161.204/32", "23.21.164.86/32", "23.21.226.79/32", "23.22.87.173/32", "23.22.87.189/32", "23.22.140.49/32", "23.22.164.241/32", "23.22.236.99/32", "23.23.16.236/32", "23.23.111.110/32", "23.23.152.43/32", "23.23.201.145/32", "23.23.209.179/32", "34.192.33.34/32", "34.192.98.253/32", "34.192.141.13/32", "34.192.213.97/32", "34.192.238.32/32", "34.193.131.3/32", "34.194.47.173/32", "34.194.118.119/32", "34.194.149.187/32", "34.194.219.117/32", "34.194.219.191/32", "34.195.23.213/32", "34.195.45.53/32", "34.195.225.185/32", "34.196.42.137/32", "34.196.80.7/32", "34.196.142.152/32", "34.196.155.48/32", "34.196.169.196/32", "34.196.187.85/32", "34.197.2.163/32", "34.197.73.235/32", "34.197.143.174/32", "34.197.149.47/32", "34.197.153.75/32", "34.197.175.171/32", "34.197.201.73/32", "34.197.209.185/32", "34.198.56.37/32", "34.198.82.162/32", "34.198.130.224/32", "34.198.143.69/32", "34.198.162.37/32", "34.199.25.251/32", "34.199.156.195/32", "34.200.74.100/32", "34.200.81.78/32", "34.200.86.207/32", "34.200.114.155/32", "34.200.159.199/32", "34.200.169.53/32", "34.200.192.85/32", "34.200.203.147/32", "34.200.205.234/32", "34.202.2.123/32", "34.202.30.230/32", "34.202.41.56/32", "34.202.49.146/32", "34.203.99.163/32", "34.203.149.5/32", "34.204.57.203/32", "34.204.104.43/32", "34.204.215.184/32", "34.205.201.97/32", "34.205.238.95/32", "34.206.101.22/32", "34.206.135.181/32", "34.207.48.202/32", "34.207.52.183/32", "34.224.114.26/32", "34.224.120.185/32", "34.224.190.94/32", "34.225.108.229/32", "34.225.133.0/32", "34.225.171.229/32", "34.226.51.110/32", "34.226.56.238/32", "34.226.69.93/32", "34.226.125.221/32", "34.227.42.212/32", "34.227.118.181/32", "34.227.131.153/32", "34.227.239.141/32", "34.228.109.18/32", "34.230.156.160/32", "34.230.234.54/32", "34.230.255.232/32", "34.231.68.10/32", "34.231.81.35/32", "34.231.114.104/32", "34.231.184.75/32", "34.232.47.104/32", "34.232.184.48/32", "34.232.187.124/32", "34.233.36.75/32", "34.233.48.215/32", "34.233.97.140/32", "34.233.148.122/32", "34.233.151.134/32", "34.233.255.1/32", "34.234.14.185/32", "34.234.154.134/32", "34.235.11.182/32", "34.236.104.213/32", "34.236.123.186/32", "34.236.132.67/32", "34.236.132.205/32", "34.236.164.210/32", "34.236.166.107/32", "34.236.209.30/32", "34.236.214.169/32", "34.236.224.25/32", "34.236.253.178/32", "34.237.80.208/32", "34.237.99.86/32", "34.237.147.161/32", "34.237.191.141/32", "34.237.207.92/32", "34.237.208.14/32", "34.237.227.70/32", "34.237.232.130/32", "34.238.18.3/32", "34.238.71.63/32", "34.238.107.122/32", "34.239.15.203/32", "34.239.68.202/32", "34.239.188.168/32", "34.239.195.85/32", "34.239.206.162/32", "35.153.10.9/32", "35.153.36.100/32", "35.153.107.141/32", "35.153.113.217/32", "35.153.124.65/32", "35.153.197.87/32", "35.168.44.111/32", "35.168.104.113/32", "35.168.183.119/32", "35.169.56.220/32", "35.169.122.102/32", "35.169.198.60/32", "35.169.212.245/32", "35.169.251.66/32", "35.170.223.99/32", "35.171.77.2/32", "35.171.215.147/32", "35.171.218.156/32", "35.172.62.192/32", "35.172.84.121/32", "35.172.188.66/32", "35.173.12.118/32", "35.173.12.250/32", "35.173.53.155/32", "35.173.140.109/32", "35.173.160.101/32", "35.173.172.43/32", "35.174.97.70/32", "35.174.214.80/32", "35.174.229.240/32", "35.174.232.222/32", "35.175.22.142/32", "35.175.34.176/32", "35.175.43.19/32", "44.193.21.207/32", "44.193.128.83/32", "44.193.136.188/32", "44.193.149.106/32", "44.193.163.140/32", "44.194.70.31/32", "44.194.163.207/32", "44.194.194.219/32", "44.194.203.102/32", "44.194.222.211/32", "44.195.16.2/32", "44.195.96.4/32", "44.195.151.174/32", "44.195.178.236/32", "44.195.228.45/32", "44.195.245.78/32", "44.195.248.164/32", "44.196.15.171/32", "44.196.132.20/32", "44.196.210.221/32", "44.196.236.142/32", "44.197.30.79/32", "44.197.95.23/32", "44.198.105.147/32", "44.198.125.78/32", "44.198.131.235/32", "44.198.134.204/32", "44.198.205.87/32", "44.198.237.57/32", "44.205.36.200/32", "44.205.58.205/32", "44.205.68.181/32", "44.205.109.175/32", "44.205.197.17/32", "44.205.215.215/32", "44.206.15.195/32", "44.206.40.110/32", "44.206.84.82/32", "44.206.129.15/32", "44.206.184.47/32", "44.207.26.24/32", "44.207.83.128/32", "44.207.127.111/32", "44.207.135.147/32", "44.207.169.172/32", "44.207.194.222/32", "44.207.229.212/32", "44.208.16.208/32", "44.208.56.213/32", "44.208.77.108/32", "44.208.83.208/32", "44.208.85.236/32", "44.208.104.56/32", "44.208.200.219/32", "44.208.203.166/32", "44.208.236.62/32", "44.209.162.73/32", "44.210.192.115/32", "44.212.148.132/32", "44.212.198.85/32", "44.212.239.227/32", "44.213.158.218/32", "44.213.163.24/32", "44.214.32.136/32", "44.214.62.148/32", "44.214.194.234/32", "44.214.207.7/32", "44.214.212.218/32", "44.215.7.194/32", "44.215.24.160/32", "44.215.63.87/32", "44.215.81.252/32", "44.215.84.21/32", "44.215.89.216/32", "44.215.161.26/32", "44.215.216.21/32", "44.215.224.27/32", "44.216.5.127/32", "44.216.7.31/32", "44.216.58.133/32", "44.216.100.41/32", "44.216.104.198/32", "44.216.243.74/32", "44.217.121.202/32", "44.217.152.47/32", "44.217.158.120/32", "44.217.254.94/32", "44.218.30.160/32", "44.218.99.76/32", "44.218.132.213/32", "44.218.149.105/32", "44.218.169.19/32", "44.218.183.161/32", "44.218.187.96/32", "44.218.200.142/32", "44.219.23.147/32", "44.219.64.51/32", "44.219.115.49/32", "44.219.143.106/32", "44.220.9.209/32", "44.221.99.201/32", "44.221.111.36/32", "44.221.132.209/32", "44.221.185.185/32", "44.221.223.252/32", "44.221.225.24/32", "44.221.244.31/32", "44.222.28.176/32", "44.222.60.175/32", "44.223.220.158/32", "44.223.230.228/32", "50.16.4.171/32", "50.16.147.3/32", "50.16.231.180/32", "50.17.1.240/32", "50.17.134.19/32", "50.17.153.9/32", "50.17.191.57/32", "50.17.207.81/32", "50.17.230.96/32", "50.19.29.68/32", "50.19.35.165/32", "50.19.59.54/32", "50.19.126.63/32", "50.19.204.59/32", "50.19.251.209/32", "52.0.29.253/32", "52.0.31.88/32", "52.0.92.85/32", "52.0.129.197/32", "52.0.175.66/32", "52.0.190.163/32", "52.0.192.213/32", "52.0.194.51/32", "52.1.58.136/32", "52.1.178.161/32", "52.1.245.227/32", "52.2.182.123/32", "52.2.196.120/32", "52.2.240.114/32", "52.2.253.241/32", "52.3.70.186/32", "52.3.104.17/32", "52.3.239.250/32", "52.4.5.89/32", "52.4.20.22/32", "52.4.106.28/32", "52.4.161.44/32", "52.4.200.143/32", "52.4.229.83/32", "52.5.9.202/32", "52.5.11.240/32", "52.5.71.38/32", "52.5.91.235/32", "52.5.101.193/32", "52.5.126.84/32", "52.5.139.177/32", "52.5.158.30/32", "52.5.237.111/32", "52.5.241.167/32", "52.6.108.202/32", "52.7.23.251/32", "52.7.57.117/32", "52.7.66.242/32", "52.7.157.23/32", "52.7.217.144/32", "52.7.224.72/32", "52.20.29.92/32", "52.20.100.136/32", "52.20.139.63/32", "52.21.48.195/32", "52.21.106.0/32", "52.21.120.37/32", "52.21.146.37/32", "52.21.166.33/32", "52.22.89.200/32", "52.22.150.64/32", "52.22.196.44/32", "52.22.210.122/32", "52.23.51.117/32", "52.23.97.153/32", "52.23.133.109/32", "52.23.146.199/32", "52.44.93.127/32", "52.44.145.57/32", "52.44.222.44/32", "52.45.82.146/32", "52.45.91.149/32", "52.45.105.249/32", "52.45.162.115/32", "52.45.180.161/32", "52.54.73.166/32", "52.54.128.50/32", "52.54.247.126/32", "52.55.118.104/32", "52.55.174.246/32", "52.55.203.221/32", "52.70.155.245/32", "52.70.247.71/32", "52.71.39.142/32", "52.71.50.120/32", "52.71.70.228/32", "52.71.225.8/32", "52.72.73.184/32", "52.72.232.179/32", "52.73.115.59/32", "52.73.158.87/32", "52.73.197.50/32", "52.73.206.11/32", "52.73.215.66/32", "52.86.53.42/32", "52.86.86.102/32", "52.86.118.49/32", "52.86.198.100/32", "52.86.235.69/32", "52.87.0.229/32", "52.87.52.141/32", "52.87.74.18/32", "52.87.118.241/32", "52.87.127.40/32", "52.200.8.184/32", "52.200.28.69/32", "52.200.84.90/32", "52.200.116.93/32", "52.201.41.193/32", "52.201.98.75/32", "52.201.206.154/32", "52.202.35.1/32", "52.202.37.123/32", "52.202.101.32/32", "52.202.103.180/32", "52.202.121.224/32", "52.202.216.228/32", "52.202.253.248/32", "52.203.32.72/32", "52.203.86.132/32", "52.203.152.175/32", "52.203.209.127/32", "52.203.244.55/32", "52.204.64.101/32", "52.204.78.168/32", "52.204.81.241/32", "52.204.124.205/32", "52.204.162.207/32", "52.204.226.159/32", "52.204.226.207/32", "52.204.226.223/32", "52.205.27.163/32", "52.205.135.221/32", "52.205.246.20/32", "52.206.46.172/32", "52.206.52.116/32", "52.206.60.64/32", "52.206.61.139/32", "52.206.102.51/32", "52.206.130.42/32", "52.206.143.135/32", "52.206.169.224/32", "52.206.181.180/32", "52.207.16.197/32", "52.207.61.91/32", "52.207.72.172/32", "52.207.120.139/32", "52.207.133.220/32", "54.80.45.194/32", "54.80.71.176/32", "54.80.74.44/32", "54.80.175.67/32", "54.80.198.127/32", "54.80.234.89/32", "54.80.243.130/32", "54.81.154.68/32", "54.81.165.33/32", "54.81.246.115/32", "54.82.11.240/32", "54.82.79.140/32", "54.82.116.145/32", "54.82.165.97/32", "54.82.184.206/32", "54.83.14.168/32", "54.83.29.252/32", "54.83.61.93/32", "54.83.178.173/32", "54.83.238.70/32", "54.84.42.70/32", "54.84.58.86/32", "54.84.96.236/32", "54.84.247.173/32", "54.84.255.139/32", "54.85.13.251/32", "54.85.42.248/32", "54.85.116.136/32", "54.85.152.251/32", "54.85.173.232/32", "54.85.182.11/32", "54.85.182.74/32", "54.85.183.163/32", "54.86.41.234/32", "54.86.102.52/32", "54.86.113.36/32", "54.86.128.12/32", "54.86.173.205/32", "54.86.200.206/32", "54.86.235.24/32", "54.87.82.164/32", "54.87.147.32/32", "54.87.230.73/32", "54.87.231.234/32", "54.88.17.4/32", "54.88.231.214/32", "54.89.1.163/32", "54.89.37.100/32", "54.89.57.217/32", "54.89.197.100/32", "54.90.15.96/32", "54.90.160.213/32", "54.90.251.223/32", "54.91.68.167/32", "54.91.90.149/32", "54.91.222.113/32", "54.92.230.27/32", "54.144.163.59/32", "54.144.165.228/32", "54.144.166.248/32", "54.145.100.187/32", "54.145.120.66/32", "54.145.201.55/32", "54.145.202.232/32", "54.146.78.43/32", "54.146.124.96/32", "54.146.217.77/32", "54.147.28.169/32", "54.152.62.114/32", "54.152.240.110/32", "54.152.250.63/32", "54.156.60.28/32", "54.156.75.181/32", "54.156.101.5/32", "54.156.218.150/32", "54.156.228.126/32", "54.157.17.251/32", "54.157.60.117/32", "54.157.89.202/32", "54.157.93.108/32", "54.157.127.12/32", "54.158.87.246/32", "54.158.126.53/32", "54.159.41.249/32", "54.159.73.59/32", "54.159.127.105/32", "54.160.29.53/32", "54.161.82.218/32", "54.162.184.206/32", "54.163.96.5/32", "54.163.119.23/32", "54.163.119.115/32", "54.163.149.34/32", "54.163.156.179/32", "54.164.253.209/32", "54.165.187.237/32", "54.165.237.175/32", "54.166.51.59/32", "54.166.114.87/32", "54.166.123.195/32", "54.166.147.9/32", "54.166.168.104/32", "54.166.186.204/32", "54.166.203.36/32", "54.166.221.13/32", "54.167.145.47/32", "54.167.149.221/32", "54.167.157.19/32", "54.167.167.144/32", "54.167.254.156/32", "54.172.147.40/32", "54.172.181.132/32", "54.172.223.244/32", "54.172.235.19/32", "54.173.3.145/32", "54.173.73.223/32", "54.173.251.40/32", "54.173.254.186/32", "54.174.33.240/32", "54.174.79.6/32", "54.174.132.51/32", "54.174.156.91/32", "54.174.185.64/32", "54.174.185.149/32", "54.174.239.164/32", "54.174.255.71/32", "54.175.92.255/32", "54.175.137.98/32", "54.175.225.187/32", "54.196.38.144/32", "54.196.107.151/32", "54.197.64.106/32", "54.197.112.73/32", "54.198.16.172/32", "54.198.102.234/32", "54.198.232.105/32", "54.198.233.242/32", "54.204.35.237/32", "54.204.59.63/32", "54.204.80.55/32", "54.204.105.187/32", "54.204.223.93/32", "54.205.3.125/32", "54.205.40.225/32", "54.205.45.210/32", "54.205.119.2/32", "54.205.136.35/32", "54.205.160.25/32", "54.205.201.74/32", "54.205.208.13/32", "54.205.213.166/32", "54.208.27.202/32", "54.208.132.243/32", "54.208.179.143/32", "54.208.246.64/32", "54.209.9.33/32", "54.209.36.112/32", "54.209.190.9/32", "54.209.191.8/32", "54.210.31.211/32", "54.210.80.37/32", "54.210.131.110/32", "54.210.198.121/32", "54.211.75.0/32", "54.211.78.165/32", "54.211.199.50/32", "54.221.32.154/32", "54.221.237.132/32", "54.224.53.14/32", "54.224.116.206/32", "54.224.224.135/32", "54.224.240.158/32", "54.225.67.8/32", "54.225.71.221/32", "54.225.113.76/32", "54.225.145.181/32", "54.225.162.45/32", "54.225.169.17/32", "54.225.221.30/32", "54.226.73.182/32", "54.227.13.229/32", "54.227.28.87/32", "54.234.167.100/32", "54.235.211.99/32", "54.235.213.173/32", "54.236.112.173/32", "54.236.158.242/32", "54.236.167.56/32", "54.236.239.164/32", "54.237.29.9/32", "54.237.184.17/32", "54.242.10.253/32", "54.242.167.254/32", "54.243.110.227/32", "54.243.144.14/32", "54.243.201.33/32", "54.243.209.16/32", "54.243.211.8/32", "67.202.14.197/32", "75.101.211.35/32", "98.80.55.141/32", "98.82.55.41/32", "98.82.243.141/32", "98.82.255.249/32", "98.83.18.54/32", "98.83.46.92/32", "98.83.70.133/32", "98.83.89.12/32", "98.83.170.172/32", "98.83.191.241/32", "98.83.248.195/32", "98.84.242.231/32", "98.85.106.40/32", "98.85.137.77/32", "98.85.140.218/32", "98.85.146.158/32", "98.85.211.76/32", "98.85.214.32/32", "98.85.218.4/32", "98.85.238.47/32", "98.85.246.245/32", "98.86.4.188/32", "98.86.24.23/32", "98.86.25.36/32", "98.86.27.21/32", "98.86.29.203/32", "98.86.50.127/32", "98.86.51.12/32", "98.86.54.86/32", "98.86.60.7/32", "98.86.78.84/32", "98.86.80.106/32", "98.86.129.100/32", "98.86.184.138/32", "98.86.187.75/32", "98.86.188.104/32", "98.86.192.5/32", "98.86.205.54/32", "98.86.207.121/32", "98.86.207.186/32", "98.86.212.35/32", "98.86.212.136/32", "98.86.214.58/32", "98.86.218.208/32", "98.86.249.175/32", "98.87.30.184/32", "98.87.36.115/32", "98.87.39.1/32", "98.87.44.13/32", "98.87.48.159/32", "98.87.52.210/32", "98.87.54.118/32", "98.87.63.1/32", "98.87.68.53/32", "98.87.69.247/32", "98.87.70.113/32", "98.87.73.104/32", "98.87.83.39/32", "98.87.91.253/32", "98.87.92.12/32", "98.87.95.44/32", "98.87.95.193/32", "98.87.100.94/32", "98.87.110.161/32", "98.87.115.179/32", "98.87.119.92/32", "98.87.125.106/32", "98.87.130.228/32", "98.87.132.166/32", "98.87.141.217/32", "98.87.145.207/32", "98.87.177.122/32", "98.87.178.41/32", "98.87.179.77/32", "98.87.179.143/32", "98.87.179.166/32", "98.87.180.82/32", "98.87.180.162/32", "98.87.180.251/32", "98.87.182.37/32", "98.87.182.62/32", "98.87.182.201/32", "98.87.183.61/32", "98.87.183.122/32", "98.87.184.28/32", "98.87.184.100/32", "100.24.86.21/32", "100.24.94.158/32", "100.25.233.255/32", "100.26.69.34/32", "100.27.91.246/32", "100.27.118.106/32", "100.27.157.160/32", "100.27.158.82/32", "100.27.169.147/32", "100.28.33.208/32", "100.28.104.194/32", "100.28.113.237/32", "100.28.151.241/32", "100.29.92.161/32", "100.29.104.31/32", "100.29.106.130/32", "100.29.122.28/32", "107.20.42.162/32", "107.20.201.23/32", "107.21.114.117/32", "107.21.229.211/32", "107.22.55.149/32", "107.22.87.58/32", "107.22.218.90/32", "107.22.241.68/32", "107.22.255.3/32", "107.23.56.227/32", "107.23.190.53/32", "107.23.242.35/32", "174.129.24.225/32", "184.72.220.200/32", "184.72.231.109/32", "184.72.242.66/32", "184.73.1.121/32", "184.73.212.232/32" ], "note": null, "sources": [ { "url": "https://developer.amazon.com/docs/amazonbot/amazonbot.html", "title": "Amazon Web Search crawlers | Amazonbot", "fetched_at": "2026-10-28T17:14:02.251Z" } ] } ], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Amazon Bedrock AgentCore Browser", "coverage": "researched", "user_agent_tokens": [], "aliases": [], "purpose": "agent", "purpose_quote": "The Amazon Bedrock AgentCore Browser provides a secure, isolated browser environment for your agents to interact with web applications.", "search_engine": false, "robots_token": null, "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "A hosted browser that AWS customers' agents drive. AWS documents no User-Agent token, robots.txt token or IP list for it. Web Bot Auth signing is optional per browser: the Web Bot Auth page says the feature is disabled by default and must be enabled when the browser is created.", "sources": [ { "url": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/browser-tool.html", "title": "Interact with web applications using Amazon Bedrock AgentCore Browser", "fetched_at": "2026-10-28T17:14:22.880Z" } ] }, { "name": "Amazon Bedrock Web Crawler", "coverage": "researched", "user_agent_tokens": [], "aliases": [], "purpose": "other", "purpose_quote": "The Amazon Bedrock provided Web Crawler connects to and crawls URLs you have selected for use in your Amazon Bedrock knowledge base.", "search_engine": false, "robots_token": "bedrockbot", "robots_policy": "honors", "robots_quote": "The crawler will first look for bedrockbot-UUID rules and then for generic bedrockbot rules in the robots.txt file.", "ip_list_agent": null, "signs": false, "note": "Crawls sites that an AWS customer selects for that customer's Bedrock knowledge base. AWS documents the robots.txt tokens (bedrockbot and bedrockbot-UUID) but not the full User-Agent string, so no User-Agent token is listed. The page says that from September 30, 2026 new Web Crawler connectors can no longer be created on customer-managed knowledge bases; existing ones keep working.", "sources": [ { "url": "https://docs.aws.amazon.com/bedrock/latest/userguide/webcrawl-data-source-connector.html", "title": "Crawl web pages for your knowledge base (Amazon Bedrock User Guide)", "fetched_at": "2026-10-28T17:14:02.631Z" } ] }, { "name": "Amazon Kendra Web Crawler", "coverage": "researched", "user_agent_tokens": [], "aliases": [], "purpose": "other", "purpose_quote": "In order to index documents on the web, customers may use Amazon Kendra Web Crawler, indicating which URL(s) should be indexed and other operational parameters.", "search_engine": false, "robots_token": "amazon-kendra", "robots_policy": "honors", "robots_quote": "You can modify the robots.txt file of your website to control how Amazon Kendra Web Crawler crawls your website.", "ip_list_agent": null, "signs": false, "note": "Indexes sites that an AWS customer selects for that customer's Kendra search index. AWS documents the robots.txt token amazon-kendra but not the full User-Agent string. The page says Amazon Kendra is no longer open to new customers.", "sources": [ { "url": "https://docs.aws.amazon.com/kendra/latest/dg/stop-web-crawler.html", "title": "Configuring the robots.txt file for Amazon Kendra Web Crawler", "fetched_at": "2026-10-28T17:14:07.844Z" } ] }, { "name": "Amazon Q Business Web Crawler", "coverage": "researched", "user_agent_tokens": [], "aliases": [], "purpose": "other", "purpose_quote": "An Amazon Q Business Web Crawler connector crawls and indexes either public facing websites or internal company websites that use HTTPS.", "search_engine": false, "robots_token": "amazon-QBusiness", "robots_policy": "honors", "robots_quote": "Amazon Q Business Web Crawler adheres to the rules set in your website’s robots.txt file, which determines the areas it is allowed or not allowed to visit.", "ip_list_agent": null, "signs": false, "note": "Indexes sites that an AWS customer selects for that customer's Amazon Q Business application. AWS documents the robots.txt token amazon-QBusiness but not the full User-Agent string. The pages say Amazon Q Business is no longer open to new customers.", "sources": [ { "url": "https://docs.aws.amazon.com/amazonq/latest/qbusiness-ug/connector-webcrawler.html", "title": "Connecting Web Crawler to Amazon Q Business", "fetched_at": "2026-10-28T17:14:17.864Z" }, { "url": "https://docs.aws.amazon.com/amazonq/latest/qbusiness-ug/stop-web-crawler.html", "title": "Configuring a robots.txt file for Amazon Q Business Web Crawler", "fetched_at": "2026-10-28T17:14:12.849Z" } ] }, { "name": "Amazonbot", "coverage": "researched", "user_agent_tokens": [ "Amazonbot" ], "aliases": [], "purpose": "training", "purpose_quote": "This helps us provide more accurate information to customers and may be used to train Amazon AI models.", "search_engine": false, "robots_token": "Amazonbot", "robots_policy": "honors", "robots_quote": "Automated crawling from these listed user agents respects the Robots Exclusion Protocol, honoring the user-agent and the allow/disallow directives.", "ip_list_agent": "amazonbot", "signs": false, "note": "Amazon's page says Amazonbot \"is used to improve our products and services\" and may be used to train Amazon AI models; training is the purpose that sets it apart from Amzn-SearchBot and Amzn-User, which the page says do not crawl for generative AI model training.", "sources": [ { "url": "https://developer.amazon.com/docs/amazonbot/amazonbot.html", "title": "Amazon Web Search crawlers | Amazonbot", "fetched_at": "2026-10-28T17:14:02.251Z" } ] }, { "name": "Amzn-SearchBot", "coverage": "researched", "user_agent_tokens": [ "Amzn-SearchBot" ], "aliases": [], "purpose": "search", "purpose_quote": "Amzn-SearchBot is used to improve search experiences in Amazon products and services.", "search_engine": false, "robots_token": "Amzn-SearchBot", "robots_policy": "honors", "robots_quote": "If robots.txt files don’t mention Amzn-SearchBot but allow other search bots, Amzn-SearchBot will crawl in accordance with the robots.txt directives given to other search bots.", "ip_list_agent": "amzn-searchbot", "signs": false, "note": "Without its own robots.txt group, Amzn-SearchBot follows the rules given to other search bots.", "sources": [ { "url": "https://developer.amazon.com/docs/amazonbot/amazonbot.html", "title": "Amazon Web Search crawlers | Amazonbot", "fetched_at": "2026-10-28T17:14:02.251Z" } ] }, { "name": "Amzn-User", "coverage": "researched", "user_agent_tokens": [ "Amzn-User" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": "Amzn-User supports user actions, such as responding to Alexa queries that require up-to-date information.", "search_engine": false, "robots_token": "Amzn-User", "robots_policy": "may_ignore", "robots_quote": "Because actions taken by Amzn-User can be initiated by a user, it may not follow all robots.txt directives.", "ip_list_agent": "amzn-user", "signs": false, "note": null, "sources": [ { "url": "https://developer.amazon.com/docs/amazonbot/amazonbot.html", "title": "Amazon Web Search crawlers | Amazonbot", "fetched_at": "2026-10-28T17:14:02.251Z" } ] }, { "name": "AmazonBuyForMe", "coverage": "community", "user_agent_tokens": [ "AmazonBuyForMe" ], "aliases": [], "purpose": "agent", "purpose_quote": null, "search_engine": false, "robots_token": "AmazonBuyForMe", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Agents\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "NovaAct", "coverage": "community", "user_agent_tokens": [ "NovaAct" ], "aliases": [], "purpose": "agent", "purpose_quote": null, "search_engine": false, "robots_token": "NovaAct", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Agents\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://developer.amazon.com/docs/amazonbot/amazonbot.html", "title": "Amazon Web Search crawlers | Amazonbot", "fetched_at": "2026-10-28T17:14:02.251Z" }, { "url": "https://docs.aws.amazon.com/bedrock/latest/userguide/webcrawl-data-source-connector.html", "title": "Crawl web pages for your knowledge base (Amazon Bedrock User Guide)", "fetched_at": "2026-10-28T17:14:02.631Z" }, { "url": "https://docs.aws.amazon.com/kendra/latest/dg/stop-web-crawler.html", "title": "Configuring the robots.txt file for Amazon Kendra Web Crawler", "fetched_at": "2026-10-28T17:14:07.844Z" }, { "url": "https://docs.aws.amazon.com/amazonq/latest/qbusiness-ug/stop-web-crawler.html", "title": "Configuring a robots.txt file for Amazon Q Business Web Crawler", "fetched_at": "2026-10-28T17:14:12.849Z" }, { "url": "https://docs.aws.amazon.com/amazonq/latest/qbusiness-ug/connector-webcrawler.html", "title": "Connecting Web Crawler to Amazon Q Business", "fetched_at": "2026-10-28T17:14:17.864Z" }, { "url": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/browser-tool.html", "title": "Interact with web applications using Amazon Bedrock AgentCore Browser", "fetched_at": "2026-10-28T17:14:22.880Z" }, { "url": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/browser-web-bot-auth.html", "title": "Reducing CAPTCHAs with Web Bot Auth (Amazon Bedrock AgentCore)", "fetched_at": "2026-10-28T17:14:27.840Z" }, { "url": "https://modpagespeed.com/blog/which-ai-crawlers-sign-their-requests/", "title": "55 of 150 AI crawler operators publish Web Bot Auth keys (third party, mod_pagespeed blog)", "fetched_at": "2026-10-28T17:14:32.840Z" }, { "url": "https://modpagespeed.com/data/web-bot-auth-2026-10-19/results.tsv", "title": "Web Bot Auth key directory probe results, 19 October 2026 (third party)", "fetched_at": "2026-10-28T17:14:34.046Z" } ] }, { "slug": "andi", "name": "Andi", "website": "https://andisearch.com/", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Andibot", "coverage": "community", "user_agent_tokens": [ "Andibot" ], "aliases": [], "purpose": "search", "purpose_quote": null, "search_engine": false, "robots_token": "Andibot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Search engine using generative AI, AI Search Assistant\". Respects robots.txt: \"Unclear at this time\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "anthropic", "name": "Anthropic", "website": "https://www.anthropic.com/", "docs_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "coverage": "researched", "note": null, "verification": { "signing": { "status": "none_found", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": "Anthropic's article does not mention request signing, and no third-party report was found. On 2026-10-26 /.well-known/http-message-signatures-directory returned 404 on claude.ai, claude.com and www.anthropic.com. No key directory found in the operator's documentation.", "sources": [] }, "ip_lists": [ { "agent": "anthropic-bots", "url": "https://claude.com/crawling/bots.json", "format": "ipranges-json", "prefix_count": 14, "fetched_at": "2026-10-28T17:14:35.180Z", "sha256": "c960bb55504c3c478d0335bc7153e9ca871c26388a69e403cbad121a5bb5c5dd", "prefixes": [ "34.152.124.140/32", "34.153.21.190/32", "34.162.27.34/32", "34.162.86.102/32", "34.162.90.102/32", "34.162.111.78/32", "34.162.153.71/32", "34.162.191.81/32", "34.162.230.222/32", "34.162.244.71/32", "34.186.252.4/32", "136.79.143.251/32", "136.83.20.215/32", "216.73.216.0/22" ], "note": null, "sources": [ { "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "title": "Does Anthropic crawl data from the web, and how can site owners block the crawler?", "fetched_at": "2026-10-28T17:14:36.240Z" } ] } ], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Claude-SearchBot", "coverage": "researched", "user_agent_tokens": [ "Claude-SearchBot" ], "aliases": [], "purpose": "search", "purpose_quote": "Claude-SearchBot navigates the web to improve search result quality for users.", "search_engine": false, "robots_token": "Claude-SearchBot", "robots_policy": "honors", "robots_quote": "Anthropic’s Bots respect “do not crawl” signals by honoring industry standard directives in robots.txt.", "ip_list_agent": "anthropic-bots", "signs": false, "note": "No full User-Agent string is given; the token is the bot name the article uses.", "sources": [ { "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "title": "Does Anthropic crawl data from the web, and how can site owners block the crawler?", "fetched_at": "2026-10-28T17:14:36.240Z" } ] }, { "name": "Claude-User", "coverage": "researched", "user_agent_tokens": [ "Claude-User" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": "When individuals ask questions to Claude, it may access websites using a Claude-User agent.", "search_engine": false, "robots_token": "Claude-User", "robots_policy": "honors", "robots_quote": "Anthropic’s Bots respect “do not crawl” signals by honoring industry standard directives in robots.txt.", "ip_list_agent": "anthropic-bots", "signs": false, "note": "The robots.txt statement covers all three of Anthropic's bots, including this user-triggered one. The article also says Claude-User lets site owners control which sites these user-initiated requests can reach. No full User-Agent string is given; the token is the bot name the article uses.", "sources": [ { "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "title": "Does Anthropic crawl data from the web, and how can site owners block the crawler?", "fetched_at": "2026-10-28T17:14:36.240Z" } ] }, { "name": "ClaudeBot", "coverage": "researched", "user_agent_tokens": [ "ClaudeBot" ], "aliases": [], "purpose": "training", "purpose_quote": "ClaudeBot helps enhance the utility and safety of our generative AI models by collecting web content that could potentially contribute to their training.", "search_engine": false, "robots_token": "ClaudeBot", "robots_policy": "honors", "robots_quote": "Anthropic’s Bots respect “do not crawl” signals by honoring industry standard directives in robots.txt.", "ip_list_agent": "anthropic-bots", "signs": false, "note": "Anthropic also says it supports the non-standard Crawl-delay directive. The article does not give full User-Agent strings; the token is the bot name it uses in its robots.txt examples.", "sources": [ { "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "title": "Does Anthropic crawl data from the web, and how can site owners block the crawler?", "fetched_at": "2026-10-28T17:14:36.240Z" } ] }, { "name": "anthropic-ai", "coverage": "community", "user_agent_tokens": [ "anthropic-ai" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "anthropic-ai", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Scrapes data to train Anthropic's AI products.\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "Claude-Code", "coverage": "community", "user_agent_tokens": [ "Claude-Code" ], "aliases": [], "purpose": "agent", "purpose_quote": null, "search_engine": false, "robots_token": "Claude-Code", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Coding Agents\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "Claude-Web", "coverage": "community", "user_agent_tokens": [ "Claude-Web" ], "aliases": [], "purpose": "agent", "purpose_quote": null, "search_engine": false, "robots_token": "Claude-Web", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Undocumented AI Agents\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "title": "Does Anthropic crawl data from the web, and how can site owners block the crawler?", "fetched_at": "2026-10-28T17:14:36.240Z" } ] }, { "slug": "apify", "name": "Apify", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt lists the operator as unclear; it is named in the agent's own description there.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "ApifyBot", "coverage": "community", "user_agent_tokens": [ "ApifyBot" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "ApifyBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Its operator is named in its ai.robots.txt description. Function: \"AI Data Providers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "ApifyWebsiteContentCrawler", "coverage": "community", "user_agent_tokens": [ "ApifyWebsiteContentCrawler" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "ApifyWebsiteContentCrawler", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Its operator is named in its ai.robots.txt description. Function: \"AI Data Providers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "apple", "name": "Apple", "website": "https://www.apple.com/", "docs_url": "https://support.apple.com/en-us/119829", "coverage": "researched", "note": null, "verification": { "signing": { "status": "none_found", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": "Apple's Applebot page does not mention Web Bot Auth or HTTP message signatures, and no third-party report was found. On 2026-10-26 https://www.apple.com/.well-known/http-message-signatures-directory returned 404 and applebot.apple.com does not resolve as a host. No key directory found in the operator's documentation.", "sources": [ { "url": "https://support.apple.com/en-us/119829", "title": "About Applebot", "fetched_at": "2026-10-28T17:14:38.742Z" } ] }, "ip_lists": [ { "agent": "applebot", "url": "https://search.developer.apple.com/applebot.json", "format": "ipranges-json", "prefix_count": 24, "fetched_at": "2026-10-28T17:14:37.671Z", "sha256": "5a5e12188891d998341f4fb50d2b31c80aba5002f5cff9d0fa110c13e4e2edd0", "prefixes": [ "17.166.20.0/24", "17.166.21.0/24", "17.166.22.0/24", "17.166.23.0/24", "17.166.24.0/24", "17.166.25.0/24", "17.166.150.0/24", "17.166.151.0/24", "17.166.152.0/24", "17.166.153.0/24", "17.166.154.0/24", "17.166.155.0/24", "17.166.192.48/28", "17.166.232.0/24", "17.166.233.0/24", "17.166.234.0/24", "17.166.235.0/24", "17.166.236.0/24", "17.166.237.0/24", "17.166.240.48/28", "17.166.248.48/28", "17.241.193.160/27", "17.241.200.160/27", "17.241.208.160/27" ], "note": null, "sources": [ { "url": "https://support.apple.com/en-us/119829", "title": "About Applebot", "fetched_at": "2026-10-28T17:14:38.742Z" } ] } ], "reverse_dns": { "suffixes": [ "applebot.apple.com" ], "forward_confirm": true, "note": null, "sources": [ { "url": "https://support.apple.com/en-us/119829", "title": "About Applebot", "fetched_at": "2026-10-28T17:14:38.742Z" } ] }, "asn": [] }, "agents": [ { "name": "Applebot", "coverage": "researched", "user_agent_tokens": [ "Applebot" ], "aliases": [], "purpose": "search", "purpose_quote": "Enabling Applebot in robots.txt allows website content to appear in search results for Apple users around the world in these products.", "search_engine": true, "robots_token": "Applebot", "robots_policy": "honors", "robots_quote": "Applebot respects standard robots.txt directives in general search crawls that are targeted at Applebot.", "ip_list_agent": "applebot", "signs": false, "note": "Search crawler for Spotlight, Siri and Safari. Apple says data it crawls may also be used to train Apple foundation models unless the site disallows Applebot-Extended. If robots.txt does not mention Applebot but mentions Googlebot, Applebot follows the Googlebot rules. Apple says Applebot does not follow crawl-delay.", "sources": [ { "url": "https://support.apple.com/en-us/119829", "title": "About Applebot", "fetched_at": "2026-10-28T17:14:38.742Z" } ] }, { "name": "Applebot-Extended", "coverage": "researched", "user_agent_tokens": [], "aliases": [], "purpose": "training", "purpose_quote": "With Applebot-Extended, web publishers can choose to opt out of their website content being used to train Apple’s general purpose foundation models powering generative AI features across Apple products, including Apple Intelligence, Services, and Developer Tools.", "search_engine": false, "robots_token": "Applebot-Extended", "robots_policy": "honors", "robots_quote": "Web publishers can opt-out from having their content used to train generative foundation models by disallowing Applebot-Extended in the robots.txt file.", "ip_list_agent": null, "signs": false, "note": "Robots.txt control token only. Apple says Applebot-Extended does not crawl webpages and is only used to decide how data crawled by Applebot is used, so requests come from Applebot. Disallowing it does not remove a site from Apple search results.", "sources": [ { "url": "https://support.apple.com/en-us/119829", "title": "About Applebot", "fetched_at": "2026-10-28T17:14:38.742Z" } ] }, { "name": "iTMS", "coverage": "researched", "user_agent_tokens": [ "iTMS" ], "aliases": [], "purpose": "other", "purpose_quote": "The iTMS user agent only crawls URLs associated with registered content on Apple Podcasts.", "search_engine": false, "robots_token": null, "robots_policy": "may_ignore", "robots_quote": "The iTMS user agent does not follow robots.txt, as it is not a general search crawler.", "ip_list_agent": null, "signs": false, "note": "Listed by Apple under special crawlers (Apple online properties). Apple says iTMS traffic may come from applebot.apple.com hosts, but does not say its addresses are in the Applebot IP list, so no IP list is linked.", "sources": [ { "url": "https://support.apple.com/en-us/119829", "title": "About Applebot", "fetched_at": "2026-10-28T17:14:38.742Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://support.apple.com/en-us/119829", "title": "About Applebot", "fetched_at": "2026-10-28T17:14:38.742Z" } ] }, { "slug": "atlassian", "name": "Atlassian", "website": "https://www.atlassian.com", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "atlassian-bot", "coverage": "community", "user_agent_tokens": [ "atlassian-bot" ], "aliases": [], "purpose": "search", "purpose_quote": null, "search_engine": false, "robots_token": "atlassian-bot", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI search, assistants and agents\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "awario", "name": "Awario", "website": null, "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Awario", "coverage": "community", "user_agent_tokens": [ "Awario" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "Awario", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Data Scrapers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "baidu", "name": "Baidu", "website": null, "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "ERNIEBot", "coverage": "community", "user_agent_tokens": [ "ERNIEBot" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "ERNIEBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Collects public web content for Baidu's ERNIE large language models.\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "YiyanBot", "coverage": "community", "user_agent_tokens": [ "YiyanBot" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": null, "search_engine": false, "robots_token": "YiyanBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Assistants\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "big-sur-ai", "name": "Big Sur AI", "website": null, "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "bigsur.ai", "coverage": "community", "user_agent_tokens": [ "bigsur.ai" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": null, "search_engine": false, "robots_token": "bigsur.ai", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Assistants\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "bixel", "name": "Bixel", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt lists the operator as unclear; it is named in the agent's own description there.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "BixelBot", "coverage": "community", "user_agent_tokens": [ "BixelBot" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "BixelBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Its operator is named in its ai.robots.txt description. Function: \"AI Data Providers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "brave-search", "name": "Brave Search", "website": null, "docs_url": "https://safe.search.brave.com/help/brave-search-crawler", "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Bravebot", "coverage": "community", "user_agent_tokens": [ "Bravebot" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "Bravebot", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Data Providers\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "brightdata-com", "name": "brightdata.com", "website": null, "docs_url": "https://brightdata.com/brightbot", "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Brightbot", "coverage": "community", "user_agent_tokens": [ "Brightbot" ], "aliases": [ "Brightbot 1.0" ], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "Brightbot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"LLM/AI training.\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "browserbase", "name": "Browserbase", "website": "https://www.browserbase.com/", "docs_url": "https://docs.browserbase.com/platform/identity/web-bot-auth", "coverage": "researched", "note": null, "verification": { "signing": { "status": "live", "directory_url": "https://www.browserbase.com/.well-known/http-message-signatures-directory", "kid_convention": "thumbprint", "purpose_field": "rag", "covers": [ "Browserbase" ], "note": "Browserbase's docs say it offers Web Bot Auth to sign browser traffic (\"Browserbase offers Web Bot Auth in beta.\") and that it must be switched on per account (\"Contact Browserbase to enable Web Bot Auth for your account.\"), so only sessions of enabled customers are signed; unsigned Browserbase sessions look like ordinary browsers. The docs describe keys published at /.well-known/http-message-signatures-directory but do not name Browserbase's own directory URL or the Signature-Agent value it sends. The URL above is that standard path on browserbase.com (the bare domain redirects to www); on 2026-10-26 it answered 200 with content type application/http-message-signatures-directory+json and two Ed25519 keys. Which Browserbase products are signed (browser sessions only, or also Fetch) is not documented.", "sources": [ { "url": "https://docs.browserbase.com/platform/identity/web-bot-auth", "title": "Web Bot Auth", "fetched_at": "2026-10-28T17:14:43.044Z" }, { "url": "https://docs.browserbase.com/platform/identity/overview", "title": "Agent Identity", "fetched_at": "2026-10-28T17:14:42.018Z" }, { "url": "https://www.browserbase.com/blog/cloudflare-browserbase-pioneering-identity", "title": "Cloudflare & Browserbase: pioneering identity for AI agents", "fetched_at": "2026-10-28T17:14:46.063Z" }, { "url": "https://www.browserbase.com/.well-known/http-message-signatures-directory", "title": "Key directory", "fetched_at": "2026-10-28T17:14:39.808Z" } ] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Browserbase", "coverage": "researched", "user_agent_tokens": [], "aliases": [], "purpose": "agent", "purpose_quote": "Browserbase is the complete platform to build and deploy agents that browse and interact with the web like humans.", "search_engine": false, "robots_token": null, "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": true, "note": "Cloud browser sessions run by Browserbase customers' agents; the traffic is driven by the customer, not by Browserbase itself. Sessions present an ordinary browser User-Agent with no Browserbase product token: for Verified sessions, choosing an OS \"changes the user agent and browser environment to match the selected platform\" (source verified), and the Fetch API is routed \"with a real browser User-Agent by default\" (source fetch). No IP ranges, reverse DNS rule or ASN are published; sessions can also exit through Browserbase-managed proxies or a customer's own proxy. robots.txt: the docs advise customers to check robots.txt (\"Check robots.txt for crawling guidelines when applicable.\", best practices on the identity page) but do not say the platform itself enforces it, so the policy is recorded as unstated. The only verifiable identity is Web Bot Auth, which is in beta and enabled per customer account (see signing).", "sources": [ { "url": "https://docs.browserbase.com/welcome/what-is-browserbase", "title": "What is Browserbase?", "fetched_at": "2026-10-28T17:14:40.877Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://docs.browserbase.com/welcome/what-is-browserbase", "title": "What is Browserbase?", "fetched_at": "2026-10-28T17:14:40.877Z" }, { "url": "https://docs.browserbase.com/platform/identity/overview", "title": "Agent Identity", "fetched_at": "2026-10-28T17:14:42.018Z" }, { "url": "https://docs.browserbase.com/platform/identity/web-bot-auth", "title": "Web Bot Auth", "fetched_at": "2026-10-28T17:14:43.044Z" }, { "url": "https://docs.browserbase.com/platform/identity/verified-customization", "title": "Verified", "fetched_at": "2026-10-28T17:14:44.006Z" }, { "url": "https://docs.browserbase.com/platform/fetch/overview", "title": "Fetch", "fetched_at": "2026-10-28T17:14:45.074Z" }, { "url": "https://www.browserbase.com/blog/cloudflare-browserbase-pioneering-identity", "title": "Cloudflare & Browserbase: pioneering identity for AI agents", "fetched_at": "2026-10-28T17:14:46.063Z" } ] }, { "slug": "buddybotlearning", "name": "BuddyBotLearning", "website": "https://www.buddybotlearning.com", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "BuddyBot", "coverage": "community", "user_agent_tokens": [ "BuddyBot" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "BuddyBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Learning Companion\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "butterfly-effect", "name": "Butterfly Effect", "website": null, "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Manus-User", "coverage": "community", "user_agent_tokens": [ "Manus-User" ], "aliases": [], "purpose": "agent", "purpose_quote": null, "search_engine": false, "robots_token": "Manus-User", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Agents\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "bytedance", "name": "ByteDance", "website": null, "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Bytespider", "coverage": "community", "user_agent_tokens": [ "Bytespider" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "Bytespider", "robots_policy": "may_ignore", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"LLM training.\". Respects robots.txt: \"No\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "DoubaoBot", "coverage": "community", "user_agent_tokens": [ "DoubaoBot" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": null, "search_engine": false, "robots_token": "DoubaoBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI crawler for ByteDance's Doubao AI assistant.\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "TikTokSpider", "coverage": "community", "user_agent_tokens": [ "TikTokSpider" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "TikTokSpider", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"LLM training.\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "Trae", "coverage": "community", "user_agent_tokens": [ "Trae" ], "aliases": [], "purpose": "agent", "purpose_quote": null, "search_engine": false, "robots_token": "Trae", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Its operator is named in its ai.robots.txt description. Function: \"AI Coding Agents\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "cartai", "name": "CartAI", "website": null, "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "CartAI Agentic Commerce", "coverage": "community", "user_agent_tokens": [ "CartAI Agentic Commerce" ], "aliases": [], "purpose": "agent", "purpose_quote": null, "search_engine": false, "robots_token": "CartAI Agentic Commerce", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Agents\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "ceramic-ai", "name": "Ceramic AI", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt lists the operator as unclear; it is named in the agent's own description there.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Terra Cotta", "coverage": "community", "user_agent_tokens": [ "Terra Cotta" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "Terra Cotta", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Its operator is named in its ai.robots.txt description. Function: \"AI Data Providers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "TerraCotta", "coverage": "community", "user_agent_tokens": [ "TerraCotta" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "TerraCotta", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Data Providers\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "cledara", "name": "Cledara", "website": null, "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "CledaraBot", "coverage": "community", "user_agent_tokens": [ "CledaraBot" ], "aliases": [], "purpose": "agent", "purpose_quote": null, "search_engine": false, "robots_token": "CledaraBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Agents\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "cloudflare", "name": "Cloudflare", "website": null, "docs_url": "https://developers.cloudflare.com/autorag", "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Cloudflare-AutoRAG", "coverage": "community", "user_agent_tokens": [ "Cloudflare-AutoRAG" ], "aliases": [], "purpose": "search", "purpose_quote": null, "search_engine": false, "robots_token": "Cloudflare-AutoRAG", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Collects data for AI search\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "CloudflareBrowserRenderingCrawler", "coverage": "community", "user_agent_tokens": [ "CloudflareBrowserRenderingCrawler" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "CloudflareBrowserRenderingCrawler", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Data Providers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "cohere", "name": "Cohere", "website": "https://cohere.com", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "cohere-ai", "coverage": "community", "user_agent_tokens": [ "cohere-ai" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": null, "search_engine": false, "robots_token": "cohere-ai", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Retrieves data to provide responses to user-initiated prompts.\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "cohere-training-data-crawler", "coverage": "community", "user_agent_tokens": [ "cohere-training-data-crawler" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "cohere-training-data-crawler", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Data Scrapers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "common-crawl", "name": "Common Crawl", "website": "https://commoncrawl.org/", "docs_url": "https://commoncrawl.org/ccbot", "coverage": "researched", "note": null, "verification": { "signing": { "status": "none_found", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": "Common Crawl's pages do not mention HTTP message signatures or Web Bot Auth. On 2026-10-26 https://commoncrawl.org/.well-known/http-message-signatures-directory returned 404. No key directory found in the operator's documentation.", "sources": [] }, "ip_lists": [ { "agent": "ccbot", "url": "https://index.commoncrawl.org/ccbot.json", "format": "ipranges-json", "prefix_count": 5, "fetched_at": "2026-10-28T17:14:46.616Z", "sha256": "0a4c3b6707326d7f083139a979062604a57edfbb251bd29f23f5a86c5107ea30", "prefixes": [ "3.41.188.32/29", "18.97.9.168/29", "18.97.14.80/29", "18.97.14.88/30", "2600:1f28:365:8000::/56" ], "note": null, "sources": [ { "url": "https://commoncrawl.org/ccbot", "title": "Common Crawl - CCBot", "fetched_at": "2026-10-28T17:14:47.730Z" } ] } ], "reverse_dns": { "suffixes": [ "crawl.commoncrawl.org" ], "forward_confirm": true, "note": null, "sources": [ { "url": "https://commoncrawl.org/faq", "title": "Common Crawl FAQ", "fetched_at": "2026-10-28T17:14:48.841Z" } ] }, "asn": [] }, "agents": [ { "name": "CCBot", "coverage": "researched", "user_agent_tokens": [ "CCBot" ], "aliases": [], "purpose": "training", "purpose_quote": "Common Crawl data, typically after filtering and processing by third parties, has been a primary training data source for many large language models.", "search_engine": false, "robots_token": "CCBot", "robots_policy": "honors", "robots_quote": "Add these lines to your robots.txt file and our crawler will stop crawling your website", "ip_list_agent": "ccbot", "signs": false, "note": "CCBot builds Common Crawl's open web archive, which anyone can download. Model training is done by third parties using the published data, not by Common Crawl itself. The FAQ also says CCBot obeys Crawl-delay. The documented User-Agent is CCBot/2.0 (https://commoncrawl.org/faq/); the FAQ says an older version identified itself as CCBot/1.0. The reverse DNS suffix comes from the documented example, where 18.97.14.84 resolves to 18-97-14-84.crawl.commoncrawl.org and that name resolves forward to the same address. Reverse DNS covers IPv4 only: the FAQ marks the IPv6 range \"no reverse DNS\".", "sources": [ { "url": "https://commoncrawl.org/about", "title": "About Common Crawl", "fetched_at": "2026-10-28T17:14:49.836Z" }, { "url": "https://commoncrawl.org/faq", "title": "Common Crawl FAQ", "fetched_at": "2026-10-28T17:14:48.841Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://commoncrawl.org/ccbot", "title": "Common Crawl - CCBot", "fetched_at": "2026-10-28T17:14:47.730Z" }, { "url": "https://commoncrawl.org/faq", "title": "Common Crawl FAQ", "fetched_at": "2026-10-28T17:14:48.841Z" }, { "url": "https://commoncrawl.org/about", "title": "About Common Crawl", "fetched_at": "2026-10-28T17:14:49.836Z" } ] }, { "slug": "cragsoftware", "name": "CragSoftware", "website": null, "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "CragCrawler", "coverage": "community", "user_agent_tokens": [ "CragCrawler" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "CragCrawler", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Data Providers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "crawlspace", "name": "Crawlspace", "website": "https://crawlspace.dev", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Crawlspace", "coverage": "community", "user_agent_tokens": [ "Crawlspace" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "Crawlspace", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Data Providers\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "datenbank", "name": "Datenbank", "website": null, "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Datenbank Crawler", "coverage": "community", "user_agent_tokens": [ "Datenbank Crawler" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "Datenbank Crawler", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Data Scrapers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "deepseek", "name": "DeepSeek", "website": null, "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "DeepSeekBot", "coverage": "community", "user_agent_tokens": [ "DeepSeekBot" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "DeepSeekBot", "robots_policy": "may_ignore", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Training language models and improving AI products\". Respects robots.txt: \"No\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "devin-ai", "name": "Devin AI", "website": null, "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Devin", "coverage": "community", "user_agent_tokens": [ "Devin" ], "aliases": [], "purpose": "agent", "purpose_quote": null, "search_engine": false, "robots_token": "Devin", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Coding Agents\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "diffbot", "name": "Diffbot", "website": "https://www.diffbot.com/", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Diffbot", "coverage": "community", "user_agent_tokens": [ "Diffbot" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "Diffbot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Data Providers\". Respects robots.txt: \"At the discretion of Diffbot users.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "Diffbot-User", "coverage": "community", "user_agent_tokens": [ "Diffbot-User" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": null, "search_engine": false, "robots_token": "Diffbot-User", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Assistants\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "direqt", "name": "Direqt", "website": "https://direqt.ai", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Anomura", "coverage": "community", "user_agent_tokens": [ "Anomura" ], "aliases": [], "purpose": "search", "purpose_quote": null, "search_engine": false, "robots_token": "Anomura", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Collects data for AI search\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "duckduckgo", "name": "DuckDuckGo", "website": "https://duckduckgo.com/", "docs_url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot", "coverage": "researched", "note": null, "verification": { "signing": { "status": "live", "directory_url": "https://assistbot.duckduckgo.com/.well-known/http-message-signatures-directory", "kid_convention": "thumbprint", "purpose_field": null, "covers": [ "DuckAssistBot" ], "note": "DuckDuckGo's help pages do not mention HTTP message signatures or Web Bot Auth. A third-party survey (mod_pagespeed blog, probe of 19 October 2026) lists this directory, which it took from Cloudflare Radar, but does not say which DuckDuckGo agent signs. On 2026-10-28 Posterna's agent lab saw Duck.ai fetch a page as DuckAssistBot/1.2, signed with the key in this directory (keyid Ov3HDsa8JQ39dPEYFvFFN-cUpnz9yNI8LDvr-5LeiBM, alg ed25519, Signature-Agent \"https://assistbot.duckduckgo.com\", 600-second window, no nonce), recorded in the public observation note, so covers lists DuckAssistBot. On 2026-10-26 the URL returned 200 with content type application/http-message-signatures-directory+json and one Ed25519 key with a kid. https://duckduckgo.com/ returns 404 for the well-known path.", "sources": [ { "url": "https://modpagespeed.com/blog/which-ai-crawlers-sign-their-requests/", "title": "55 of 150 AI crawler operators publish Web Bot Auth keys (third party, mod_pagespeed blog)", "fetched_at": "2026-10-28T17:14:32.840Z" }, { "url": "https://modpagespeed.com/data/web-bot-auth-2026-10-19/results.tsv", "title": "Web Bot Auth key directory probe results, 19 October 2026 (third party)", "fetched_at": "2026-10-28T17:14:34.046Z" }, { "url": "https://github.com/posternahq/posterna/blob/main/docs/notes/real-traffic-checks.md#manual-lab-observations", "title": "Posterna real-traffic checks, manual lab observations, 28 October 2026 (first-party observation)", "fetched_at": "2026-10-28T17:14:55.658Z" }, { "url": "https://assistbot.duckduckgo.com/.well-known/http-message-signatures-directory", "title": "Key directory", "fetched_at": "2026-10-28T17:14:52.955Z" } ] }, "ip_lists": [ { "agent": "duckduckbot", "url": "https://duckduckgo.com/duckduckbot.json", "format": "ipranges-json", "prefix_count": 486, "fetched_at": "2026-10-28T17:14:50.883Z", "sha256": "fd9039362dfaaad236d50c07ddd12bf0cc7f0d530fbff59b8feb1b22e0db2459", "prefixes": [ "4.144.148.23/32", "4.144.182.50/32", "4.144.227.142/32", "4.144.251.39/32", "4.149.142.7/32", "4.149.159.71/32", "4.149.159.102/32", "4.149.216.220/32", "4.150.142.218/32", "4.154.193.167/32", "4.156.30.123/32", "4.156.136.79/32", "4.156.154.107/32", "4.157.77.76/32", "4.172.49.103/32", "4.177.16.196/32", "4.182.10.198/32", "4.182.131.108/32", "4.195.133.120/32", "4.201.125.59/32", "4.201.141.71/32", "4.201.197.203/32", "4.201.206.133/32", "4.201.220.8/32", "4.207.220.92/32", "4.209.224.56/32", "4.213.46.14/32", "4.226.40.135/32", "4.228.76.163/32", "4.237.155.54/32", "4.237.244.80/32", "4.242.119.13/32", "4.246.10.252/32", "4.248.234.26/32", "4.249.216.104/32", "4.249.223.51/32", "4.249.238.11/32", "4.254.123.56/32", "4.255.35.121/32", "4.255.39.80/32", "13.86.35.212/32", "13.89.106.77/32", "13.90.178.139/32", "20.3.1.178/32", "20.3.60.212/32", "20.8.252.26/32", "20.9.114.242/32", "20.12.141.99/32", "20.12.249.123/32", "20.12.252.21/32", "20.13.44.19/32", "20.15.142.178/32", "20.23.88.231/32", "20.29.149.44/32", "20.29.154.56/32", "20.40.133.240/32", "20.40.147.172/32", "20.43.150.85/32", "20.43.150.93/32", "20.43.152.28/32", "20.43.172.120/32", "20.44.222.1/32", "20.49.129.236/32", "20.49.136.28/32", "20.50.48.159/32", "20.50.48.192/32", "20.50.49.0/32", "20.50.49.25/32", "20.50.49.40/32", "20.50.49.55/32", "20.50.49.237/32", "20.50.50.46/32", "20.50.50.118/32", "20.50.50.121/32", "20.50.50.123/32", "20.50.50.130/32", "20.50.50.134/32", "20.50.50.145/32", "20.50.50.146/32", "20.50.50.163/32", "20.50.168.91/32", "20.53.78.106/32", "20.53.78.123/32", "20.53.78.138/32", "20.53.78.144/32", "20.53.78.236/32", "20.53.91.2/32", "20.53.92.211/32", "20.53.134.160/32", "20.54.224.39/32", "20.56.197.58/32", "20.56.197.63/32", "20.59.0.65/32", "20.61.34.40/32", "20.61.142.192/32", "20.62.224.44/32", "20.64.185.45/32", "20.69.74.250/32", "20.69.129.205/32", "20.69.131.45/32", "20.69.138.149/32", "20.69.138.189/32", "20.69.144.137/32", "20.69.146.171/32", "20.69.147.21/32", "20.69.171.204/32", "20.71.12.143/32", "20.71.69.210/32", "20.72.197.110/32", "20.72.203.250/32", "20.72.218.29/32", "20.72.221.33/32", "20.72.221.45/32", "20.72.225.13/32", "20.72.242.93/32", "20.72.242.209/32", "20.73.132.240/32", "20.73.202.147/32", "20.75.144.152/32", "20.75.254.61/32", "20.77.146.108/32", "20.79.226.26/32", "20.79.238.198/32", "20.79.239.66/32", "20.80.114.203/32", "20.80.120.41/32", "20.80.129.80/32", "20.80.135.7/32", "20.80.156.88/32", "20.81.115.13/32", "20.82.24.207/32", "20.84.138.61/32", "20.84.219.45/32", "20.84.251.223/32", "20.85.138.75/32", "20.85.167.56/32", "20.85.187.146/32", "20.88.168.65/32", "20.93.28.24/32", "20.93.49.96/32", "20.99.154.113/32", "20.99.255.235/32", "20.100.136.36/32", "20.100.140.155/32", "20.101.17.173/32", "20.102.34.205/32", "20.109.129.162/32", "20.109.130.190/32", "20.109.241.18/32", "20.112.58.44/32", "20.113.3.121/32", "20.113.14.159/32", "20.115.181.187/32", "20.118.11.251/32", "20.119.127.146/32", "20.121.75.32/32", "20.166.171.150/32", "20.170.75.54/32", "20.175.232.228/32", "20.185.79.15/32", "20.185.79.47/32", "20.190.21.172/32", "20.191.44.16/32", "20.191.44.22/32", "20.191.44.119/32", "20.191.44.234/32", "20.191.45.212/32", "20.193.12.126/32", "20.193.24.10/32", "20.193.24.251/32", "20.193.25.197/32", "20.193.27.215/32", "20.193.45.113/32", "20.195.58.189/32", "20.195.108.47/32", "20.197.209.11/32", "20.197.209.27/32", "20.198.132.126/32", "20.198.152.208/32", "20.201.15.208/32", "20.204.228.200/32", "20.204.240.172/32", "20.204.241.148/32", "20.204.242.19/32", "20.204.242.101/32", "20.204.243.55/32", "20.204.246.81/32", "20.204.246.254/32", "20.207.72.11/32", "20.207.72.21/32", "20.207.72.110/32", "20.207.72.113/32", "20.207.97.190/32", "20.207.99.197/32", "20.207.107.181/32", "20.207.117.66/32", "20.212.90.107/32", "20.212.139.34/32", "20.216.200.223/32", "20.219.43.246/32", "20.219.45.67/32", "20.219.45.190/32", "20.221.10.115/32", "20.221.84.185/32", "20.226.133.105/32", "20.231.254.179/32", "20.232.51.46/32", "20.232.249.95/32", "20.236.40.93/32", "20.236.195.196/32", "20.236.198.192/32", "20.236.227.113/32", "20.236.233.23/32", "20.236.247.180/32", "20.237.1.51/32", "20.241.135.19/32", "20.241.147.151/32", "20.241.153.203/32", "20.241.164.5/32", "20.241.241.100/32", "20.242.133.53/32", "20.242.135.165/32", "20.242.211.139/32", "20.242.215.53/32", "20.242.237.240/32", "20.246.132.168/32", "20.246.225.34/32", "20.247.235.11/32", "20.250.51.113/32", "20.252.9.6/32", "20.252.16.193/32", "20.252.41.18/32", "20.252.63.162/32", "20.252.88.10/32", "20.253.59.76/32", "20.253.96.199/32", "20.253.114.51/32", "40.64.67.220/32", "40.64.78.45/32", "40.64.88.145/32", "40.64.105.247/32", "40.64.106.11/32", "40.64.106.152/32", "40.65.78.86/32", "40.76.162.17/32", "40.76.162.191/32", "40.76.162.208/32", "40.76.162.247/32", "40.76.163.7/32", "40.76.163.23/32", "40.76.173.151/32", "40.81.250.205/32", "40.82.218.203/32", "40.88.21.235/32", "40.88.195.62/32", "40.89.243.175/32", "40.114.182.45/32", "40.114.182.153/32", "40.114.182.172/32", "40.114.183.88/32", "40.114.183.196/32", "40.114.183.251/32", "40.119.232.50/32", "40.119.232.146/32", "40.119.232.215/32", "40.119.232.218/32", "40.119.232.251/32", "40.121.121.54/32", "40.127.154.196/32", "48.192.16.86/32", "48.192.74.178/32", "48.200.33.227/32", "48.202.155.200/32", "48.202.171.124/32", "48.204.32.87/32", "48.206.18.152/32", "48.217.23.236/32", "48.217.129.210/32", "48.217.212.89/32", "48.223.209.110/32", "51.8.31.28/32", "51.8.71.117/32", "51.8.253.152/32", "51.104.144.31/32", "51.104.146.225/32", "51.104.146.235/32", "51.104.160.167/32", "51.104.160.177/32", "51.104.161.32/32", "51.104.162.149/32", "51.104.163.250/32", "51.104.164.109/32", "51.104.164.147/32", "51.104.164.189/32", "51.104.164.215/32", "51.104.166.111/32", "51.104.167.19/32", "51.104.167.52/32", "51.104.167.54/32", "51.104.167.61/32", "51.104.167.71/32", "51.104.167.87/32", "51.104.167.88/32", "51.104.167.95/32", "51.104.167.96/32", "51.104.167.104/32", "51.104.167.110/32", "51.104.180.26/32", "51.104.180.47/32", "51.104.180.53/32", "51.107.40.209/32", "51.116.131.221/32", "51.120.48.122/32", "51.138.90.161/32", "51.138.90.206/32", "51.138.90.233/32", "51.143.114.45/32", "51.145.185.16/32", "52.142.24.149/32", "52.142.26.175/32", "52.143.95.162/32", "52.143.95.204/32", "52.143.241.111/32", "52.143.242.6/32", "52.143.243.117/32", "52.143.244.81/32", "52.143.247.235/32", "52.146.57.51/32", "52.146.58.236/32", "52.146.59.12/32", "52.146.59.154/32", "52.146.59.156/32", "52.146.63.80/32", "52.148.161.87/32", "52.148.165.38/32", "52.149.25.43/32", "52.149.28.18/32", "52.149.28.83/32", "52.149.30.45/32", "52.149.56.151/32", "52.149.58.27/32", "52.149.58.69/32", "52.149.58.139/32", "52.149.58.173/32", "52.149.60.38/32", "52.149.61.51/32", "52.154.60.82/32", "52.154.169.50/32", "52.154.169.200/32", "52.154.170.26/32", "52.154.170.28/32", "52.154.170.88/32", "52.154.170.96/32", "52.154.170.113/32", "52.154.170.117/32", "52.154.170.122/32", "52.154.170.209/32", "52.154.170.229/32", "52.154.170.243/32", "52.154.171.0/32", "52.154.171.44/32", "52.154.171.70/32", "52.154.171.87/32", "52.154.171.150/32", "52.154.171.196/32", "52.154.171.205/32", "52.154.171.235/32", "52.154.171.250/32", "52.154.172.2/32", "52.154.204.93/32", "52.185.93.208/32", "52.186.37.211/32", "52.188.89.106/32", "52.189.70.103/32", "52.190.37.160/32", "52.191.82.242/32", "52.191.222.158/32", "52.224.16.221/32", "52.224.16.229/32", "52.224.19.152/32", "52.224.20.174/32", "52.224.20.181/32", "52.224.20.186/32", "52.224.20.190/32", "52.224.20.193/32", "52.224.20.203/32", "52.224.20.204/32", "52.224.20.223/32", "52.224.20.227/32", "52.224.20.249/32", "52.224.21.4/32", "52.224.21.19/32", "52.224.21.20/32", "52.224.21.23/32", "52.224.21.27/32", "52.224.21.49/32", "52.224.21.51/32", "52.224.21.53/32", "52.224.21.55/32", "52.224.21.61/32", "52.224.147.21/32", "52.224.191.245/32", "52.242.224.168/32", "52.250.46.221/32", "57.151.44.140/32", "57.152.17.237/32", "57.152.62.107/32", "57.152.72.128/32", "57.155.18.44/32", "57.165.205.172/32", "57.167.88.38/32", "64.236.15.33/32", "64.236.93.100/32", "64.236.118.43/32", "68.219.152.220/32", "72.144.95.185/32", "74.163.169.106/32", "74.163.208.206/32", "74.179.200.126/32", "74.179.232.116/32", "74.242.194.48/32", "98.70.219.37/32", "104.43.54.127/32", "104.43.55.116/32", "104.43.55.117/32", "104.43.55.166/32", "104.43.55.167/32", "108.141.83.74/32", "128.203.82.154/32", "128.203.132.152/32", "128.203.181.79/32", "128.203.214.241/32", "128.203.236.110/32", "130.107.228.224/32", "131.145.39.118/32", "131.189.183.241/32", "132.164.209.198/32", "132.196.106.146/32", "132.220.1.71/32", "134.33.207.189/32", "135.171.251.169/32", "135.233.63.237/32", "135.234.221.112/32", "135.235.183.95/32", "172.168.12.93/32", "172.168.43.43/32", "172.168.51.216/32", "172.168.53.53/32", "172.168.81.217/32", "172.168.94.160/32", "172.168.115.250/32", "172.168.133.76/32", "172.168.137.102/32", "172.168.152.88/32", "172.168.173.230/32", "172.168.190.186/32", "172.168.227.120/32", "172.168.254.119/32", "172.169.7.33/32", "172.169.17.165/32", "172.169.28.184/32", "172.169.36.43/32", "172.169.53.208/32", "172.169.60.134/32", "172.169.93.148/32", "172.169.164.109/32", "172.169.177.131/32", "172.169.181.237/32", "172.171.37.221/32", "172.179.105.87/32", "172.179.128.118/32", "172.188.250.220/32", "172.189.40.18/32", "172.193.184.42/32", "172.193.197.146/32", "172.193.245.229/32", "172.194.141.49/32", "172.199.55.212/32", "172.210.111.68/32", "191.233.3.197/32", "191.233.3.202/32", "191.234.216.4/32", "191.234.216.178/32", "191.235.201.214/32", "191.235.202.38/32", "191.235.202.48/32" ], "note": null, "sources": [ { "url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot", "title": "Is DuckDuckBot related to DuckDuckGo? (DuckDuckGo Help Pages)", "fetched_at": "2026-10-28T17:14:54.033Z" } ] }, { "agent": "duckassistbot", "url": "https://duckduckgo.com/duckassistbot.json", "format": "ipranges-json", "prefix_count": 486, "fetched_at": "2026-10-28T17:14:51.960Z", "sha256": "fd9039362dfaaad236d50c07ddd12bf0cc7f0d530fbff59b8feb1b22e0db2459", "prefixes": [ "4.144.148.23/32", "4.144.182.50/32", "4.144.227.142/32", "4.144.251.39/32", "4.149.142.7/32", "4.149.159.71/32", "4.149.159.102/32", "4.149.216.220/32", "4.150.142.218/32", "4.154.193.167/32", "4.156.30.123/32", "4.156.136.79/32", "4.156.154.107/32", "4.157.77.76/32", "4.172.49.103/32", "4.177.16.196/32", "4.182.10.198/32", "4.182.131.108/32", "4.195.133.120/32", "4.201.125.59/32", "4.201.141.71/32", "4.201.197.203/32", "4.201.206.133/32", "4.201.220.8/32", "4.207.220.92/32", "4.209.224.56/32", "4.213.46.14/32", "4.226.40.135/32", "4.228.76.163/32", "4.237.155.54/32", "4.237.244.80/32", "4.242.119.13/32", "4.246.10.252/32", "4.248.234.26/32", "4.249.216.104/32", "4.249.223.51/32", "4.249.238.11/32", "4.254.123.56/32", "4.255.35.121/32", "4.255.39.80/32", "13.86.35.212/32", "13.89.106.77/32", "13.90.178.139/32", "20.3.1.178/32", "20.3.60.212/32", "20.8.252.26/32", "20.9.114.242/32", "20.12.141.99/32", "20.12.249.123/32", "20.12.252.21/32", "20.13.44.19/32", "20.15.142.178/32", "20.23.88.231/32", "20.29.149.44/32", "20.29.154.56/32", "20.40.133.240/32", "20.40.147.172/32", "20.43.150.85/32", "20.43.150.93/32", "20.43.152.28/32", "20.43.172.120/32", "20.44.222.1/32", "20.49.129.236/32", "20.49.136.28/32", "20.50.48.159/32", "20.50.48.192/32", "20.50.49.0/32", "20.50.49.25/32", "20.50.49.40/32", "20.50.49.55/32", "20.50.49.237/32", "20.50.50.46/32", "20.50.50.118/32", "20.50.50.121/32", "20.50.50.123/32", "20.50.50.130/32", "20.50.50.134/32", "20.50.50.145/32", "20.50.50.146/32", "20.50.50.163/32", "20.50.168.91/32", "20.53.78.106/32", "20.53.78.123/32", "20.53.78.138/32", "20.53.78.144/32", "20.53.78.236/32", "20.53.91.2/32", "20.53.92.211/32", "20.53.134.160/32", "20.54.224.39/32", "20.56.197.58/32", "20.56.197.63/32", "20.59.0.65/32", "20.61.34.40/32", "20.61.142.192/32", "20.62.224.44/32", "20.64.185.45/32", "20.69.74.250/32", "20.69.129.205/32", "20.69.131.45/32", "20.69.138.149/32", "20.69.138.189/32", "20.69.144.137/32", "20.69.146.171/32", "20.69.147.21/32", "20.69.171.204/32", "20.71.12.143/32", "20.71.69.210/32", "20.72.197.110/32", "20.72.203.250/32", "20.72.218.29/32", "20.72.221.33/32", "20.72.221.45/32", "20.72.225.13/32", "20.72.242.93/32", "20.72.242.209/32", "20.73.132.240/32", "20.73.202.147/32", "20.75.144.152/32", "20.75.254.61/32", "20.77.146.108/32", "20.79.226.26/32", "20.79.238.198/32", "20.79.239.66/32", "20.80.114.203/32", "20.80.120.41/32", "20.80.129.80/32", "20.80.135.7/32", "20.80.156.88/32", "20.81.115.13/32", "20.82.24.207/32", "20.84.138.61/32", "20.84.219.45/32", "20.84.251.223/32", "20.85.138.75/32", "20.85.167.56/32", "20.85.187.146/32", "20.88.168.65/32", "20.93.28.24/32", "20.93.49.96/32", "20.99.154.113/32", "20.99.255.235/32", "20.100.136.36/32", "20.100.140.155/32", "20.101.17.173/32", "20.102.34.205/32", "20.109.129.162/32", "20.109.130.190/32", "20.109.241.18/32", "20.112.58.44/32", "20.113.3.121/32", "20.113.14.159/32", "20.115.181.187/32", "20.118.11.251/32", "20.119.127.146/32", "20.121.75.32/32", "20.166.171.150/32", "20.170.75.54/32", "20.175.232.228/32", "20.185.79.15/32", "20.185.79.47/32", "20.190.21.172/32", "20.191.44.16/32", "20.191.44.22/32", "20.191.44.119/32", "20.191.44.234/32", "20.191.45.212/32", "20.193.12.126/32", "20.193.24.10/32", "20.193.24.251/32", "20.193.25.197/32", "20.193.27.215/32", "20.193.45.113/32", "20.195.58.189/32", "20.195.108.47/32", "20.197.209.11/32", "20.197.209.27/32", "20.198.132.126/32", "20.198.152.208/32", "20.201.15.208/32", "20.204.228.200/32", "20.204.240.172/32", "20.204.241.148/32", "20.204.242.19/32", "20.204.242.101/32", "20.204.243.55/32", "20.204.246.81/32", "20.204.246.254/32", "20.207.72.11/32", "20.207.72.21/32", "20.207.72.110/32", "20.207.72.113/32", "20.207.97.190/32", "20.207.99.197/32", "20.207.107.181/32", "20.207.117.66/32", "20.212.90.107/32", "20.212.139.34/32", "20.216.200.223/32", "20.219.43.246/32", "20.219.45.67/32", "20.219.45.190/32", "20.221.10.115/32", "20.221.84.185/32", "20.226.133.105/32", "20.231.254.179/32", "20.232.51.46/32", "20.232.249.95/32", "20.236.40.93/32", "20.236.195.196/32", "20.236.198.192/32", "20.236.227.113/32", "20.236.233.23/32", "20.236.247.180/32", "20.237.1.51/32", "20.241.135.19/32", "20.241.147.151/32", "20.241.153.203/32", "20.241.164.5/32", "20.241.241.100/32", "20.242.133.53/32", "20.242.135.165/32", "20.242.211.139/32", "20.242.215.53/32", "20.242.237.240/32", "20.246.132.168/32", "20.246.225.34/32", "20.247.235.11/32", "20.250.51.113/32", "20.252.9.6/32", "20.252.16.193/32", "20.252.41.18/32", "20.252.63.162/32", "20.252.88.10/32", "20.253.59.76/32", "20.253.96.199/32", "20.253.114.51/32", "40.64.67.220/32", "40.64.78.45/32", "40.64.88.145/32", "40.64.105.247/32", "40.64.106.11/32", "40.64.106.152/32", "40.65.78.86/32", "40.76.162.17/32", "40.76.162.191/32", "40.76.162.208/32", "40.76.162.247/32", "40.76.163.7/32", "40.76.163.23/32", "40.76.173.151/32", "40.81.250.205/32", "40.82.218.203/32", "40.88.21.235/32", "40.88.195.62/32", "40.89.243.175/32", "40.114.182.45/32", "40.114.182.153/32", "40.114.182.172/32", "40.114.183.88/32", "40.114.183.196/32", "40.114.183.251/32", "40.119.232.50/32", "40.119.232.146/32", "40.119.232.215/32", "40.119.232.218/32", "40.119.232.251/32", "40.121.121.54/32", "40.127.154.196/32", "48.192.16.86/32", "48.192.74.178/32", "48.200.33.227/32", "48.202.155.200/32", "48.202.171.124/32", "48.204.32.87/32", "48.206.18.152/32", "48.217.23.236/32", "48.217.129.210/32", "48.217.212.89/32", "48.223.209.110/32", "51.8.31.28/32", "51.8.71.117/32", "51.8.253.152/32", "51.104.144.31/32", "51.104.146.225/32", "51.104.146.235/32", "51.104.160.167/32", "51.104.160.177/32", "51.104.161.32/32", "51.104.162.149/32", "51.104.163.250/32", "51.104.164.109/32", "51.104.164.147/32", "51.104.164.189/32", "51.104.164.215/32", "51.104.166.111/32", "51.104.167.19/32", "51.104.167.52/32", "51.104.167.54/32", "51.104.167.61/32", "51.104.167.71/32", "51.104.167.87/32", "51.104.167.88/32", "51.104.167.95/32", "51.104.167.96/32", "51.104.167.104/32", "51.104.167.110/32", "51.104.180.26/32", "51.104.180.47/32", "51.104.180.53/32", "51.107.40.209/32", "51.116.131.221/32", "51.120.48.122/32", "51.138.90.161/32", "51.138.90.206/32", "51.138.90.233/32", "51.143.114.45/32", "51.145.185.16/32", "52.142.24.149/32", "52.142.26.175/32", "52.143.95.162/32", "52.143.95.204/32", "52.143.241.111/32", "52.143.242.6/32", "52.143.243.117/32", "52.143.244.81/32", "52.143.247.235/32", "52.146.57.51/32", "52.146.58.236/32", "52.146.59.12/32", "52.146.59.154/32", "52.146.59.156/32", "52.146.63.80/32", "52.148.161.87/32", "52.148.165.38/32", "52.149.25.43/32", "52.149.28.18/32", "52.149.28.83/32", "52.149.30.45/32", "52.149.56.151/32", "52.149.58.27/32", "52.149.58.69/32", "52.149.58.139/32", "52.149.58.173/32", "52.149.60.38/32", "52.149.61.51/32", "52.154.60.82/32", "52.154.169.50/32", "52.154.169.200/32", "52.154.170.26/32", "52.154.170.28/32", "52.154.170.88/32", "52.154.170.96/32", "52.154.170.113/32", "52.154.170.117/32", "52.154.170.122/32", "52.154.170.209/32", "52.154.170.229/32", "52.154.170.243/32", "52.154.171.0/32", "52.154.171.44/32", "52.154.171.70/32", "52.154.171.87/32", "52.154.171.150/32", "52.154.171.196/32", "52.154.171.205/32", "52.154.171.235/32", "52.154.171.250/32", "52.154.172.2/32", "52.154.204.93/32", "52.185.93.208/32", "52.186.37.211/32", "52.188.89.106/32", "52.189.70.103/32", "52.190.37.160/32", "52.191.82.242/32", "52.191.222.158/32", "52.224.16.221/32", "52.224.16.229/32", "52.224.19.152/32", "52.224.20.174/32", "52.224.20.181/32", "52.224.20.186/32", "52.224.20.190/32", "52.224.20.193/32", "52.224.20.203/32", "52.224.20.204/32", "52.224.20.223/32", "52.224.20.227/32", "52.224.20.249/32", "52.224.21.4/32", "52.224.21.19/32", "52.224.21.20/32", "52.224.21.23/32", "52.224.21.27/32", "52.224.21.49/32", "52.224.21.51/32", "52.224.21.53/32", "52.224.21.55/32", "52.224.21.61/32", "52.224.147.21/32", "52.224.191.245/32", "52.242.224.168/32", "52.250.46.221/32", "57.151.44.140/32", "57.152.17.237/32", "57.152.62.107/32", "57.152.72.128/32", "57.155.18.44/32", "57.165.205.172/32", "57.167.88.38/32", "64.236.15.33/32", "64.236.93.100/32", "64.236.118.43/32", "68.219.152.220/32", "72.144.95.185/32", "74.163.169.106/32", "74.163.208.206/32", "74.179.200.126/32", "74.179.232.116/32", "74.242.194.48/32", "98.70.219.37/32", "104.43.54.127/32", "104.43.55.116/32", "104.43.55.117/32", "104.43.55.166/32", "104.43.55.167/32", "108.141.83.74/32", "128.203.82.154/32", "128.203.132.152/32", "128.203.181.79/32", "128.203.214.241/32", "128.203.236.110/32", "130.107.228.224/32", "131.145.39.118/32", "131.189.183.241/32", "132.164.209.198/32", "132.196.106.146/32", "132.220.1.71/32", "134.33.207.189/32", "135.171.251.169/32", "135.233.63.237/32", "135.234.221.112/32", "135.235.183.95/32", "172.168.12.93/32", "172.168.43.43/32", "172.168.51.216/32", "172.168.53.53/32", "172.168.81.217/32", "172.168.94.160/32", "172.168.115.250/32", "172.168.133.76/32", "172.168.137.102/32", "172.168.152.88/32", "172.168.173.230/32", "172.168.190.186/32", "172.168.227.120/32", "172.168.254.119/32", "172.169.7.33/32", "172.169.17.165/32", "172.169.28.184/32", "172.169.36.43/32", "172.169.53.208/32", "172.169.60.134/32", "172.169.93.148/32", "172.169.164.109/32", "172.169.177.131/32", "172.169.181.237/32", "172.171.37.221/32", "172.179.105.87/32", "172.179.128.118/32", "172.188.250.220/32", "172.189.40.18/32", "172.193.184.42/32", "172.193.197.146/32", "172.193.245.229/32", "172.194.141.49/32", "172.199.55.212/32", "172.210.111.68/32", "191.233.3.197/32", "191.233.3.202/32", "191.234.216.4/32", "191.234.216.178/32", "191.235.201.214/32", "191.235.202.38/32", "191.235.202.48/32" ], "note": null, "sources": [ { "url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot", "title": "Is DuckAssistBot related to DuckDuckGo? (DuckDuckGo Help Pages)", "fetched_at": "2026-10-28T17:14:54.726Z" } ] } ], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "DuckAssistBot", "coverage": "researched", "user_agent_tokens": [ "DuckAssistBot" ], "aliases": [], "purpose": "search", "purpose_quote": "DuckAssistBot is a web crawler for DuckDuckGo Search that crawls pages in real-time for our AI-assisted answers", "search_engine": false, "robots_token": "DuckAssistBot", "robots_policy": "honors", "robots_quote": "Once you’ve updated the file to disallow the DuckAssistBot user agent, the change will take effect after 72 hours and DuckAssistBot will stop crawling your site.", "ip_list_agent": "duckassistbot", "signs": true, "note": "Fetches pages in real time for AI-assisted answers in DuckDuckGo search results. The page says \"This data is not used in any way to train AI models.\" and that opting out does not affect organic search rankings. The documented User-Agent is DuckAssistBot/1.2; (+http://duckduckgo.com/duckassistbot.html).", "sources": [ { "url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot", "title": "Is DuckAssistBot related to DuckDuckGo? (DuckDuckGo Help Pages)", "fetched_at": "2026-10-28T17:14:54.726Z" } ] }, { "name": "DuckDuckBot", "coverage": "researched", "user_agent_tokens": [ "DuckDuckBot" ], "aliases": [], "purpose": "search", "purpose_quote": "DuckDuckBot’s job is to constantly improve our search results and offer users the best and most secure search experience possible.", "search_engine": false, "robots_token": "DuckDuckBot", "robots_policy": "honors", "robots_quote": "It respects WWW::RobotRules and originates from these IP addresses", "ip_list_agent": "duckduckbot", "signs": false, "note": "DuckDuckGo's search crawler. search_engine is false only because the directory flags a short list of search crawlers for now. WWW::RobotRules (linked on the page) is a Perl robots.txt parser. The documented User-Agent is DuckDuckBot/1.1; (+http://duckduckgo.com/duckduckbot.html).", "sources": [ { "url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot", "title": "Is DuckDuckBot related to DuckDuckGo? (DuckDuckGo Help Pages)", "fetched_at": "2026-10-28T17:14:54.033Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot", "title": "Is DuckDuckBot related to DuckDuckGo? (DuckDuckGo Help Pages)", "fetched_at": "2026-10-28T17:14:54.033Z" }, { "url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot", "title": "Is DuckAssistBot related to DuckDuckGo? (DuckDuckGo Help Pages)", "fetched_at": "2026-10-28T17:14:54.726Z" }, { "url": "https://modpagespeed.com/blog/which-ai-crawlers-sign-their-requests/", "title": "55 of 150 AI crawler operators publish Web Bot Auth keys (third party, mod_pagespeed blog)", "fetched_at": "2026-10-28T17:14:32.840Z" }, { "url": "https://modpagespeed.com/data/web-bot-auth-2026-10-19/results.tsv", "title": "Web Bot Auth key directory probe results, 19 October 2026 (third party)", "fetched_at": "2026-10-28T17:14:34.046Z" }, { "url": "https://github.com/posternahq/posterna/blob/main/docs/notes/real-traffic-checks.md#manual-lab-observations", "title": "Posterna real-traffic checks, manual lab observations, 28 October 2026 (first-party observation)", "fetched_at": "2026-10-28T17:14:55.658Z" } ] }, { "slug": "echobox", "name": "Echobox", "website": null, "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Echobot Bot", "coverage": "community", "user_agent_tokens": [ "Echobot Bot" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "Echobot Bot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Data Scrapers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "EchoboxBot", "coverage": "community", "user_agent_tokens": [ "EchoboxBot" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "EchoboxBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Data collection to support AI-powered products.\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "elevenlabs", "name": "ElevenLabs", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt lists the operator as unclear; it is named in the agent's own description there.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "ElevenlabsBot", "coverage": "community", "user_agent_tokens": [ "ElevenlabsBot" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": null, "search_engine": false, "robots_token": "ElevenlabsBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Its operator is named in its ai.robots.txt description. Function: \"AI Assistants\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "exa", "name": "Exa", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt lists the operator as unclear; it is named in the agent's own description there.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "ExaBot", "coverage": "community", "user_agent_tokens": [ "ExaBot" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "ExaBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Its operator is named in its ai.robots.txt description. Function: \"AI Data Providers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "ExaSearchBot", "coverage": "community", "user_agent_tokens": [ "ExaSearchBot" ], "aliases": [], "purpose": "search", "purpose_quote": null, "search_engine": false, "robots_token": "ExaSearchBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Search Crawlers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "ExaUser", "coverage": "community", "user_agent_tokens": [ "ExaUser" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": null, "search_engine": false, "robots_token": "ExaUser", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Its operator is named in its ai.robots.txt description. Function: \"AI Assistants\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "factset", "name": "Factset", "website": null, "docs_url": "https://www.factset.com/ai", "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Factset_spyderbot", "coverage": "community", "user_agent_tokens": [ "Factset_spyderbot" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "Factset_spyderbot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI model training.\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "firecrawl", "name": "Firecrawl", "website": null, "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "FirecrawlAgent", "coverage": "community", "user_agent_tokens": [ "FirecrawlAgent" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "FirecrawlAgent", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Data Providers\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "geisthaus", "name": "GeistHaus", "website": null, "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "GeistHaus-PageFetcher", "coverage": "community", "user_agent_tokens": [ "GeistHaus-PageFetcher" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": null, "search_engine": false, "robots_token": "GeistHaus-PageFetcher", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Assistants\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "google", "name": "Google", "website": "https://www.google.com/", "docs_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/overview-google-crawlers", "coverage": "researched", "note": null, "verification": { "signing": { "status": "live", "directory_url": "https://agent.bot.goog/.well-known/http-message-signatures-directory", "kid_convention": "short_id", "purpose_field": null, "covers": [ "Google-Agent" ], "note": "Experimental. Google says only a subset of Google-Agent requests are signed, authenticated as https://agent.bot.goog, with the Signature-Agent header set to g=\"https://agent.bot.goog\" and the Signature and Signature-Input headers labeled g. Google says not every request is signed and recommends falling back to IP, reverse DNS and user-agent checks.", "sources": [ { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/web-bot-auth", "title": "Authenticate requests with Web Bot Auth (experimental)", "fetched_at": "2026-10-28T17:15:07.481Z" }, { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "title": "List of Google user-triggered fetchers", "fetched_at": "2026-10-28T17:15:05.416Z" }, { "url": "https://agent.bot.goog/.well-known/http-message-signatures-directory", "title": "Key directory", "fetched_at": "2026-10-28T17:15:01.850Z" } ] }, "ip_lists": [ { "agent": "common-crawlers", "url": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json", "format": "ipranges-json", "prefix_count": 317, "fetched_at": "2026-10-28T17:14:57.615Z", "sha256": "cfd0549c5b192e758662a1dc732d31a2449fcc9b9a0f4fe33c2b978ca240b35a", "prefixes": [ "34.22.85.0/27", "34.64.82.64/28", "34.65.242.112/28", "34.80.50.80/28", "34.88.194.0/28", "34.89.10.80/28", "34.89.198.80/28", "34.96.162.48/28", "34.100.182.96/28", "34.101.50.144/28", "34.118.66.0/28", "34.118.254.0/28", "34.126.178.96/28", "34.146.150.144/28", "34.147.110.144/28", "34.151.74.144/28", "34.152.50.64/28", "34.154.114.144/28", "34.155.98.32/28", "34.165.18.176/28", "34.175.160.64/28", "34.176.130.16/28", "35.247.243.240/28", "66.249.64.0/27", "66.249.64.32/27", "66.249.64.64/27", "66.249.64.96/27", "66.249.64.128/27", "66.249.64.160/27", "66.249.64.192/27", "66.249.64.224/27", "66.249.65.0/27", "66.249.65.32/27", "66.249.65.64/27", "66.249.65.96/27", "66.249.65.128/27", "66.249.65.160/27", "66.249.65.192/27", "66.249.65.224/27", "66.249.66.0/27", "66.249.66.32/27", "66.249.66.64/27", "66.249.66.96/27", "66.249.66.128/27", "66.249.66.160/27", "66.249.66.192/27", "66.249.66.224/27", "66.249.67.0/27", "66.249.67.32/27", "66.249.67.64/27", "66.249.68.0/27", "66.249.68.32/27", "66.249.68.64/27", "66.249.68.96/27", "66.249.68.128/27", "66.249.68.160/27", "66.249.68.192/27", "66.249.68.224/27", "66.249.69.0/27", "66.249.69.32/27", "66.249.69.64/27", "66.249.69.96/27", "66.249.69.128/27", "66.249.69.160/27", "66.249.69.192/27", "66.249.69.224/27", "66.249.70.0/27", "66.249.70.32/27", "66.249.70.64/27", "66.249.70.96/27", "66.249.70.128/27", "66.249.70.160/27", "66.249.70.192/27", "66.249.70.224/27", "66.249.71.0/27", "66.249.71.32/27", "66.249.71.64/27", "66.249.71.96/27", "66.249.71.128/27", "66.249.71.160/27", "66.249.71.192/27", "66.249.71.224/27", "66.249.72.0/27", "66.249.72.32/27", "66.249.72.64/27", "66.249.72.96/27", "66.249.72.128/27", "66.249.72.160/27", "66.249.72.192/27", "66.249.72.224/27", "66.249.73.0/27", "66.249.73.32/27", "66.249.73.64/27", "66.249.73.96/27", "66.249.73.128/27", "66.249.73.160/27", "66.249.73.192/27", "66.249.73.224/27", "66.249.74.0/27", "66.249.74.32/27", "66.249.74.64/27", "66.249.74.96/27", "66.249.74.128/27", "66.249.74.160/27", "66.249.74.192/27", "66.249.74.224/27", "66.249.75.0/27", "66.249.75.32/27", "66.249.75.64/27", "66.249.75.96/27", "66.249.75.128/27", "66.249.75.160/27", "66.249.75.192/27", "66.249.75.224/27", "66.249.76.0/27", "66.249.76.32/27", "66.249.76.64/27", "66.249.76.96/27", "66.249.76.128/27", "66.249.76.160/27", "66.249.76.192/27", "66.249.76.224/27", "66.249.77.0/27", "66.249.77.32/27", "66.249.77.64/27", "66.249.77.96/27", "66.249.77.128/27", "66.249.77.160/27", "66.249.77.192/27", "66.249.77.224/27", "66.249.78.0/27", "66.249.78.32/27", "66.249.78.64/27", "66.249.78.96/27", "66.249.78.128/27", "66.249.78.160/27", "66.249.78.192/27", "66.249.78.224/27", "66.249.79.0/27", "66.249.79.32/27", "66.249.79.64/27", "66.249.79.128/27", "66.249.79.160/27", "66.249.79.192/27", "66.249.79.224/27", "192.178.4.0/27", "192.178.4.32/27", "192.178.4.64/27", "192.178.4.96/27", "192.178.4.128/27", "192.178.4.160/27", "192.178.4.192/27", "192.178.4.224/27", "192.178.5.0/27", "192.178.6.0/27", "192.178.6.32/27", "192.178.6.64/27", "192.178.6.96/27", "192.178.6.128/27", "192.178.6.160/27", "192.178.6.192/27", "192.178.6.224/27", "192.178.7.0/27", "192.178.7.32/27", "192.178.7.64/27", "192.178.7.96/27", "192.178.7.128/27", "192.178.7.160/27", "192.178.7.192/27", "192.178.7.224/27", "2001:4860:4801:2::/64", "2001:4860:4801:c::/64", "2001:4860:4801:f::/64", "2001:4860:4801:10::/64", "2001:4860:4801:11::/64", "2001:4860:4801:12::/64", "2001:4860:4801:13::/64", "2001:4860:4801:14::/64", "2001:4860:4801:15::/64", "2001:4860:4801:16::/64", "2001:4860:4801:17::/64", "2001:4860:4801:18::/64", "2001:4860:4801:19::/64", "2001:4860:4801:1a::/64", "2001:4860:4801:1b::/64", "2001:4860:4801:1c::/64", "2001:4860:4801:1d::/64", "2001:4860:4801:1e::/64", "2001:4860:4801:1f::/64", "2001:4860:4801:20::/64", "2001:4860:4801:21::/64", "2001:4860:4801:22::/64", "2001:4860:4801:23::/64", "2001:4860:4801:24::/64", "2001:4860:4801:25::/64", "2001:4860:4801:26::/64", "2001:4860:4801:27::/64", "2001:4860:4801:28::/64", "2001:4860:4801:29::/64", "2001:4860:4801:2a::/64", "2001:4860:4801:2b::/64", "2001:4860:4801:2c::/64", "2001:4860:4801:2d::/64", "2001:4860:4801:2e::/64", "2001:4860:4801:2f::/64", "2001:4860:4801:30::/64", "2001:4860:4801:31::/64", "2001:4860:4801:32::/64", "2001:4860:4801:33::/64", "2001:4860:4801:34::/64", "2001:4860:4801:35::/64", "2001:4860:4801:36::/64", "2001:4860:4801:37::/64", "2001:4860:4801:38::/64", "2001:4860:4801:39::/64", "2001:4860:4801:3a::/64", "2001:4860:4801:3b::/64", "2001:4860:4801:3c::/64", "2001:4860:4801:3d::/64", "2001:4860:4801:3e::/64", "2001:4860:4801:3f::/64", "2001:4860:4801:40::/64", "2001:4860:4801:41::/64", "2001:4860:4801:42::/64", "2001:4860:4801:43::/64", "2001:4860:4801:44::/64", "2001:4860:4801:45::/64", "2001:4860:4801:46::/64", "2001:4860:4801:47::/64", "2001:4860:4801:48::/64", "2001:4860:4801:49::/64", "2001:4860:4801:4a::/64", "2001:4860:4801:4b::/64", "2001:4860:4801:4c::/64", "2001:4860:4801:4d::/64", "2001:4860:4801:4e::/64", "2001:4860:4801:50::/64", "2001:4860:4801:51::/64", "2001:4860:4801:52::/64", "2001:4860:4801:53::/64", "2001:4860:4801:54::/64", "2001:4860:4801:55::/64", "2001:4860:4801:56::/64", "2001:4860:4801:57::/64", "2001:4860:4801:58::/64", "2001:4860:4801:59::/64", "2001:4860:4801:60::/64", "2001:4860:4801:61::/64", "2001:4860:4801:62::/64", "2001:4860:4801:63::/64", "2001:4860:4801:64::/64", "2001:4860:4801:65::/64", "2001:4860:4801:66::/64", "2001:4860:4801:67::/64", "2001:4860:4801:68::/64", "2001:4860:4801:69::/64", "2001:4860:4801:6a::/64", "2001:4860:4801:6b::/64", "2001:4860:4801:6c::/64", "2001:4860:4801:6d::/64", "2001:4860:4801:6e::/64", "2001:4860:4801:6f::/64", "2001:4860:4801:70::/64", "2001:4860:4801:71::/64", "2001:4860:4801:72::/64", "2001:4860:4801:73::/64", "2001:4860:4801:74::/64", "2001:4860:4801:75::/64", "2001:4860:4801:76::/64", "2001:4860:4801:77::/64", "2001:4860:4801:78::/64", "2001:4860:4801:79::/64", "2001:4860:4801:7a::/64", "2001:4860:4801:7b::/64", "2001:4860:4801:7c::/64", "2001:4860:4801:7d::/64", "2001:4860:4801:7e::/64", "2001:4860:4801:7f::/64", "2001:4860:4801:80::/64", "2001:4860:4801:81::/64", "2001:4860:4801:82::/64", "2001:4860:4801:83::/64", "2001:4860:4801:84::/64", "2001:4860:4801:85::/64", "2001:4860:4801:86::/64", "2001:4860:4801:87::/64", "2001:4860:4801:88::/64", "2001:4860:4801:90::/64", "2001:4860:4801:91::/64", "2001:4860:4801:92::/64", "2001:4860:4801:93::/64", "2001:4860:4801:94::/64", "2001:4860:4801:95::/64", "2001:4860:4801:96::/64", "2001:4860:4801:97::/64", "2001:4860:4801:a0::/64", "2001:4860:4801:a1::/64", "2001:4860:4801:a2::/64", "2001:4860:4801:a3::/64", "2001:4860:4801:a4::/64", "2001:4860:4801:a5::/64", "2001:4860:4801:a6::/64", "2001:4860:4801:a7::/64", "2001:4860:4801:a8::/64", "2001:4860:4801:a9::/64", "2001:4860:4801:aa::/64", "2001:4860:4801:ab::/64", "2001:4860:4801:ac::/64", "2001:4860:4801:ad::/64", "2001:4860:4801:ae::/64", "2001:4860:4801:b0::/64", "2001:4860:4801:b1::/64", "2001:4860:4801:b2::/64", "2001:4860:4801:b3::/64", "2001:4860:4801:b4::/64", "2001:4860:4801:b5::/64", "2001:4860:4801:b6::/64" ], "note": null, "sources": [ { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "title": "List of Google's common crawlers", "fetched_at": "2026-10-28T17:15:04.095Z" } ] }, { "agent": "special-crawlers", "url": "https://developers.google.com/static/crawling/ipranges/special-crawlers.json", "format": "ipranges-json", "prefix_count": 272, "fetched_at": "2026-10-28T17:14:58.834Z", "sha256": "2ba077149e5490cbcd8011e8b08976311dd397eac8a183ab3a7d396e371a49c0", "prefixes": [ "66.249.87.0/27", "66.249.87.32/27", "66.249.87.64/27", "66.249.87.96/27", "66.249.87.128/27", "66.249.87.160/27", "66.249.87.192/27", "66.249.87.224/27", "66.249.89.0/27", "66.249.89.32/27", "66.249.89.64/27", "66.249.89.96/27", "66.249.89.128/27", "66.249.89.160/27", "66.249.89.192/27", "66.249.89.224/27", "66.249.90.0/27", "66.249.90.32/27", "66.249.90.64/27", "66.249.90.96/27", "66.249.90.128/27", "66.249.90.160/27", "66.249.90.192/27", "66.249.90.224/27", "66.249.91.0/27", "66.249.91.32/27", "66.249.91.64/27", "66.249.91.96/27", "66.249.91.128/27", "66.249.91.160/27", "66.249.91.192/27", "66.249.91.224/27", "66.249.92.0/27", "66.249.92.32/27", "66.249.92.64/27", "66.249.92.96/27", "66.249.92.128/27", "66.249.92.160/27", "66.249.92.192/27", "66.249.92.224/27", "72.14.199.0/27", "72.14.199.32/27", "72.14.199.64/27", "72.14.199.96/27", "72.14.199.128/27", "72.14.199.160/27", "72.14.199.192/27", "72.14.199.224/27", "74.125.148.0/27", "74.125.148.32/27", "74.125.148.64/27", "74.125.148.96/27", "74.125.148.128/27", "74.125.148.160/27", "74.125.148.192/27", "74.125.148.224/27", "74.125.149.0/27", "74.125.149.32/27", "74.125.149.64/27", "74.125.149.96/27", "74.125.149.128/27", "74.125.149.160/27", "74.125.149.192/27", "74.125.149.224/27", "74.125.150.0/27", "74.125.150.32/27", "74.125.150.64/27", "74.125.150.96/27", "74.125.150.128/27", "74.125.150.160/27", "74.125.150.192/27", "74.125.151.0/27", "74.125.151.32/27", "74.125.151.64/27", "74.125.151.96/27", "74.125.151.128/27", "74.125.151.160/27", "74.125.151.192/27", "74.125.151.224/27", "74.125.216.0/27", "74.125.216.32/27", "74.125.216.64/27", "74.125.216.96/27", "74.125.216.128/27", "74.125.216.160/27", "74.125.216.192/27", "74.125.216.224/27", "74.125.217.0/27", "74.125.217.32/27", "74.125.217.64/27", "74.125.217.96/27", "74.125.217.128/27", "74.125.217.160/27", "74.125.217.192/27", "74.125.217.224/27", "74.125.218.0/27", "74.125.218.32/27", "74.125.218.64/27", "74.125.218.96/27", "74.125.218.128/27", "74.125.218.160/27", "74.125.218.192/27", "74.125.218.224/27", "74.125.219.0/27", "74.125.219.32/27", "74.125.219.64/27", "74.125.219.96/27", "74.125.219.128/27", "74.125.219.160/27", "74.125.219.192/27", "74.125.219.224/27", "108.177.2.0/27", "108.177.2.32/27", "108.177.2.64/27", "108.177.2.96/27", "108.177.2.128/27", "108.177.2.160/27", "108.177.2.192/27", "108.177.2.224/27", "192.178.16.0/27", "192.178.16.32/27", "192.178.16.64/27", "192.178.16.96/27", "192.178.16.128/27", "192.178.16.160/27", "192.178.16.192/27", "192.178.16.224/27", "192.178.17.0/27", "209.85.238.0/27", "209.85.238.32/27", "209.85.238.64/27", "209.85.238.96/27", "209.85.238.128/27", "209.85.238.160/27", "209.85.238.192/27", "209.85.238.224/27", "2001:4860:4801:2008::/64", "2001:4860:4801:200c::/64", "2001:4860:4801:200d::/64", "2001:4860:4801:2010::/64", "2001:4860:4801:2011::/64", "2001:4860:4801:2012::/64", "2001:4860:4801:2013::/64", "2001:4860:4801:2014::/64", "2001:4860:4801:2015::/64", "2001:4860:4801:2016::/64", "2001:4860:4801:2017::/64", "2001:4860:4801:2018::/64", "2001:4860:4801:2019::/64", "2001:4860:4801:201a::/64", "2001:4860:4801:201b::/64", "2001:4860:4801:201c::/64", "2001:4860:4801:201d::/64", "2001:4860:4801:201e::/64", "2001:4860:4801:201f::/64", "2001:4860:4801:2020::/64", "2001:4860:4801:2021::/64", "2001:4860:4801:2022::/64", "2001:4860:4801:2023::/64", "2001:4860:4801:2024::/64", "2001:4860:4801:2025::/64", "2001:4860:4801:2026::/64", "2001:4860:4801:2027::/64", "2001:4860:4801:2028::/64", "2001:4860:4801:2029::/64", "2001:4860:4801:202a::/64", "2001:4860:4801:202b::/64", "2001:4860:4801:202c::/64", "2001:4860:4801:202d::/64", "2001:4860:4801:202e::/64", "2001:4860:4801:202f::/64", "2001:4860:4801:2030::/64", "2001:4860:4801:2031::/64", "2001:4860:4801:2032::/64", "2001:4860:4801:2033::/64", "2001:4860:4801:2034::/64", "2001:4860:4801:2035::/64", "2001:4860:4801:2036::/64", "2001:4860:4801:2037::/64", "2001:4860:4801:2038::/64", "2001:4860:4801:2039::/64", "2001:4860:4801:203a::/64", "2001:4860:4801:203b::/64", "2001:4860:4801:203c::/64", "2001:4860:4801:203d::/64", "2001:4860:4801:203e::/64", "2001:4860:4801:203f::/64", "2001:4860:4801:2040::/64", "2001:4860:4801:2041::/64", "2001:4860:4801:2042::/64", "2001:4860:4801:2043::/64", "2001:4860:4801:2044::/64", "2001:4860:4801:2045::/64", "2001:4860:4801:2046::/64", "2001:4860:4801:2047::/64", "2001:4860:4801:2048::/64", "2001:4860:4801:2049::/64", "2001:4860:4801:204a::/64", "2001:4860:4801:204b::/64", "2001:4860:4801:204c::/64", "2001:4860:4801:204d::/64", "2001:4860:4801:204e::/64", "2001:4860:4801:2050::/64", "2001:4860:4801:2051::/64", "2001:4860:4801:2052::/64", "2001:4860:4801:2053::/64", "2001:4860:4801:2054::/64", "2001:4860:4801:2055::/64", "2001:4860:4801:2056::/64", "2001:4860:4801:2060::/64", "2001:4860:4801:2061::/64", "2001:4860:4801:2062::/64", "2001:4860:4801:2063::/64", "2001:4860:4801:2064::/64", "2001:4860:4801:2065::/64", "2001:4860:4801:2066::/64", "2001:4860:4801:2067::/64", "2001:4860:4801:2068::/64", "2001:4860:4801:2069::/64", "2001:4860:4801:206a::/64", "2001:4860:4801:206b::/64", "2001:4860:4801:206c::/64", "2001:4860:4801:206d::/64", "2001:4860:4801:206e::/64", "2001:4860:4801:206f::/64", "2001:4860:4801:2070::/64", "2001:4860:4801:2071::/64", "2001:4860:4801:2072::/64", "2001:4860:4801:2073::/64", "2001:4860:4801:2074::/64", "2001:4860:4801:2075::/64", "2001:4860:4801:2076::/64", "2001:4860:4801:2077::/64", "2001:4860:4801:2078::/64", "2001:4860:4801:2079::/64", "2001:4860:4801:207a::/64", "2001:4860:4801:207b::/64", "2001:4860:4801:207c::/64", "2001:4860:4801:207d::/64", "2001:4860:4801:207e::/64", "2001:4860:4801:207f::/64", "2001:4860:4801:2080::/64", "2001:4860:4801:2081::/64", "2001:4860:4801:2082::/64", "2001:4860:4801:2083::/64", "2001:4860:4801:2084::/64", "2001:4860:4801:2085::/64", "2001:4860:4801:2086::/64", "2001:4860:4801:2087::/64", "2001:4860:4801:2088::/64", "2001:4860:4801:2090::/64", "2001:4860:4801:2091::/64", "2001:4860:4801:2092::/64", "2001:4860:4801:2093::/64", "2001:4860:4801:2094::/64", "2001:4860:4801:2095::/64", "2001:4860:4801:2096::/64", "2001:4860:4801:2097::/64", "2001:4860:4801:20a0::/64", "2001:4860:4801:20a1::/64", "2001:4860:4801:20a2::/64", "2001:4860:4801:20a3::/64", "2001:4860:4801:20a4::/64", "2001:4860:4801:20a5::/64", "2001:4860:4801:20a6::/64", "2001:4860:4801:20b0::/64", "2001:4860:4801:20b1::/64", "2001:4860:4801:20b2::/64", "2001:4860:4801:20b3::/64", "2001:4860:4801:20b4::/64", "2001:4860:4801:20b5::/64", "2001:4860:4801:20b6::/64" ], "note": null, "sources": [ { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-special-case-crawlers", "title": "List of Google's special-case crawlers", "fetched_at": "2026-10-28T17:15:04.522Z" } ] }, { "agent": "user-triggered-fetchers-google", "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers-google.json", "format": "ipranges-json", "prefix_count": 504, "fetched_at": "2026-10-28T17:14:59.800Z", "sha256": "c858d37a985f3b32124341b0fb1980bd9077e0b16383c8ce4b5e4048da1e2498", "prefixes": [ "64.233.172.0/27", "64.233.172.32/27", "64.233.172.64/27", "64.233.172.96/27", "64.233.172.128/27", "64.233.172.160/27", "64.233.172.192/27", "64.233.172.224/27", "64.233.173.0/27", "64.233.173.32/27", "64.233.173.64/27", "64.233.173.96/27", "64.233.173.128/27", "64.233.173.160/27", "64.233.173.192/27", "64.233.173.224/27", "66.102.6.0/27", "66.102.6.32/27", "66.102.6.64/27", "66.102.6.96/27", "66.102.6.128/27", "66.102.6.160/27", "66.102.6.192/27", "66.102.6.224/27", "66.102.7.0/27", "66.102.7.32/27", "66.102.7.64/27", "66.102.7.96/27", "66.102.7.128/27", "66.102.7.160/27", "66.102.7.192/27", "66.102.7.224/27", "66.102.8.0/27", "66.102.8.32/27", "66.102.8.64/27", "66.102.8.96/27", "66.102.8.128/27", "66.102.8.160/27", "66.102.8.192/27", "66.102.8.224/27", "66.102.9.0/27", "66.102.9.32/27", "66.102.9.64/27", "66.102.9.96/27", "66.102.9.128/27", "66.102.9.160/27", "66.102.9.192/27", "66.102.9.224/27", "66.249.80.0/27", "66.249.80.32/27", "66.249.80.64/27", "66.249.80.96/27", "66.249.80.128/27", "66.249.80.160/27", "66.249.80.192/27", "66.249.80.224/27", "66.249.81.0/27", "66.249.81.32/27", "66.249.81.64/27", "66.249.81.96/27", "66.249.81.128/27", "66.249.81.160/27", "66.249.81.192/27", "66.249.81.224/27", "66.249.82.0/27", "66.249.82.32/27", "66.249.82.64/27", "66.249.82.96/27", "66.249.82.128/27", "66.249.82.160/27", "66.249.82.192/27", "66.249.82.224/27", "66.249.83.0/27", "66.249.83.32/27", "66.249.83.64/27", "66.249.83.96/27", "66.249.83.128/27", "66.249.83.160/27", "66.249.83.192/27", "66.249.83.224/27", "66.249.84.0/27", "66.249.84.32/27", "66.249.84.64/27", "66.249.84.96/27", "66.249.84.128/27", "66.249.84.160/27", "66.249.84.192/27", "66.249.84.224/27", "66.249.85.0/27", "66.249.85.32/27", "66.249.85.64/27", "66.249.85.96/27", "66.249.85.128/27", "66.249.85.160/27", "66.249.88.0/27", "66.249.88.32/27", "66.249.88.64/27", "66.249.88.96/27", "66.249.88.128/27", "66.249.88.160/27", "66.249.88.192/27", "66.249.88.224/27", "66.249.93.0/27", "66.249.93.32/27", "66.249.93.64/27", "66.249.93.96/27", "66.249.93.128/27", "66.249.93.160/27", "66.249.93.192/27", "66.249.93.224/27", "74.125.208.0/27", "74.125.208.32/27", "74.125.208.64/27", "74.125.208.96/27", "74.125.208.128/27", "74.125.208.160/27", "74.125.208.192/27", "74.125.208.224/27", "74.125.209.0/27", "74.125.209.32/27", "74.125.209.64/27", "74.125.209.96/27", "74.125.209.128/27", "74.125.209.160/27", "74.125.209.192/27", "74.125.210.0/27", "74.125.210.32/27", "74.125.210.64/27", "74.125.210.96/27", "74.125.210.128/27", "74.125.210.160/27", "74.125.210.192/27", "74.125.210.224/27", "74.125.211.0/27", "74.125.211.32/27", "74.125.211.64/27", "74.125.211.96/27", "74.125.212.0/27", "74.125.212.32/27", "74.125.212.64/27", "74.125.212.96/27", "74.125.212.128/27", "74.125.212.160/27", "74.125.212.192/27", "74.125.212.224/27", "74.125.213.0/27", "74.125.213.32/27", "74.125.213.64/27", "74.125.213.96/27", "74.125.214.0/27", "74.125.214.32/27", "74.125.214.64/27", "74.125.214.96/27", "74.125.214.128/27", "74.125.214.160/27", "74.125.214.192/27", "74.125.214.224/27", "74.125.215.0/27", "74.125.215.32/27", "74.125.215.64/27", "74.125.215.96/27", "74.125.215.128/27", "74.125.215.160/27", "74.125.215.192/27", "74.125.215.224/27", "142.250.32.0/27", "142.250.32.32/27", "142.250.32.64/27", "142.250.32.96/27", "142.250.32.128/27", "142.250.32.160/27", "142.250.32.192/27", "142.250.32.224/27", "142.250.33.0/27", "142.250.33.32/27", "142.250.33.64/27", "142.250.33.96/27", "142.250.33.128/27", "172.253.180.0/27", "172.253.180.32/27", "172.253.180.64/27", "172.253.181.0/27", "172.253.181.32/27", "172.253.181.64/27", "172.253.181.96/27", "172.253.181.128/27", "172.253.181.160/27", "172.253.181.192/27", "172.253.181.224/27", "172.253.182.0/27", "172.253.182.32/27", "172.253.182.64/27", "172.253.182.96/27", "172.253.182.128/27", "172.253.183.0/27", "172.253.183.32/27", "172.253.183.64/27", "172.253.183.96/27", "172.253.183.128/27", "172.253.183.160/27", "172.253.183.192/27", "172.253.183.224/27", "192.178.8.0/27", "192.178.8.32/27", "192.178.8.64/27", "192.178.8.96/27", "192.178.8.128/27", "192.178.9.0/27", "192.178.9.32/27", "192.178.9.64/27", "192.178.9.96/27", "192.178.9.128/27", "192.178.9.160/27", "192.178.9.192/27", "192.178.9.224/27", "192.178.10.0/27", "192.178.10.32/27", "192.178.10.64/27", "192.178.10.96/27", "192.178.10.128/27", "192.178.11.0/27", "192.178.11.32/27", "192.178.11.64/27", "192.178.11.96/27", "192.178.11.128/27", "192.178.11.160/27", "192.178.11.192/27", "192.178.11.224/27", "192.178.12.0/27", "192.178.12.32/27", "192.178.12.64/27", "192.178.12.96/27", "192.178.12.128/27", "192.178.12.160/27", "192.178.12.192/27", "192.178.13.0/27", "192.178.13.32/27", "192.178.13.64/27", "192.178.13.96/27", "192.178.13.128/27", "192.178.13.160/27", "192.178.13.192/27", "192.178.13.224/27", "192.178.14.0/27", "192.178.15.0/27", "192.178.15.32/27", "192.178.15.64/27", "192.178.15.96/27", "192.178.15.128/27", "192.178.15.160/27", "192.178.15.192/27", "192.178.15.224/27", "2001:4860:4801:4004::/64", "2001:4860:4801:4008::/64", "2001:4860:4801:4009::/64", "2001:4860:4801:400f::/64", "2001:4860:4801:4010::/64", "2001:4860:4801:4015::/64", "2001:4860:4801:4016::/64", "2001:4860:4801:4017::/64", "2001:4860:4801:4018::/64", "2001:4860:4801:401a::/64", "2001:4860:4801:401b::/64", "2001:4860:4801:401c::/64", "2001:4860:4801:4020::/64", "2001:4860:4801:4021::/64", "2001:4860:4801:4022::/64", "2001:4860:4801:4023::/64", "2001:4860:4801:4024::/64", "2001:4860:4801:4025::/64", "2001:4860:4801:4026::/64", "2001:4860:4801:4027::/64", "2001:4860:4801:4028::/64", "2001:4860:4801:4029::/64", "2001:4860:4801:402a::/64", "2001:4860:4801:402b::/64", "2001:4860:4801:4030::/64", "2001:4860:4801:4031::/64", "2001:4860:4801:4032::/64", "2001:4860:4801:4033::/64", "2001:4860:4801:4034::/64", "2001:4860:4801:4035::/64", "2001:4860:4801:4036::/64", "2001:4860:4801:4037::/64", "2001:4860:4801:4038::/64", "2001:4860:4801:4039::/64", "2001:4860:4801:403a::/64", "2001:4860:4801:403b::/64", "2001:4860:4801:403c::/64", "2001:4860:4801:403d::/64", "2001:4860:4801:403e::/64", "2001:4860:4801:403f::/64", "2001:4860:4801:4040::/64", "2001:4860:4801:4041::/64", "2001:4860:4801:4042::/64", "2001:4860:4801:4043::/64", "2001:4860:4801:4044::/64", "2001:4860:4801:4045::/64", "2001:4860:4801:4046::/64", "2001:4860:4801:4047::/64", "2001:4860:4801:4048::/64", "2001:4860:4801:4049::/64", "2001:4860:4801:404a::/64", "2001:4860:4801:404b::/64", "2001:4860:4801:404c::/64", "2001:4860:4801:404d::/64", "2001:4860:4801:404e::/64", "2001:4860:4801:404f::/64", "2001:4860:4801:4050::/64", "2001:4860:4801:4051::/64", "2001:4860:4801:4052::/64", "2001:4860:4801:4053::/64", "2001:4860:4801:4054::/64", "2001:4860:4801:4055::/64", "2001:4860:4801:4056::/64", "2001:4860:4801:4057::/64", "2001:4860:4801:4058::/64", "2001:4860:4801:4059::/64", "2001:4860:4801:405a::/64", "2001:4860:4801:405b::/64", "2001:4860:4801:405c::/64", "2001:4860:4801:405d::/64", "2001:4860:4801:405e::/64", "2001:4860:4801:405f::/64", "2001:4860:4801:4060::/64", "2001:4860:4801:4061::/64", "2001:4860:4801:4062::/64", "2001:4860:4801:4063::/64", "2001:4860:4801:4064::/64", "2001:4860:4801:4065::/64", "2001:4860:4801:4066::/64", "2001:4860:4801:4067::/64", "2001:4860:4801:4068::/64", "2001:4860:4801:4069::/64", "2001:4860:4801:406a::/64", "2001:4860:4801:406b::/64", "2001:4860:4801:406c::/64", "2001:4860:4801:406d::/64", "2001:4860:4801:406e::/64", "2001:4860:4801:4070::/64", "2001:4860:4801:4071::/64", "2001:4860:4801:4072::/64", "2001:4860:4801:4073::/64", "2001:4860:4801:4074::/64", "2001:4860:4801:4075::/64", "2001:4860:4801:4076::/64", "2001:4860:4801:4077::/64", "2001:4860:4801:4078::/64", "2001:4860:4801:4079::/64", "2001:4860:4801:407a::/64", "2001:4860:4801:407b::/64", "2001:4860:4801:407c::/64", "2001:4860:4801:407d::/64", "2001:4860:4801:407e::/64", "2001:4860:4801:407f::/64", "2001:4860:4801:4080::/64", "2001:4860:4801:4081::/64", "2001:4860:4801:4082::/64", "2001:4860:4801:4083::/64", "2001:4860:4801:4084::/64", "2001:4860:4801:4085::/64", "2001:4860:4801:4086::/64", "2001:4860:4801:4087::/64", "2001:4860:4801:4088::/64", "2001:4860:4801:4089::/64", "2001:4860:4801:408a::/64", "2001:4860:4801:408b::/64", "2001:4860:4801:4090::/64", "2001:4860:4801:4091::/64", "2001:4860:4801:4092::/64", "2001:4860:4801:4093::/64", "2001:4860:4801:4094::/64", "2001:4860:4801:4095::/64", "2001:4860:4801:4096::/64", "2001:4860:4801:40a0::/64", "2001:4860:4801:40a1::/64", "2001:4860:4801:40a2::/64", "2001:4860:4801:40a3::/64", "2001:4860:4801:40a4::/64", "2001:4860:4801:40b0::/64", "2001:4860:4801:40b1::/64", "2001:4860:4801:40b2::/64", "2001:4860:4801:40b3::/64", "2001:4860:4801:40b4::/64", "2001:4860:4801:40b5::/64", "2001:4860:4801:40b6::/64", "2001:4860:4801:40b7::/64", "2001:4860:4801:40b8::/64", "2001:4860:4801:40b9::/64", "2001:4860:4801:40ba::/64", "2001:4860:4801:40c0::/64", "2001:4860:4801:40c1::/64", "2001:4860:4801:40c2::/64", "2001:4860:4801:40c3::/64", "2001:4860:4801:40c4::/64", "2001:4860:4801:40c5::/64", "2001:4860:4801:40c6::/64", "2001:4860:4801:40d0::/64", "2001:4860:4801:40d1::/64", "2001:4860:4801:40d2::/64", "2001:4860:4801:40d3::/64", "2001:4860:4801:40d4::/64", "2001:4860:4801:40d5::/64", "2001:4860:4801:40d6::/64", "2001:4860:4801:40d7::/64", "2001:4860:4801:40d8::/64", "2001:4860:4801:40d9::/64", "2001:4860:4801:40da::/64", "2001:4860:4801:40db::/64", "2001:4860:4801:40dc::/64", "2001:4860:4801:40dd::/64", "2001:4860:4801:40de::/64", "2001:4860:4801:40df::/64", "2001:4860:4801:40e0::/64", "2001:4860:4801:40e1::/64", "2001:4860:4801:40e2::/64", "2001:4860:4801:40e3::/64", "2001:4860:4801:40f0::/64", "2001:4860:4801:40f1::/64", "2001:4860:4801:40f2::/64", "2001:4860:4801:40f3::/64", "2001:4860:4801:40f4::/64", "2001:4860:4801:40f5::/64", "2001:4860:4801:40f6::/64", "2001:4860:4801:40f7::/64", "2001:4860:4801:40f8::/64", "2001:4860:4801:40f9::/64", "2001:4860:4801:40fa::/64", "2001:4860:4801:40fb::/64", "2001:4860:4801:40fc::/64", "2001:4860:4801:40fd::/64", "2001:4860:4801:40fe::/64", "2001:4860:4801:40ff::/64", "2001:4860:4801:4100::/64", "2001:4860:4801:4101::/64", "2001:4860:4801:4102::/64", "2001:4860:4801:4103::/64", "2001:4860:4801:4104::/64", "2001:4860:4801:4105::/64", "2001:4860:4801:4106::/64", "2001:4860:4801:4107::/64", "2001:4860:4801:4110::/64", "2001:4860:4801:4111::/64", "2001:4860:4801:4112::/64", "2001:4860:4801:4113::/64", "2001:4860:4801:4114::/64", "2001:4860:4801:4120::/64", "2001:4860:4801:4121::/64", "2001:4860:4801:4122::/64", "2001:4860:4801:4123::/64", "2001:4860:4801:4124::/64", "2001:4860:4801:4125::/64", "2001:4860:4801:4126::/64", "2001:4860:4801:4127::/64", "2001:4860:4801:4128::/64", "2001:4860:4801:4129::/64", "2001:4860:4801:412a::/64", "2001:4860:4801:412b::/64", "2001:4860:4801:412c::/64", "2001:4860:4801:412d::/64", "2001:4860:4801:412e::/64", "2001:4860:4801:412f::/64", "2001:4860:4801:4130::/64", "2001:4860:4801:4131::/64", "2001:4860:4801:4132::/64", "2001:4860:4801:4133::/64", "2001:4860:4801:4134::/64", "2001:4860:4801:4135::/64", "2001:4860:4801:4136::/64", "2001:4860:4801:4140::/64", "2001:4860:4801:4141::/64", "2001:4860:4801:4142::/64", "2001:4860:4801:4143::/64", "2001:4860:4801:4144::/64", "2001:4860:4801:4145::/64", "2001:4860:4801:4146::/64", "2001:4860:4801:4147::/64", "2001:4860:4801:4150::/64", "2001:4860:4801:4151::/64", "2001:4860:4801:4152::/64", "2001:4860:4801:4153::/64", "2001:4860:4801:4154::/64", "2001:4860:4801:4155::/64", "2001:4860:4801:4156::/64", "2001:4860:4801:4157::/64", "2001:4860:4801:4158::/64", "2001:4860:4801:4159::/64", "2001:4860:4801:415a::/64", "2001:4860:4801:415b::/64", "2001:4860:4801:415c::/64", "2001:4860:4801:415d::/64", "2001:4860:4801:415e::/64", "2001:4860:4801:4160::/64", "2001:4860:4801:4161::/64", "2001:4860:4801:4162::/64", "2001:4860:4801:4163::/64", "2001:4860:4801:4164::/64", "2001:4860:4801:4165::/64", "2001:4860:4801:4166::/64", "2001:4860:4801:4167::/64", "2001:4860:4801:4168::/64", "2001:4860:4801:4169::/64", "2001:4860:4801:416a::/64", "2001:4860:4801:416b::/64" ], "note": null, "sources": [ { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "title": "List of Google user-triggered fetchers", "fetched_at": "2026-10-28T17:15:05.416Z" } ] }, { "agent": "user-triggered-agents", "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-agents.json", "format": "ipranges-json", "prefix_count": 20, "fetched_at": "2026-10-28T17:15:00.933Z", "sha256": "6818dbea2b29b4b6464224b2d00625da8b4b24c2864989b5c963dde4cf40ff29", "prefixes": [ "74.125.232.0/28", "74.125.232.16/28", "74.125.232.32/28", "74.125.232.48/28", "74.125.232.64/28", "74.125.232.80/28", "74.125.232.96/28", "74.125.232.112/28", "136.121.16.0/24", "136.121.24.0/21", "136.121.40.0/21", "136.122.0.0/16", "2001:4860:c::/124", "2001:4860:c::10/124", "2001:4860:c::20/124", "2001:4860:c::30/124", "2001:4860:c::40/124", "2001:4860:c::50/124", "2001:4860:c::60/124", "2001:4860:c::70/124" ], "note": null, "sources": [ { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "title": "List of Google user-triggered fetchers", "fetched_at": "2026-10-28T17:15:05.416Z" } ] } ], "reverse_dns": { "suffixes": [ "googlebot.com", "google.com" ], "forward_confirm": true, "note": "googleusercontent.com is left out on purpose: Google uses it for user-triggered fetchers run by Google Cloud customers, so it does not identify a Google crawler. The matching customer IP list (user-triggered-fetchers.json) is left out for the same reason.", "sources": [ { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/verify-google-requests", "title": "Verify requests from Google crawlers and fetchers", "fetched_at": "2026-10-28T17:15:06.902Z" } ] }, "asn": [] }, "agents": [ { "name": "AdsBot", "coverage": "researched", "user_agent_tokens": [ "AdsBot-Google" ], "aliases": [], "purpose": "ads", "purpose_quote": "Crawling preferences addressed to the AdsBot-Google user agent affect Google Ads' ability to check web page ad quality.", "search_engine": false, "robots_token": "AdsBot-Google", "robots_policy": "may_ignore", "robots_quote": "The special-case crawlers may ignore robots.txt rules and so they operate from a different IP range than the common crawlers.", "ip_list_agent": "special-crawlers", "signs": false, "note": "Special-case crawler. Google says it ignores the global user agent (*); it obeys a group addressed to AdsBot-Google.", "sources": [ { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-special-case-crawlers", "title": "List of Google's special-case crawlers", "fetched_at": "2026-10-28T17:15:04.522Z" } ] }, { "name": "AdsBot Mobile Web", "coverage": "researched", "user_agent_tokens": [ "AdsBot-Google-Mobile" ], "aliases": [], "purpose": "ads", "purpose_quote": "Crawling preferences addressed to the AdsBot-Google-Mobile user agent affect Google Ads' ability to check web page ad quality.", "search_engine": false, "robots_token": "AdsBot-Google-Mobile", "robots_policy": "may_ignore", "robots_quote": "The special-case crawlers may ignore robots.txt rules and so they operate from a different IP range than the common crawlers.", "ip_list_agent": "special-crawlers", "signs": false, "note": "Special-case crawler. Google says it ignores the global user agent (*); it obeys a group addressed to AdsBot-Google-Mobile. An older iPhone variant with the same token is listed by Google as retired.", "sources": [ { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-special-case-crawlers", "title": "List of Google's special-case crawlers", "fetched_at": "2026-10-28T17:15:04.522Z" } ] }, { "name": "APIs-Google", "coverage": "researched", "user_agent_tokens": [ "APIs-Google" ], "aliases": [], "purpose": "other", "purpose_quote": "APIs-Google is the user agent used by Google APIs to deliver push notification messages.", "search_engine": false, "robots_token": "APIs-Google", "robots_policy": "may_ignore", "robots_quote": "The special-case crawlers may ignore robots.txt rules and so they operate from a different IP range than the common crawlers.", "ip_list_agent": "special-crawlers", "signs": false, "note": "Special-case crawler. Google says it ignores the global user agent (*) and does not follow rules for the Googlebot user agent; it obeys a group addressed to APIs-Google. The APIs-Google page says a reverse DNS lookup shows the googlebot.com or google.com domain.", "sources": [ { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/apis-user-agent", "title": "APIs-Google user agent", "fetched_at": "2026-10-28T17:15:08.643Z" }, { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-special-case-crawlers", "title": "List of Google's special-case crawlers", "fetched_at": "2026-10-28T17:15:04.522Z" } ] }, { "name": "Chrome Web Store", "coverage": "researched", "user_agent_tokens": [ "Google-CWS" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": "The Chrome Web Store fetcher requests URLs that developers provide in the metadata of their Chrome extensions and themes.", "search_engine": false, "robots_token": null, "robots_policy": "may_ignore", "robots_quote": "Because the fetch was requested by a user, these fetchers generally ignore robots.txt rules.", "ip_list_agent": "user-triggered-fetchers-google", "signs": false, "note": "User-triggered fetcher. Google does not map each fetcher to an IP file; it says fetchers controlled by Google originate from user-triggered-fetchers-google.json and resolve to a google.com hostname, so this Google product fetcher is linked to that file.", "sources": [ { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "title": "List of Google user-triggered fetchers", "fetched_at": "2026-10-28T17:15:05.416Z" } ] }, { "name": "Feedfetcher", "coverage": "researched", "user_agent_tokens": [ "FeedFetcher-Google" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": "Feedfetcher is used for crawling RSS or Atom feeds for Google News and WebSub.", "search_engine": false, "robots_token": null, "robots_policy": "may_ignore", "robots_quote": "Feedfetcher behaves as a direct agent of the human user, not as a robot, so it ignores robots.txt entries.", "ip_list_agent": "user-triggered-fetchers-google", "signs": false, "note": "User-triggered fetcher. The Feedfetcher page says its IP addresses are in user-triggered-fetchers-google.json. Google suggests serving a 404, 410 or other error status to the Feedfetcher-Google user agent to stop it.", "sources": [ { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "title": "List of Google user-triggered fetchers", "fetched_at": "2026-10-28T17:15:05.416Z" }, { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/feedfetcher", "title": "Feedfetcher", "fetched_at": "2026-10-28T17:15:09.460Z" } ] }, { "name": "Gemini Notebook", "coverage": "researched", "user_agent_tokens": [ "Google-GeminiNotebook" ], "aliases": [ "Google-NotebookLM" ], "purpose": "user_fetch", "purpose_quote": "The Gemini Notebook fetcher requests individual URLs that Gemini Notebook users have provided as sources for their projects.", "search_engine": false, "robots_token": null, "robots_policy": "may_ignore", "robots_quote": "Because the fetch was requested by a user, these fetchers generally ignore robots.txt rules.", "ip_list_agent": "user-triggered-fetchers-google", "signs": false, "note": "User-triggered fetcher. Google lists Google-NotebookLM as its former agent, supported until August 2026. IP file inferred from Google's statement that fetchers controlled by Google use user-triggered-fetchers-google.json.", "sources": [ { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "title": "List of Google user-triggered fetchers", "fetched_at": "2026-10-28T17:15:05.416Z" } ] }, { "name": "Google Messages", "coverage": "researched", "user_agent_tokens": [ "GoogleMessages" ], "aliases": [], "purpose": "preview", "purpose_quote": "Google Messages fetcher is used to generate link previews for URLs sent in chat messages.", "search_engine": false, "robots_token": null, "robots_policy": "may_ignore", "robots_quote": "Because the fetch was requested by a user, these fetchers generally ignore robots.txt rules.", "ip_list_agent": "user-triggered-fetchers-google", "signs": false, "note": "User-triggered fetcher. IP file inferred from Google's statement that fetchers controlled by Google use user-triggered-fetchers-google.json.", "sources": [ { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "title": "List of Google user-triggered fetchers", "fetched_at": "2026-10-28T17:15:05.416Z" } ] }, { "name": "Google Pinpoint", "coverage": "researched", "user_agent_tokens": [ "Google-Pinpoint" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": "The Google-Pinpoint fetcher requests individual URLs that Pinpoint users specified as sources for their personal collections of documents.", "search_engine": false, "robots_token": null, "robots_policy": "may_ignore", "robots_quote": "Because the fetch was requested by a user, these fetchers generally ignore robots.txt rules.", "ip_list_agent": "user-triggered-fetchers-google", "signs": false, "note": "User-triggered fetcher. IP file inferred from Google's statement that fetchers controlled by Google use user-triggered-fetchers-google.json.", "sources": [ { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "title": "List of Google user-triggered fetchers", "fetched_at": "2026-10-28T17:15:05.416Z" } ] }, { "name": "Google Publisher Center", "coverage": "researched", "user_agent_tokens": [ "GoogleProducer" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": "Google Publisher Center fetches and processes feeds that publishers explicitly supplied for use in Google News landing pages.", "search_engine": false, "robots_token": null, "robots_policy": "may_ignore", "robots_quote": "Because the fetch was requested by a user, these fetchers generally ignore robots.txt rules.", "ip_list_agent": "user-triggered-fetchers-google", "signs": false, "note": "User-triggered fetcher. IP file inferred from Google's statement that fetchers controlled by Google use user-triggered-fetchers-google.json.", "sources": [ { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "title": "List of Google user-triggered fetchers", "fetched_at": "2026-10-28T17:15:05.416Z" } ] }, { "name": "Google Read Aloud", "coverage": "researched", "user_agent_tokens": [ "Google-Read-Aloud" ], "aliases": [ "google-speakr" ], "purpose": "user_fetch", "purpose_quote": "Upon user request, Google Read Aloud fetches and reads out web pages using text-to-speech (TTS).", "search_engine": false, "robots_token": null, "robots_policy": "may_ignore", "robots_quote": "Since Google Read Aloud is initiated by a user, not a result of automated web crawl, you can't opt out by using a robots.txt file.", "ip_list_agent": "user-triggered-fetchers-google", "signs": false, "note": "User-triggered fetcher. google-speakr is the older, deprecated name of the user agent. Google offers the nopagereadaloud meta tag as the opt-out. IP file inferred from Google's statement that fetchers controlled by Google use user-triggered-fetchers-google.json.", "sources": [ { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "title": "List of Google user-triggered fetchers", "fetched_at": "2026-10-28T17:15:05.416Z" }, { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/read-aloud-user-agent", "title": "Google Read Aloud user agent", "fetched_at": "2026-10-28T17:15:10.396Z" } ] }, { "name": "Google Site Verifier", "coverage": "researched", "user_agent_tokens": [ "Google-Site-Verification" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": "Google Site Verifier fetches Search Console verification tokens.", "search_engine": false, "robots_token": null, "robots_policy": "may_ignore", "robots_quote": "Because the fetch was requested by a user, these fetchers generally ignore robots.txt rules.", "ip_list_agent": "user-triggered-fetchers-google", "signs": false, "note": "User-triggered fetcher. IP file inferred from Google's statement that fetchers controlled by Google use user-triggered-fetchers-google.json.", "sources": [ { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "title": "List of Google user-triggered fetchers", "fetched_at": "2026-10-28T17:15:05.416Z" } ] }, { "name": "Google StoreBot", "coverage": "researched", "user_agent_tokens": [ "Storebot-Google" ], "aliases": [], "purpose": "search", "purpose_quote": "Crawling preferences addressed to the Storebot-Google user agent affect all surfaces of Google Shopping (for example, the Shopping tab in Google Search and Google Shopping).", "search_engine": false, "robots_token": "Storebot-Google", "robots_policy": "honors", "robots_quote": "They always obey robots.txt rules when crawling automatically.", "ip_list_agent": "common-crawlers", "signs": false, "note": "Common crawler for Google Shopping. Google says it affects the Shopping tab in Google Search, but it is a shopping crawler rather than one of the Googlebot search crawlers, so search_engine is false here (a judgment call). Its only robots.txt token is Storebot-Google.", "sources": [ { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "title": "List of Google's common crawlers", "fetched_at": "2026-10-28T17:15:04.095Z" } ] }, { "name": "Google-Agent", "coverage": "researched", "user_agent_tokens": [ "Google-Agent" ], "aliases": [], "purpose": "agent", "purpose_quote": "Google-Agent is used by agents hosted on Google infrastructure to navigate the web and perform actions upon user request.", "search_engine": false, "robots_token": null, "robots_policy": "may_ignore", "robots_quote": "Because the fetch was requested by a user, these fetchers generally ignore robots.txt rules.", "ip_list_agent": "user-triggered-agents", "signs": true, "note": "Listed by Google with the user-triggered fetchers; Google says it uses IP ranges from user-triggered-agents.json. Google signs a subset of its requests with Web Bot Auth (experimental), identified as https://agent.bot.goog; see signing.", "sources": [ { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "title": "List of Google user-triggered fetchers", "fetched_at": "2026-10-28T17:15:05.416Z" } ] }, { "name": "Google-CloudVertexBot", "coverage": "researched", "user_agent_tokens": [ "Google-CloudVertexBot" ], "aliases": [], "purpose": "other", "purpose_quote": "Crawling preferences addressed to the Google-CloudVertexBot user agent affect crawls requested by the site owners' for building Vertex AI Agents.", "search_engine": false, "robots_token": "Google-CloudVertexBot", "robots_policy": "honors", "robots_quote": "They always obey robots.txt rules when crawling automatically.", "ip_list_agent": "common-crawlers", "signs": false, "note": "Common crawler. Google documents only a User-Agent substring (Google-CloudVertexBot), not a full string. Google lists two robots.txt tokens for it, Google-CloudVertexBot and Googlebot, and says it has no effect on Google Search or other products.", "sources": [ { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "title": "List of Google's common crawlers", "fetched_at": "2026-10-28T17:15:04.095Z" } ] }, { "name": "Google-Extended", "coverage": "researched", "user_agent_tokens": [], "aliases": [], "purpose": "training", "purpose_quote": "Google-Extended is a standalone product token that web publishers can use to manage whether content Google crawls from their sites may be used for training future generations of Gemini models", "search_engine": false, "robots_token": "Google-Extended", "robots_policy": "honors", "robots_quote": "Crawling is done with existing Google user agent strings; the robots.txt user-agent token is used in a control capacity.", "ip_list_agent": null, "signs": false, "note": "Robots.txt control token only, with no HTTP user agent string of its own: requests come from Google's existing crawlers (such as Googlebot). Google says it also governs grounding in Gemini Apps and Grounding with Google Search on Vertex AI, and does not affect inclusion or ranking in Google Search. The purpose quote is the opening of a longer sentence.", "sources": [ { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "title": "List of Google's common crawlers", "fetched_at": "2026-10-28T17:15:04.095Z" } ] }, { "name": "Google-InspectionTool", "coverage": "researched", "user_agent_tokens": [ "Google-InspectionTool" ], "aliases": [], "purpose": "other", "purpose_quote": "Crawling preferences addressed to the Google-InspectionTool user agent affect Search testing tools such as the Rich Result Test and URL inspection in Search Console.", "search_engine": false, "robots_token": "Google-InspectionTool", "robots_policy": "honors", "robots_quote": "They always obey robots.txt rules when crawling automatically.", "ip_list_agent": "common-crawlers", "signs": false, "note": "Common crawler used by Search Console testing tools. Google says it has no effect on Google Search or other products. Google lists two robots.txt tokens for it, Google-InspectionTool and Googlebot.", "sources": [ { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "title": "List of Google's common crawlers", "fetched_at": "2026-10-28T17:15:04.095Z" } ] }, { "name": "Google-Safety", "coverage": "researched", "user_agent_tokens": [ "Google-Safety" ], "aliases": [], "purpose": "other", "purpose_quote": "The Google-Safety user agent handles abuse-specific crawling, such as malware discovery for publicly posted links on Google properties.", "search_engine": false, "robots_token": null, "robots_policy": "may_ignore", "robots_quote": "The Google-Safety user agent ignores robots.txt rules.", "ip_list_agent": "special-crawlers", "signs": false, "note": "Special-case crawler with no robots.txt token: Google says it is unaffected by crawling preferences.", "sources": [ { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-special-case-crawlers", "title": "List of Google's special-case crawlers", "fetched_at": "2026-10-28T17:15:04.522Z" } ] }, { "name": "Googlebot", "coverage": "researched", "user_agent_tokens": [ "Googlebot" ], "aliases": [ "Googlebot Desktop", "Googlebot Smartphone" ], "purpose": "search", "purpose_quote": "Crawling preferences addressed to the Googlebot user agent affect Google Search (including Discover and all Google Search features), as well as other products such as Google Images, Video mode, Google News, and Discover.", "search_engine": true, "robots_token": "Googlebot", "robots_policy": "honors", "robots_quote": "They always obey robots.txt rules when crawling automatically.", "ip_list_agent": "common-crawlers", "signs": false, "note": "Common crawler. Googlebot Smartphone and Googlebot Desktop are the two documented subtypes; both obey the Googlebot robots.txt token. Reverse DNS crawl-***-***-***-***.googlebot.com or geo-crawl-***-***-***-***.geo.googlebot.com.", "sources": [ { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "title": "List of Google's common crawlers", "fetched_at": "2026-10-28T17:15:04.095Z" } ] }, { "name": "Googlebot Image", "coverage": "researched", "user_agent_tokens": [ "Googlebot-Image" ], "aliases": [], "purpose": "search", "purpose_quote": "Crawling preferences addressed to the Googlebot-Image user agent affect Google Images, Discover, Video mode, and all features in Google Search where images, logos, and favicons are presented.", "search_engine": true, "robots_token": "Googlebot-Image", "robots_policy": "honors", "robots_quote": "They always obey robots.txt rules when crawling automatically.", "ip_list_agent": "common-crawlers", "signs": false, "note": "Common crawler. Google lists two robots.txt tokens for it, Googlebot-Image and Googlebot; matching either one applies a rule.", "sources": [ { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "title": "List of Google's common crawlers", "fetched_at": "2026-10-28T17:15:04.095Z" } ] }, { "name": "Googlebot News", "coverage": "researched", "user_agent_tokens": [], "aliases": [], "purpose": "search", "purpose_quote": "Crawling preferences addressed to the Googlebot-News user agent affect the Google News product, including news.google.com and the Google News app.", "search_engine": false, "robots_token": "Googlebot-News", "robots_policy": "honors", "robots_quote": "They always obey robots.txt rules when crawling automatically.", "ip_list_agent": null, "signs": false, "note": "Robots.txt token only. Google says Googlebot-News has no separate HTTP user agent string and crawling is done with various Googlebot user agent strings, so its requests arrive as Googlebot (which is marked as the search engine crawler). Documented as affecting Google News rather than Google Search, so search_engine is false here.", "sources": [ { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "title": "List of Google's common crawlers", "fetched_at": "2026-10-28T17:15:04.095Z" } ] }, { "name": "Googlebot Video", "coverage": "researched", "user_agent_tokens": [ "Googlebot-Video" ], "aliases": [], "purpose": "search", "purpose_quote": "Crawling preferences addressed to the Googlebot-Video user agent affect video-related Google Search features and other products dependent on videos.", "search_engine": true, "robots_token": "Googlebot-Video", "robots_policy": "honors", "robots_quote": "They always obey robots.txt rules when crawling automatically.", "ip_list_agent": "common-crawlers", "signs": false, "note": "Common crawler. Google lists two robots.txt tokens for it, Googlebot-Video and Googlebot.", "sources": [ { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "title": "List of Google's common crawlers", "fetched_at": "2026-10-28T17:15:04.095Z" } ] }, { "name": "GoogleOther", "coverage": "researched", "user_agent_tokens": [ "GoogleOther" ], "aliases": [], "purpose": "other", "purpose_quote": "GoogleOther is the generic crawler that may be used by various product teams for fetching publicly accessible content from sites.", "search_engine": false, "robots_token": "GoogleOther", "robots_policy": "honors", "robots_quote": "They always obey robots.txt rules when crawling automatically.", "ip_list_agent": "common-crawlers", "signs": false, "note": "Common crawler. Google gives one-off crawls for internal research and development as an example use.", "sources": [ { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "title": "List of Google's common crawlers", "fetched_at": "2026-10-28T17:15:04.095Z" } ] }, { "name": "GoogleOther-Image", "coverage": "researched", "user_agent_tokens": [ "GoogleOther-Image" ], "aliases": [], "purpose": "other", "purpose_quote": "GoogleOther-Image is the version of GoogleOther optimized for fetching publicly accessible image URLs.", "search_engine": false, "robots_token": "GoogleOther-Image", "robots_policy": "honors", "robots_quote": "They always obey robots.txt rules when crawling automatically.", "ip_list_agent": "common-crawlers", "signs": false, "note": "Common crawler. Google lists two robots.txt tokens for it, GoogleOther-Image and GoogleOther.", "sources": [ { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "title": "List of Google's common crawlers", "fetched_at": "2026-10-28T17:15:04.095Z" } ] }, { "name": "GoogleOther-Video", "coverage": "researched", "user_agent_tokens": [ "GoogleOther-Video" ], "aliases": [], "purpose": "other", "purpose_quote": "GoogleOther-Video is the version of GoogleOther optimized for fetching publicly accessible video URLs.", "search_engine": false, "robots_token": "GoogleOther-Video", "robots_policy": "honors", "robots_quote": "They always obey robots.txt rules when crawling automatically.", "ip_list_agent": "common-crawlers", "signs": false, "note": "Common crawler. Google lists two robots.txt tokens for it, GoogleOther-Video and GoogleOther.", "sources": [ { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "title": "List of Google's common crawlers", "fetched_at": "2026-10-28T17:15:04.095Z" } ] }, { "name": "Mediapartners-Google", "coverage": "researched", "user_agent_tokens": [ "Mediapartners-Google" ], "aliases": [], "purpose": "ads", "purpose_quote": "The Mediapartners-Google crawler visits participating sites in order to provide them with relevant ads.", "search_engine": false, "robots_token": "Mediapartners-Google", "robots_policy": "may_ignore", "robots_quote": "The special-case crawlers may ignore robots.txt rules and so they operate from a different IP range than the common crawlers.", "ip_list_agent": "special-crawlers", "signs": false, "note": "Special-case crawler (AdSense and Ad Manager). Google says it ignores the global user agent (*); it obeys a group addressed to Mediapartners-Google.", "sources": [ { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-special-case-crawlers", "title": "List of Google's special-case crawlers", "fetched_at": "2026-10-28T17:15:04.522Z" } ] }, { "name": "CloudVertexBot", "coverage": "community", "user_agent_tokens": [ "CloudVertexBot" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "CloudVertexBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Data Scrapers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "Gemini-Deep-Research", "coverage": "community", "user_agent_tokens": [ "Gemini-Deep-Research" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": null, "search_engine": false, "robots_token": "Gemini-Deep-Research", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Assistants\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "Google-Firebase", "coverage": "community", "user_agent_tokens": [ "Google-Firebase" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": null, "search_engine": false, "robots_token": "Google-Firebase", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Used as part of AI apps developed by users of Google's Firebase AI products.\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "Google-Gemini-CLI", "coverage": "community", "user_agent_tokens": [ "Google-Gemini-CLI" ], "aliases": [], "purpose": "agent", "purpose_quote": null, "search_engine": false, "robots_token": "Google-Gemini-CLI", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Coding Agents\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "GoogleAgent-Mariner", "coverage": "community", "user_agent_tokens": [ "GoogleAgent-Mariner" ], "aliases": [], "purpose": "agent", "purpose_quote": null, "search_engine": false, "robots_token": "GoogleAgent-Mariner", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Agents\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "GoogleAgent-URLContext", "coverage": "community", "user_agent_tokens": [ "GoogleAgent-URLContext" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": null, "search_engine": false, "robots_token": "GoogleAgent-URLContext", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Assistants\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "NotebookLM", "coverage": "community", "user_agent_tokens": [ "NotebookLM" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": null, "search_engine": false, "robots_token": "NotebookLM", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Assistants\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/overview-google-crawlers", "title": "Overview of Google crawlers and fetchers (user agents)", "fetched_at": "2026-10-28T17:15:02.933Z" }, { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "title": "List of Google's common crawlers", "fetched_at": "2026-10-28T17:15:04.095Z" }, { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-special-case-crawlers", "title": "List of Google's special-case crawlers", "fetched_at": "2026-10-28T17:15:04.522Z" }, { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "title": "List of Google user-triggered fetchers", "fetched_at": "2026-10-28T17:15:05.416Z" }, { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/verify-google-requests", "title": "Verify requests from Google crawlers and fetchers", "fetched_at": "2026-10-28T17:15:06.902Z" }, { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/web-bot-auth", "title": "Authenticate requests with Web Bot Auth (experimental)", "fetched_at": "2026-10-28T17:15:07.481Z" }, { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/apis-user-agent", "title": "APIs-Google user agent", "fetched_at": "2026-10-28T17:15:08.643Z" }, { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/feedfetcher", "title": "Feedfetcher", "fetched_at": "2026-10-28T17:15:09.460Z" }, { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/read-aloud-user-agent", "title": "Google Read Aloud user agent", "fetched_at": "2026-10-28T17:15:10.396Z" } ] }, { "slug": "hark", "name": "Hark", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt lists the operator as unclear; it is named in the agent's own description there.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "HarkBot", "coverage": "community", "user_agent_tokens": [ "HarkBot" ], "aliases": [], "purpose": "agent", "purpose_quote": null, "search_engine": false, "robots_token": "HarkBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Its operator is named in its ai.robots.txt description. Function: \"AI Agents\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "huawei", "name": "Huawei", "website": null, "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "PanguBot", "coverage": "community", "user_agent_tokens": [ "PanguBot" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "PanguBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Data Scrapers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "PetalBot", "coverage": "community", "user_agent_tokens": [ "PetalBot" ], "aliases": [], "purpose": "search", "purpose_quote": null, "search_engine": false, "robots_token": "PetalBot", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Used to provide recommendations in Hauwei assistant and AI search services.\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "iask", "name": "iAsk", "website": null, "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "iaskspider", "coverage": "community", "user_agent_tokens": [ "iaskspider" ], "aliases": [ "iaskspider/2.0" ], "purpose": "user_fetch", "purpose_quote": null, "search_engine": false, "robots_token": "iaskspider", "robots_policy": "may_ignore", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Crawls sites to provide answers to user queries.\". Respects robots.txt: \"No\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "imagesift", "name": "ImageSift", "website": "https://imagesift.com", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "ImagesiftBot", "coverage": "community", "user_agent_tokens": [ "ImagesiftBot" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "ImagesiftBot", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"ImageSiftBot is a web crawler that scrapes the internet for publicly available images to support their suite of web intelligence products\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "img2dataset", "name": "img2dataset", "website": null, "docs_url": "https://github.com/rom1504/img2dataset", "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "img2dataset", "coverage": "community", "user_agent_tokens": [ "img2dataset" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "img2dataset", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Scrapes images for use in LLMs.\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "iss-corporate", "name": "ISS-Corporate", "website": "https://iss-cyber.com", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "ISSCyberRiskCrawler", "coverage": "community", "user_agent_tokens": [ "ISSCyberRiskCrawler" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "ISSCyberRiskCrawler", "robots_policy": "may_ignore", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Scrapes data to train machine learning models.\". Respects robots.txt: \"No\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "kagi", "name": "Kagi", "website": null, "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "kagi-fetcher", "coverage": "community", "user_agent_tokens": [ "kagi-fetcher" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": null, "search_engine": false, "robots_token": "kagi-fetcher", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Assistants\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "kangaroo-llm", "name": "Kangaroo LLM", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt lists the operator as unclear; it is named in the agent's own description there.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Kangaroo Bot", "coverage": "community", "user_agent_tokens": [ "Kangaroo Bot" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "Kangaroo Bot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Its operator is named in its ai.robots.txt description. Function: \"AI Data Scrapers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "keenable", "name": "Keenable", "website": "https://keenable.ai/", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "KeenableBot", "coverage": "community", "user_agent_tokens": [ "KeenableBot" ], "aliases": [], "purpose": "search", "purpose_quote": null, "search_engine": false, "robots_token": "KeenableBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Search Crawlers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "kimi", "name": "Kimi", "website": "https://www.kimi.ai/", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "MoonshotBot", "coverage": "community", "user_agent_tokens": [ "MoonshotBot" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "MoonshotBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"MoonshotBot is a training crawler.\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "klaviyo", "name": "Klaviyo", "website": "https://www.klaviyo.com", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "KlaviyoAIBot", "coverage": "community", "user_agent_tokens": [ "KlaviyoAIBot" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": null, "search_engine": false, "robots_token": "KlaviyoAIBot", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Assistants\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "large-scale-artificial-intelligence-open-network", "name": "Large-scale Artificial Intelligence Open Network", "website": "https://laion.ai/", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "LAIONDownloader", "coverage": "community", "user_agent_tokens": [ "LAIONDownloader" ], "aliases": [], "purpose": "search", "purpose_quote": null, "search_engine": false, "robots_token": "LAIONDownloader", "robots_policy": "may_ignore", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI tools and models for machine learning research.\". Respects robots.txt: \"No\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "lightpanda-client-users", "name": "Lightpanda client users", "website": null, "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Lightpanda", "coverage": "community", "user_agent_tokens": [ "Lightpanda" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "Lightpanda", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Data Scrapers\". Respects robots.txt: \"At the discretion of Lightpanda users.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "liner", "name": "Liner", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt lists the operator as unclear; it is named in the agent's own description there.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "LinerBot", "coverage": "community", "user_agent_tokens": [ "LinerBot" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": null, "search_engine": false, "robots_token": "LinerBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Its operator is named in its ai.robots.txt description. Function: \"AI Assistants\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "linguee", "name": "Linguee", "website": "https://www.linguee.com", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Linguee Bot", "coverage": "community", "user_agent_tokens": [ "Linguee Bot" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "Linguee Bot", "robots_policy": "may_ignore", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI powered translation service\". Respects robots.txt: \"No\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "lyrenth", "name": "Lyrenth", "website": "https://lyrenth.com", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "AIWebIndex", "coverage": "community", "user_agent_tokens": [ "AIWebIndex" ], "aliases": [], "purpose": "search", "purpose_quote": null, "search_engine": false, "robots_token": "AIWebIndex", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Search Crawlers\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "meltwater", "name": "Meltwater", "website": null, "docs_url": "https://www.meltwater.com/en/suite/consumer-intelligence", "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "YaK", "coverage": "community", "user_agent_tokens": [ "YaK" ], "aliases": [], "purpose": "search", "purpose_quote": null, "search_engine": false, "robots_token": "YaK", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"According to the Meltwater Consumer Intelligence page 'By applying AI, data science, and market research expertise to a live feed of global data sources, we transform unstructured data into actionable insights allowing better decision-making'.\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "meta", "name": "Meta", "website": "https://www.meta.com/", "docs_url": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers", "coverage": "researched", "note": null, "verification": { "signing": { "status": "live", "directory_url": "https://www.meta.com/.well-known/http-message-signatures-directory", "kid_convention": "none", "purpose_field": null, "covers": [], "note": "Meta's crawler documentation does not mention HTTP message signatures or Web Bot Auth. A third-party survey (mod_pagespeed blog, probe of 19 October 2026) lists this directory, which it took from Cloudflare Radar, and attributes it to a bot it calls Meta-ExternalTest; Meta's own page does not list that bot, so covers is left empty. On 2026-10-26 the URL returned 200 with content type application/http-message-signatures-directory+json and five Ed25519 keys without kid values. https://meta.com/ redirects (301) to the www host; https://www.facebook.com/ returns 404 for the well-known path. The developers.facebook.com page answered 400 to a browser-like User-Agent, 200 HTML to the directory builder's User-Agent and 200 Markdown to curl's default User-Agent; the quotes here use the HTML wording (curly apostrophes), while the Markdown rendering has straight ones.", "sources": [ { "url": "https://modpagespeed.com/blog/which-ai-crawlers-sign-their-requests/", "title": "55 of 150 AI crawler operators publish Web Bot Auth keys (third party, mod_pagespeed blog)", "fetched_at": "2026-10-28T17:14:32.840Z" }, { "url": "https://modpagespeed.com/data/web-bot-auth-2026-10-19/results.tsv", "title": "Web Bot Auth key directory probe results, 19 October 2026 (third party)", "fetched_at": "2026-10-28T17:14:34.046Z" }, { "url": "https://www.meta.com/.well-known/http-message-signatures-directory", "title": "Key directory", "fetched_at": "2026-10-28T17:15:11.426Z" } ] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "FacebookExternalHit", "coverage": "researched", "user_agent_tokens": [ "facebookexternalhit" ], "aliases": [], "purpose": "preview", "purpose_quote": "The primary purpose of FacebookExternalHit is to crawl the content of an app or website that was shared on one of Meta’s family of apps, such as Facebook, Instagram, or Messenger.", "search_engine": false, "robots_token": "facebookexternalhit", "robots_policy": "may_ignore", "robots_quote": "Note that the FacebookExternalHit crawler might bypass robots.txt when performing security or integrity checks, such as checking for malware or malicious content.", "ip_list_agent": null, "signs": false, "note": "Meta's page gives no robots.txt example for this crawler; robots_token is the product token from its documented User-Agent string. The page tells site owners to allow-list the crawler's user agent strings or, as the more secure option, its IP addresses, but it publishes no IP list or ASN.", "sources": [ { "url": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers", "title": "Meta Web Crawlers", "fetched_at": "2026-10-28T17:15:12.666Z" } ] }, { "name": "Meta-ExternalAds", "coverage": "researched", "user_agent_tokens": [ "meta-externalads" ], "aliases": [], "purpose": "ads", "purpose_quote": "The Meta-ExternalAds crawler crawls the web for use cases such as improving advertising and other business-related products and services.", "search_engine": false, "robots_token": "meta-externalads", "robots_policy": "honors", "robots_quote": "In order to block these crawlers, add a disallow for the relevant crawler to robots.txt.", "ip_list_agent": null, "signs": false, "note": null, "sources": [ { "url": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers", "title": "Meta Web Crawlers", "fetched_at": "2026-10-28T17:15:12.666Z" } ] }, { "name": "Meta-ExternalAgent", "coverage": "researched", "user_agent_tokens": [ "meta-externalagent" ], "aliases": [], "purpose": "training", "purpose_quote": "The Meta-ExternalAgent crawler crawls the web for use cases such as training foundation AI models or improving products by indexing content directly.", "search_engine": false, "robots_token": "meta-externalagent", "robots_policy": "honors", "robots_quote": "In order to block these crawlers, add a disallow for the relevant crawler to robots.txt.", "ip_list_agent": null, "signs": false, "note": null, "sources": [ { "url": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers", "title": "Meta Web Crawlers", "fetched_at": "2026-10-28T17:15:12.666Z" } ] }, { "name": "Meta-ExternalFetcher", "coverage": "researched", "user_agent_tokens": [ "meta-externalfetcher" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": "The Meta-ExternalFetcher crawler fetches individual links at a user’s request and supports product functions such as evaluating and improving agentic AI capabilities", "search_engine": false, "robots_token": "meta-externalfetcher", "robots_policy": "may_ignore", "robots_quote": "The Meta-ExternalFetcher crawler may bypass robots.txt because it performs fetches that were requested by the user.", "ip_list_agent": null, "signs": false, "note": "Meta's page gives no robots.txt example for this crawler; robots_token is the product token from its documented User-Agent string. The same sentence goes on to say it helps \"AI navigate websites to complete tasks for users\", so it also serves agent-style tasks.", "sources": [ { "url": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers", "title": "Meta Web Crawlers", "fetched_at": "2026-10-28T17:15:12.666Z" } ] }, { "name": "Meta-WebIndexer", "coverage": "researched", "user_agent_tokens": [ "meta-webindexer" ], "aliases": [], "purpose": "search", "purpose_quote": "The Meta-WebIndexer crawler navigates the web to improve Meta AI search result quality for users.", "search_engine": false, "robots_token": "meta-webindexer", "robots_policy": "honors", "robots_quote": "In order to block these crawlers, add a disallow for the relevant crawler to robots.txt.", "ip_list_agent": null, "signs": false, "note": null, "sources": [ { "url": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers", "title": "Meta Web Crawlers", "fetched_at": "2026-10-28T17:15:12.666Z" } ] }, { "name": "FacebookBot", "coverage": "community", "user_agent_tokens": [ "FacebookBot" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "FacebookBot", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Training language models\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "Muse", "coverage": "community", "user_agent_tokens": [ "Muse" ], "aliases": [], "purpose": "agent", "purpose_quote": null, "search_engine": false, "robots_token": "Muse", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Its operator is named in its ai.robots.txt description. Function: \"AI Agents\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers", "title": "Meta Web Crawlers", "fetched_at": "2026-10-28T17:15:12.666Z" }, { "url": "https://modpagespeed.com/blog/which-ai-crawlers-sign-their-requests/", "title": "55 of 150 AI crawler operators publish Web Bot Auth keys (third party, mod_pagespeed blog)", "fetched_at": "2026-10-28T17:14:32.840Z" }, { "url": "https://modpagespeed.com/data/web-bot-auth-2026-10-19/results.tsv", "title": "Web Bot Auth key directory probe results, 19 October 2026 (third party)", "fetched_at": "2026-10-28T17:14:34.046Z" } ] }, { "slug": "microsoft", "name": "Microsoft", "website": "https://www.microsoft.com/", "docs_url": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0", "coverage": "researched", "note": null, "verification": { "signing": { "status": "none_found", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": "Bing's crawler and verification pages do not mention Web Bot Auth or HTTP message signatures, and no third-party report was found. On 2026-10-26 the key directory path returned 404 on www.bing.com and www.microsoft.com. No key directory found in the operator's documentation.", "sources": [ { "url": "https://www.bing.com/webmasters/api/help/htmlcontent?ArticleId=8c184ec0", "title": "Which Crawlers Does Bing Use? (article body of https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0)", "fetched_at": "2026-10-28T17:15:17.244Z" }, { "url": "https://www.bing.com/webmasters/api/help/htmlcontent?ArticleId=3905dc26", "title": "How to Verify Bingbot (article body of https://www.bing.com/webmasters/help/how-to-verify-bingbot-3905dc26)", "fetched_at": "2026-10-28T17:15:18.366Z" } ] }, "ip_lists": [ { "agent": "bingbot", "url": "https://www.bing.com/toolbox/bingbot.json", "format": "ipranges-json", "prefix_count": 28, "fetched_at": "2026-10-28T17:15:15.955Z", "sha256": "a2d1cf229544205a24c61fb7c459408b1afe18f9fd5b172fe181f0eb73c1594d", "prefixes": [ "13.66.139.0/24", "13.66.144.0/24", "13.67.10.16/28", "13.69.66.240/28", "13.71.172.224/28", "20.15.133.160/27", "20.36.108.32/28", "20.43.120.16/28", "20.74.197.0/28", "20.79.107.240/28", "20.125.163.80/28", "40.77.139.0/25", "40.77.167.0/24", "40.77.177.0/24", "40.77.178.0/23", "40.77.188.0/22", "40.77.202.0/24", "40.79.131.208/28", "40.79.186.176/28", "51.105.67.0/28", "52.167.144.0/24", "52.231.148.0/28", "65.55.210.0/24", "139.217.52.0/28", "157.55.39.0/24", "191.233.204.224/28", "199.30.24.0/23", "207.46.13.0/24" ], "note": null, "sources": [ { "url": "https://www.bing.com/webmasters/api/help/htmlcontent?ArticleId=3905dc26", "title": "How to Verify Bingbot (article body of https://www.bing.com/webmasters/help/how-to-verify-bingbot-3905dc26)", "fetched_at": "2026-10-28T17:15:18.366Z" } ] } ], "reverse_dns": { "suffixes": [ "search.msn.com" ], "forward_confirm": true, "note": null, "sources": [ { "url": "https://www.bing.com/webmasters/api/help/htmlcontent?ArticleId=3905dc26", "title": "How to Verify Bingbot (article body of https://www.bing.com/webmasters/help/how-to-verify-bingbot-3905dc26)", "fetched_at": "2026-10-28T17:15:18.366Z" } ] }, "asn": [] }, "agents": [ { "name": "AdIdxBot", "coverage": "researched", "user_agent_tokens": [ "adidxbot" ], "aliases": [], "purpose": "ads", "purpose_quote": "AdIdxBot is the crawler used by Bing Ads.", "search_engine": false, "robots_token": null, "robots_policy": "honors", "robots_quote": "Robots.txt files can be configured to tell Bing crawlers how to interact with your website.", "ip_list_agent": null, "signs": false, "note": "Its user agent strings carry the token as adidxbot/2.0. Bing names no robots.txt token for it; the robots quote is Bing's general statement about its crawlers. Bing publishes an IP list only for Bingbot, so none is linked.", "sources": [ { "url": "https://www.bing.com/webmasters/api/help/htmlcontent?ArticleId=8c184ec0", "title": "Which Crawlers Does Bing Use? (article body of https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0)", "fetched_at": "2026-10-28T17:15:17.244Z" } ] }, { "name": "Bingbot", "coverage": "researched", "user_agent_tokens": [ "bingbot" ], "aliases": [ "MSNBOT" ], "purpose": "search", "purpose_quote": "Bingbot is our standard crawler and handles most of our crawling needs each day.", "search_engine": true, "robots_token": "Bingbot", "robots_policy": "honors", "robots_quote": "Bingbot honors robots.txt directives, including the crawl-delay: setting", "ip_list_agent": "bingbot", "signs": false, "note": "Bing's search crawler; its user agent strings carry the token as bingbot/2.0. The robots quote is the opening of a longer sentence and keeps the narrow no-break space (\\u202F) the page has before crawl-delay. The Verify Bingbot article refers to Bingbot traffic using a Bingbot or MSNBOT user agent string, hence the alias. Bing's guidelines say Copilot and grounding results rely on the same crawl, and use NOARCHIVE and NOCACHE meta directives (not a separate robots.txt token) for Copilot use.", "sources": [ { "url": "https://www.bing.com/webmasters/api/help/htmlcontent?ArticleId=8c184ec0", "title": "Which Crawlers Does Bing Use? (article body of https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0)", "fetched_at": "2026-10-28T17:15:17.244Z" }, { "url": "https://www.bing.com/webmasters/api/help/htmlcontent?ArticleId=25c19802", "title": "How to Report an Issue with Bingbot (article body of https://www.bing.com/webmasters/help/how-to-report-an-issue-with-bingbot-25c19802)", "fetched_at": "2026-10-28T17:15:20.236Z" }, { "url": "https://www.bing.com/webmasters/api/help/htmlcontent?ArticleId=2195837f", "title": "Verify Bingbot (article body of https://www.bing.com/webmasters/help/verify-bingbot-2195837f)", "fetched_at": "2026-10-28T17:15:19.391Z" } ] }, { "name": "BingPreview", "coverage": "researched", "user_agent_tokens": [], "aliases": [], "purpose": "preview", "purpose_quote": "BingPreview generates page snapshots for Bing.", "search_engine": false, "robots_token": null, "robots_policy": "honors", "robots_quote": "Robots.txt files can be configured to tell Bing crawlers how to interact with your website.", "ip_list_agent": null, "signs": false, "note": "Not a robots-only token: Bing's crawler page lists, for BingPreview, the same user agent strings as Bingbot (bingbot/2.0), so no separate BingPreview token is documented and its requests cannot be told apart from Bingbot by user agent. Bing names no robots.txt token for it.", "sources": [ { "url": "https://www.bing.com/webmasters/api/help/htmlcontent?ArticleId=8c184ec0", "title": "Which Crawlers Does Bing Use? (article body of https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0)", "fetched_at": "2026-10-28T17:15:17.244Z" } ] }, { "name": "BingVideoPreview", "coverage": "researched", "user_agent_tokens": [ "BingVideoPreview" ], "aliases": [], "purpose": "preview", "purpose_quote": "BingVideoPreview is a crawler used by Microsoft to provide previews of videos only in Bing.", "search_engine": false, "robots_token": null, "robots_policy": "honors", "robots_quote": "Robots.txt files can be configured to tell Bing crawlers how to interact with your website.", "ip_list_agent": null, "signs": false, "note": "The purpose quote keeps the non-breaking space (\\u00A0) the page has after BingVideoPreview. Bing names no robots.txt token for it. No IP list is documented for it.", "sources": [ { "url": "https://www.bing.com/webmasters/api/help/htmlcontent?ArticleId=8c184ec0", "title": "Which Crawlers Does Bing Use? (article body of https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0)", "fetched_at": "2026-10-28T17:15:17.244Z" } ] }, { "name": "MicrosoftPreview", "coverage": "researched", "user_agent_tokens": [ "MicrosoftPreview" ], "aliases": [], "purpose": "preview", "purpose_quote": "MicrosoftPreview generates page snapshots for Microsoft products.", "search_engine": false, "robots_token": null, "robots_policy": "honors", "robots_quote": "Robots.txt files can be configured to tell Bing crawlers how to interact with your website.", "ip_list_agent": null, "signs": false, "note": "Bing names no robots.txt token for it; the robots quote is Bing's general statement about its crawlers. No IP list is documented for it.", "sources": [ { "url": "https://www.bing.com/webmasters/api/help/htmlcontent?ArticleId=8c184ec0", "title": "Which Crawlers Does Bing Use? (article body of https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0)", "fetched_at": "2026-10-28T17:15:17.244Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://www.bing.com/webmasters/api/help/htmlcontent?ArticleId=8c184ec0", "title": "Which Crawlers Does Bing Use? (article body of https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0)", "fetched_at": "2026-10-28T17:15:17.244Z" }, { "url": "https://www.bing.com/webmasters/api/help/htmlcontent?ArticleId=3905dc26", "title": "How to Verify Bingbot (article body of https://www.bing.com/webmasters/help/how-to-verify-bingbot-3905dc26)", "fetched_at": "2026-10-28T17:15:18.366Z" }, { "url": "https://www.bing.com/webmasters/api/help/htmlcontent?ArticleId=2195837f", "title": "Verify Bingbot (article body of https://www.bing.com/webmasters/help/verify-bingbot-2195837f)", "fetched_at": "2026-10-28T17:15:19.391Z" }, { "url": "https://www.bing.com/webmasters/api/help/htmlcontent?ArticleId=25c19802", "title": "How to Report an Issue with Bingbot (article body of https://www.bing.com/webmasters/help/how-to-report-an-issue-with-bingbot-25c19802)", "fetched_at": "2026-10-28T17:15:20.236Z" } ] }, { "slug": "mistral", "name": "Mistral AI", "website": "https://mistral.ai/", "docs_url": "https://docs.mistral.ai/robots", "coverage": "researched", "note": null, "verification": { "signing": { "status": "none_found", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": "Mistral's crawler page does not mention request signing, and no third-party report was found. On 2026-10-26 /.well-known/http-message-signatures-directory returned 404 on mistral.ai and docs.mistral.ai; chat.mistral.ai returned its web app HTML (status 200), not a key directory. No key directory found in the operator's documentation.", "sources": [] }, "ip_lists": [ { "agent": "mistralai-user", "url": "https://mistral.ai/mistralai-user-ips.json", "format": "ipranges-json", "prefix_count": 4, "fetched_at": "2026-10-28T17:15:21.307Z", "sha256": "6226c5829c7edb91995769a9f3f0a52b686b4a18cc5a8d6d92479d2908f793f6", "prefixes": [ "20.240.160.1/32", "20.240.160.161/32", "20.240.194.83/32", "51.12.243.114/32" ], "note": null, "sources": [ { "url": "https://docs.mistral.ai/robots", "title": "Mistral crawlers", "fetched_at": "2026-10-28T17:15:23.565Z" } ] }, { "agent": "mistralai-index", "url": "https://mistral.ai/mistralai-index-ips.json", "format": "ipranges-json", "prefix_count": 2, "fetched_at": "2026-10-28T17:15:22.652Z", "sha256": "35bf7aede93ea6a24b12606cf0abb809d8fe9846b2e054e0174dc53f91921c20", "prefixes": [ "20.240.192.194/32", "135.225.57.92/32" ], "note": null, "sources": [ { "url": "https://docs.mistral.ai/robots", "title": "Mistral crawlers", "fetched_at": "2026-10-28T17:15:23.565Z" } ] } ], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "MistralAI-Index", "coverage": "researched", "user_agent_tokens": [ "MistralAI-Index" ], "aliases": [], "purpose": "search", "purpose_quote": "It indexes content for Mistral search, which helps answer user questions in Vibe.", "search_engine": false, "robots_token": "MistralAI-Index", "robots_policy": "honors", "robots_quote": "To help webmasters manage how their sites and content interact with AI, Mistral uses specific robots.txt tags.", "ip_list_agent": "mistralai-index", "signs": false, "note": "The robots.txt statement is the page's general one for all three agents; there is no MistralAI-Index specific sentence. Mistral says content crawled by MistralAI-Index is not used for generative AI training.", "sources": [ { "url": "https://docs.mistral.ai/robots", "title": "Mistral crawlers", "fetched_at": "2026-10-28T17:15:23.565Z" } ] }, { "name": "MistralAI-Training", "coverage": "researched", "user_agent_tokens": [ "MistralAI-Training" ], "aliases": [], "purpose": "training", "purpose_quote": "MistralAI-Training crawls web content to help build datasets for training Mistral generative AI models.", "search_engine": false, "robots_token": "MistralAI-Training", "robots_policy": "honors", "robots_quote": "Webmasters can disallow this user agent in their robots.txt file.", "ip_list_agent": null, "signs": false, "note": "Mistral's page publishes no IP list for MistralAI-Training.", "sources": [ { "url": "https://docs.mistral.ai/robots", "title": "Mistral crawlers", "fetched_at": "2026-10-28T17:15:23.565Z" } ] }, { "name": "MistralAI-User", "coverage": "researched", "user_agent_tokens": [ "MistralAI-User" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": "When users ask Vibe a question, it may visit a web page to help answer and include a link to the source in its response.", "search_engine": false, "robots_token": "MistralAI-User", "robots_policy": "honors", "robots_quote": "MistralAI-User governs which sites these user requests can be made to.", "ip_list_agent": "mistralai-user", "signs": false, "note": "Mistral lists MistralAI-User among its robots.txt tags and does not say it may ignore robots.txt. Mistral says it is not used for automatic crawling or for generative AI training.", "sources": [ { "url": "https://docs.mistral.ai/robots", "title": "Mistral crawlers", "fetched_at": "2026-10-28T17:15:23.565Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://docs.mistral.ai/robots", "title": "Mistral crawlers", "fetched_at": "2026-10-28T17:15:23.565Z" } ] }, { "slug": "moonshot-ai", "name": "Moonshot AI", "website": null, "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Kimi-Agent", "coverage": "community", "user_agent_tokens": [ "Kimi-Agent" ], "aliases": [], "purpose": "agent", "purpose_quote": null, "search_engine": false, "robots_token": "Kimi-Agent", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Agents\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "Kimi-SearchBot", "coverage": "community", "user_agent_tokens": [ "Kimi-SearchBot" ], "aliases": [], "purpose": "search", "purpose_quote": null, "search_engine": false, "robots_token": "Kimi-SearchBot", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Search Crawlers\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "Kimi-User", "coverage": "community", "user_agent_tokens": [ "Kimi-User" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": null, "search_engine": false, "robots_token": "Kimi-User", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Assistants\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "KimiBot", "coverage": "community", "user_agent_tokens": [ "KimiBot" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "KimiBot", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Data Scrapers\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "mozilla", "name": "Mozilla", "website": null, "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Mozilla-Tabstack", "coverage": "community", "user_agent_tokens": [ "Mozilla-Tabstack" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "Mozilla-Tabstack", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Data Providers\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "naget-inc", "name": "Naget Inc", "website": null, "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "NagetBot", "coverage": "community", "user_agent_tokens": [ "NagetBot" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "NagetBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI data scraper\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "nbot", "name": "NBot", "website": "https://nbot.ai", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "nbot", "coverage": "community", "user_agent_tokens": [ "nbot" ], "aliases": [], "purpose": "search", "purpose_quote": null, "search_engine": false, "robots_token": "nbot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Search Crawlers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "netestate", "name": "netEstate", "website": null, "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "netEstate Imprint Crawler", "coverage": "community", "user_agent_tokens": [ "netEstate Imprint Crawler" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "netEstate Imprint Crawler", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Data Scrapers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "nict", "name": "NICT", "website": "https://nict.go.jp", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "ICC-Crawler", "coverage": "community", "user_agent_tokens": [ "ICC-Crawler" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "ICC-Crawler", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Scrapes data to train and support AI technologies.\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "openai", "name": "OpenAI", "website": "https://openai.com/", "docs_url": "https://developers.openai.com/api/docs/bots", "coverage": "researched", "note": null, "verification": { "signing": { "status": "reported", "directory_url": "https://chatgpt.com/.well-known/http-message-signatures-directory", "kid_convention": null, "purpose_field": null, "covers": [ "ChatGPT Agent" ], "note": "The directory URL comes from a third-party write-up (Castle). OpenAI's own crawler page does not mention signing. OpenAI's help article on allowlisting ChatGPT agent (help.openai.com/en/articles/11845367, now titled \"ChatGPT Work's Cloud browser allowlisting\") appears from search results to document RFC 9421 signing with Signature-Agent \"https://chatgpt.com\", but it returned HTTP 403 to curl and to WebFetch on 2026-10-26, so it is not cited here. If that article can be read, basis can become documented. ChatGPT agent is listed under covers by its ai.robots.txt key; it has no OpenAI-documented User-Agent token here. Could not confirm in this build (blocked_by_robots: robots.txt for https://chatgpt.com disallows /.well-known/http-message-signatures-directory for PosternaDirectoryBuilder).", "sources": [ { "url": "https://blog.castle.io/how-to-authenticate-openai-operator-requests-using-http-message-signatures/", "title": "How to authenticate OpenAI Operator requests using HTTP message signatures (Castle, third party)", "fetched_at": "2026-10-28T17:15:32.581Z" } ] }, "ip_lists": [ { "agent": "gptbot", "url": "https://openai.com/gptbot.json", "format": "ipranges-json", "prefix_count": 18, "fetched_at": "2026-10-28T17:15:24.999Z", "sha256": "a24d1fa34f094d303400bdf13c45c5e9726e0b553780d16a52f9ba1b790acc72", "prefixes": [ "4.227.36.0/25", "20.125.66.80/28", "20.171.206.0/24", "20.171.207.0/24", "52.230.152.0/24", "74.7.175.128/25", "74.7.227.0/25", "74.7.227.128/25", "74.7.228.0/25", "74.7.230.0/25", "74.7.241.0/25", "74.7.241.128/25", "74.7.242.0/25", "74.7.243.128/25", "74.7.244.0/25", "132.196.86.0/24", "172.182.202.0/25", "172.182.207.0/25" ], "note": null, "sources": [ { "url": "https://developers.openai.com/api/docs/bots", "title": "Overview of OpenAI Crawlers", "fetched_at": "2026-10-28T17:15:31.412Z" } ] }, { "agent": "searchbot", "url": "https://openai.com/searchbot.json", "format": "ipranges-json", "prefix_count": 39, "fetched_at": "2026-10-28T17:15:26.084Z", "sha256": "81806b1665e9bbbc5e0f307b847ad34f7027abc19e0d2d5d3eca1864bf361be6", "prefixes": [ "4.227.36.0/25", "13.66.216.176/28", "20.14.99.96/28", "20.25.151.224/28", "20.42.10.176/28", "20.168.18.32/28", "20.169.6.224/28", "20.169.7.48/28", "20.169.77.0/25", "20.171.53.224/28", "20.171.123.64/28", "23.102.145.48/28", "40.67.175.0/25", "40.90.214.16/28", "51.8.102.0/24", "74.7.175.128/25", "74.7.228.0/25", "74.7.228.128/25", "74.7.229.0/25", "74.7.229.128/25", "74.7.230.0/25", "74.7.241.128/25", "74.7.242.128/25", "74.7.243.0/25", "74.7.244.0/25", "104.210.140.128/28", "135.234.64.0/24", "172.182.193.80/28", "172.182.193.224/28", "172.182.194.32/28", "172.182.194.144/28", "172.182.195.48/28", "172.182.209.208/28", "172.182.211.192/28", "172.182.213.192/28", "172.182.224.0/28", "172.203.190.80/28", "172.203.190.112/28", "172.203.190.128/28" ], "note": null, "sources": [ { "url": "https://developers.openai.com/api/docs/bots", "title": "Overview of OpenAI Crawlers", "fetched_at": "2026-10-28T17:15:31.412Z" } ] }, { "agent": "adsbot", "url": "https://openai.com/adsbot.json", "format": "ipranges-json", "prefix_count": 2, "fetched_at": "2026-10-28T17:15:27.091Z", "sha256": "460b800ffb026c5ec1b9540d285f2ade8e1c4a654e65dae315e2bae650b5d74d", "prefixes": [ "20.168.90.128/25", "130.131.229.128/25" ], "note": null, "sources": [ { "url": "https://developers.openai.com/api/docs/bots", "title": "Overview of OpenAI Crawlers", "fetched_at": "2026-10-28T17:15:31.412Z" } ] }, { "agent": "chatgpt-user", "url": "https://openai.com/chatgpt-user.json", "format": "ipranges-json", "prefix_count": 235, "fetched_at": "2026-10-28T17:15:28.095Z", "sha256": "9d4d2d9db7a781488f3bbf7ed1327229a7e5a308918e63fd2ecbe2a4f6b90a45", "prefixes": [ "4.151.71.176/28", "4.151.119.48/28", "4.189.118.208/28", "4.189.119.48/28", "4.197.22.112/28", "4.197.64.0/28", "4.197.64.16/28", "4.197.64.48/28", "4.197.64.64/28", "4.197.115.112/28", "4.201.232.64/28", "4.205.128.176/28", "4.218.24.64/28", "4.226.200.16/28", "4.226.226.32/28", "9.129.0.0/16", "9.160.34.144/28", "9.160.36.16/28", "9.160.96.16/28", "9.160.128.16/28", "9.160.128.64/28", "9.160.163.128/28", "9.163.101.48/28", "9.205.8.48/28", "9.205.8.64/28", "9.205.25.128/28", "9.205.30.128/28", "9.205.30.144/28", "9.205.30.176/28", "9.205.128.32/28", "9.205.128.48/28", "9.223.181.208/28", "9.234.96.192/28", "9.234.97.96/28", "13.65.138.112/28", "13.67.72.16/28", "13.71.2.208/28", "13.76.32.208/28", "13.76.116.80/28", "13.83.167.128/28", "13.83.237.176/28", "20.45.178.144/28", "20.55.229.144/28", "20.57.199.192/28", "20.63.180.96/28", "20.63.221.64/28", "20.78.130.48/28", "20.83.243.176/28", "20.98.18.80/28", "20.102.212.144/28", "20.113.211.112/28", "20.113.225.112/28", "20.125.112.224/28", "20.125.144.144/28", "20.161.75.208/28", "20.168.7.192/28", "20.169.73.32/28", "20.169.73.64/28", "20.169.78.48/28", "20.169.78.64/28", "20.169.78.128/28", "20.169.78.160/28", "20.169.78.192/28", "20.169.86.224/28", "20.169.86.240/28", "20.170.184.16/28", "20.170.184.32/28", "20.170.184.48/28", "20.170.184.64/28", "20.170.184.80/28", "20.172.29.32/28", "20.199.211.160/28", "20.199.242.0/28", "20.200.212.240/28", "20.206.101.192/28", "20.210.211.192/28", "20.212.62.208/28", "20.215.187.208/28", "20.215.219.128/28", "20.215.219.160/28", "20.215.219.208/28", "20.218.30.240/28", "20.219.71.192/28", "20.219.161.192/28", "20.219.184.96/28", "20.222.36.192/28", "20.227.140.32/28", "20.228.106.176/28", "20.235.87.224/28", "20.249.63.208/28", "20.250.6.128/28", "20.250.136.64/28", "20.254.201.208/28", "23.98.142.176/28", "23.98.186.64/28", "23.98.186.96/28", "23.98.186.176/28", "23.98.186.192/28", "23.101.217.176/28", "23.102.140.144/28", "23.102.141.32/28", "40.74.200.208/28", "40.81.67.96/28", "40.81.234.144/28", "40.84.221.208/28", "40.84.221.224/28", "40.116.73.208/28", "40.119.36.240/28", "40.124.161.0/28", "48.221.40.176/28", "48.221.184.80/28", "48.221.184.96/28", "51.8.155.48/28", "51.8.155.112/28", "51.57.0.96/28", "51.59.24.64/28", "51.59.24.80/28", "51.59.48.80/28", "51.116.2.80/28", "51.116.221.96/28", "52.148.129.32/28", "52.156.77.144/28", "52.159.227.32/28", "52.159.249.96/28", "52.161.49.96/28", "52.165.212.48/28", "52.172.129.160/28", "52.173.219.96/28", "52.173.219.112/28", "52.173.221.16/28", "52.173.221.176/28", "52.173.221.208/28", "52.173.234.16/28", "52.173.234.80/28", "52.183.217.240/28", "52.190.137.16/28", "52.190.137.144/28", "52.190.139.48/28", "52.190.142.64/28", "52.190.190.16/28", "52.190.251.112/28", "52.225.75.208/28", "52.231.30.48/28", "52.231.34.176/28", "52.231.39.144/28", "52.231.50.64/28", "52.236.94.144/28", "52.241.146.208/28", "52.242.132.224/28", "52.242.132.240/28", "52.255.109.80/28", "52.255.109.96/28", "52.255.109.144/28", "52.255.111.0/28", "52.255.111.80/28", "57.154.174.112/28", "57.154.175.0/28", "57.154.187.32/28", "68.154.28.96/28", "68.220.57.64/28", "70.153.32.16/28", "70.153.32.32/28", "70.153.87.224/28", "70.153.189.192/28", "70.156.152.96/28", "74.7.35.48/28", "74.7.35.112/28", "74.7.36.64/28", "74.7.36.80/28", "74.7.36.96/28", "74.161.200.96/28", "74.224.217.64/28", "74.226.253.160/28", "74.248.37.160/28", "85.211.128.16/28", "85.211.128.32/28", "104.208.184.192/28", "104.210.139.192/28", "104.210.139.224/28", "128.85.198.32/28", "132.196.82.48/28", "134.33.102.192/28", "134.138.52.16/28", "134.138.52.64/28", "134.138.52.96/28", "134.138.52.128/28", "134.138.57.64/28", "134.138.57.80/28", "134.138.57.176/28", "134.138.58.32/28", "134.149.233.80/28", "135.13.64.240/28", "135.116.136.160/28", "135.220.73.208/28", "135.220.73.240/28", "135.222.99.32/28", "135.237.131.208/28", "135.237.133.48/28", "137.135.191.176/28", "138.91.46.96/28", "145.132.136.96/28", "158.158.5.32/28", "168.63.252.240/28", "172.162.248.64/28", "172.170.1.80/28", "172.170.225.0/28", "172.170.241.80/28", "172.172.206.48/28", "172.175.152.224/28", "172.178.140.144/28", "172.178.141.112/28", "172.178.141.128/28", "172.183.143.224/28", "172.183.222.128/28", "172.192.112.208/28", "172.197.160.192/28", "172.197.203.16/28", "172.198.58.176/28", "172.198.78.192/28", "172.198.79.112/28", "172.198.161.128/28", "172.199.137.80/28", "172.204.28.224/28", "172.204.96.80/28", "172.205.189.192/28", "172.206.38.240/28", "172.207.1.32/28", "172.212.172.160/28", "172.215.215.32/28", "191.232.238.96/28", "191.233.197.0/28", "191.233.199.160/28", "191.234.167.144/28", "191.235.99.160/28", "191.237.249.64/28" ], "note": null, "sources": [ { "url": "https://developers.openai.com/api/docs/bots", "title": "Overview of OpenAI Crawlers", "fetched_at": "2026-10-28T17:15:31.412Z" } ] }, { "agent": "chatgpt-connectors", "url": "https://openai.com/chatgpt-connectors.json", "format": "ipranges-json", "prefix_count": 288, "fetched_at": "2026-10-28T17:15:29.164Z", "sha256": "3d147a86e688b4aafc6f12d9696a0d987229e66fba0082e8b493fe980518b305", "prefixes": [ "3.12.200.18/32", "3.140.2.201/32", "4.7.10.112/30", "4.7.11.196/30", "4.14.111.0/28", "4.17.25.128/29", "4.19.160.0/28", "4.38.166.228/30", "4.53.139.144/28", "4.151.71.176/28", "4.151.119.48/28", "4.151.200.38/32", "4.155.146.196/32", "4.185.216.109/32", "4.189.118.208/28", "4.197.64.0/28", "4.197.64.48/28", "4.197.115.112/28", "4.197.172.116/32", "4.201.232.64/28", "4.205.128.176/28", "4.217.235.100/32", "4.218.24.64/28", "4.226.200.16/28", "4.226.226.32/28", "4.245.198.13/32", "8.244.149.100/30", "9.129.0.0/16", "9.160.96.16/28", "9.160.128.16/28", "9.160.128.64/28", "9.205.8.48/28", "9.205.8.64/28", "9.205.128.32/28", "9.205.128.48/28", "9.234.96.192/28", "9.234.97.96/28", "12.12.47.194/32", "12.12.56.24/29", "12.12.56.32/29", "12.12.56.224/28", "12.12.56.240/28", "12.77.42.78/32", "12.79.34.30/32", "12.79.201.188/30", "12.79.202.152/30", "12.79.202.156/30", "12.79.202.228/30", "12.79.202.232/30", "12.79.225.144/30", "12.105.90.64/27", "12.108.172.96/28", "12.117.245.68/30", "12.129.184.64/26", "12.162.186.200/29", "13.65.138.112/28", "13.67.72.16/28", "13.71.2.208/28", "13.71.25.29/32", "13.76.32.208/28", "13.76.116.80/28", "13.83.237.176/28", "13.223.161.115/32", "13.237.176.161/32", "13.238.110.96/32", "15.168.252.168/32", "18.218.234.253/32", "20.44.100.224/28", "20.45.178.144/28", "20.55.229.144/28", "20.57.199.192/28", "20.63.221.64/28", "20.74.221.21/32", "20.78.130.48/28", "20.98.18.80/28", "20.102.212.144/28", "20.125.40.252/32", "20.125.112.224/28", "20.162.96.163/32", "20.168.7.192/28", "20.168.48.117/32", "20.169.78.48/28", "20.169.78.64/28", "20.169.86.224/28", "20.170.184.16/28", "20.170.184.32/28", "20.170.184.48/28", "20.170.184.64/28", "20.170.184.80/28", "20.171.137.175/32", "20.172.29.32/28", "20.184.36.134/32", "20.203.144.245/32", "20.206.101.192/28", "20.212.62.208/28", "20.215.187.208/28", "20.215.219.208/28", "20.219.124.186/32", "20.219.161.192/28", "20.219.184.96/28", "20.227.140.32/28", "20.228.106.176/28", "20.235.87.224/28", "20.241.32.36/32", "20.249.63.208/28", "20.250.136.64/28", "20.254.201.208/28", "23.98.186.64/28", "23.98.186.96/28", "23.101.217.176/28", "23.102.141.32/28", "24.82.185.0/29", "40.88.27.77/32", "40.118.236.137/32", "40.119.36.240/28", "40.122.118.93/32", "40.122.118.119/32", "40.122.118.202/32", "40.124.161.0/28", "43.202.230.227/32", "44.221.134.118/32", "44.249.227.138/32", "45.147.211.96/29", "48.218.181.198/32", "48.221.40.176/28", "48.221.184.80/28", "48.221.184.96/28", "50.145.17.208/30", "50.145.17.212/30", "50.145.17.216/29", "50.145.17.224/29", "50.151.105.128/30", "50.151.105.136/29", "50.213.205.80/29", "50.235.235.72/29", "51.4.112.173/32", "51.57.0.96/28", "51.59.24.64/28", "51.59.24.80/28", "51.59.48.80/28", "52.2.184.223/32", "52.6.94.121/32", "52.17.188.55/32", "52.43.161.225/32", "52.119.123.85/32", "52.143.181.161/32", "52.148.129.32/28", "52.165.212.48/28", "52.172.129.160/28", "52.173.221.16/28", "52.173.234.16/28", "52.173.234.80/28", "52.190.137.16/28", "52.190.137.144/28", "52.190.139.48/28", "52.190.142.64/28", "52.190.251.112/28", "52.208.217.159/32", "52.231.30.48/28", "52.231.39.144/28", "52.242.132.224/28", "52.242.132.240/28", "52.255.109.80/28", "52.255.109.96/28", "52.255.109.144/28", "52.255.111.0/28", "54.180.197.31/32", "54.227.131.66/32", "56.155.71.179/32", "57.133.92.112/31", "57.154.174.112/28", "57.154.187.32/28", "61.105.58.228/30", "62.96.221.184/29", "64.71.12.112/28", "64.124.21.196/32", "64.124.191.96/28", "66.193.99.66/32", "67.207.103.240/28", "68.154.28.96/28", "68.155.152.41/32", "68.220.57.64/28", "70.153.32.16/28", "70.153.32.32/28", "70.156.152.96/28", "72.146.20.246/32", "74.7.35.48/28", "74.7.35.112/28", "74.7.36.64/28", "74.7.36.80/28", "74.7.36.96/28", "74.161.200.96/28", "74.224.217.64/28", "74.226.253.160/28", "74.248.37.160/28", "74.248.148.7/32", "76.77.188.112/29", "77.75.96.48/29", "79.244.198.212/30", "80.169.53.32/28", "85.211.128.16/28", "85.211.128.32/28", "98.87.72.221/32", "100.31.168.162/32", "102.37.57.54/32", "104.192.219.204/30", "104.208.184.192/28", "104.210.139.192/28", "104.210.139.224/28", "108.179.20.6/31", "112.220.228.112/29", "115.42.241.224/29", "128.177.85.168/30", "128.177.174.162/32", "130.33.24.99/32", "132.196.82.48/28", "134.33.102.192/28", "134.138.52.16/28", "134.138.52.64/28", "134.138.57.64/28", "134.138.57.80/28", "134.149.233.80/28", "135.13.64.240/28", "135.116.136.160/28", "135.220.40.201/32", "135.220.73.208/28", "135.220.208.92/32", "135.222.99.32/28", "135.234.27.89/32", "135.237.133.48/28", "137.135.191.176/28", "145.132.136.96/28", "148.76.185.192/27", "148.109.10.28/30", "148.109.36.240/28", "149.97.160.16/28", "152.44.170.32/29", "159.180.234.92/30", "172.162.248.64/28", "172.167.32.228/32", "172.167.161.96/32", "172.170.1.80/28", "172.170.225.0/28", "172.170.241.80/28", "172.171.234.186/32", "172.172.206.48/28", "172.175.152.224/28", "172.177.53.240/28", "172.183.143.224/28", "172.191.70.179/32", "172.191.238.68/32", "172.192.112.208/28", "172.198.58.176/28", "172.198.79.112/28", "172.199.137.80/28", "172.203.39.49/32", "172.204.96.80/28", "172.206.38.240/28", "172.207.1.32/28", "172.207.173.200/32", "172.214.226.198/32", "172.215.215.32/28", "173.195.76.0/26", "180.222.194.124/30", "184.73.124.134/32", "191.232.238.96/28", "191.233.251.27/32", "191.234.167.144/28", "191.237.249.64/28", "194.46.223.16/28", "195.171.64.176/28", "199.47.142.0/23", "199.241.201.152/29", "203.125.229.136/29", "203.149.223.128/29", "208.52.97.112/29", "208.69.43.136/29", "208.80.35.32/27", "208.184.8.84/30", "208.184.8.104/29", "209.247.142.56/30", "209.247.151.176/28", "209.249.37.128/26", "209.249.246.178/31", "213.122.44.84/31", "216.64.170.234/32", "217.111.182.45/32", "217.111.242.24/29" ], "note": "OpenAI publishes these as the \"ChatGPT connectors\" ranges for ChatGPT integrations, used for \"Plugins, connectors, GPT Actions, and agentic commerce\". Not linked to an agent: OpenAI documents no user-agent token for these requests. ChatGPT-User, which OpenAI also mentions with GPT Actions, keeps its own list (chatgpt-user.json), and an agent can point at only one list.", "sources": [ { "url": "https://developers.openai.com/api/docs/guides/ip-addresses", "title": "IP egress ranges", "fetched_at": "2026-10-28T17:15:33.800Z" } ] }, { "agent": "chatgpt-agents", "url": "https://openai.com/chatgpt-agents.json", "format": "ipranges-json", "prefix_count": 1758, "fetched_at": "2026-10-28T17:15:30.140Z", "sha256": "bd10b33475827bc318432ebc86e7a18bc5089b0a73ffe934be63cffd26c2130a", "prefixes": [ "4.150.3.64/28", "4.151.5.0/28", "4.151.71.64/28", "4.151.71.144/28", "4.151.86.96/28", "4.151.95.128/28", "4.151.95.208/28", "4.151.134.128/28", "4.151.142.96/28", "4.151.211.224/28", "4.151.244.224/28", "4.154.55.160/28", "4.154.62.160/28", "4.154.90.96/28", "4.154.244.32/28", "4.154.244.208/28", "4.154.246.80/28", "4.154.247.0/28", "4.154.247.96/28", "4.155.7.160/28", "4.155.64.16/28", "4.155.68.144/28", "4.155.223.80/28", "4.155.238.112/28", "4.155.239.160/28", "4.155.249.0/28", "4.155.255.64/28", "4.155.255.192/28", "4.196.157.160/28", "4.196.182.192/28", "4.196.182.208/28", "4.196.183.16/28", "4.196.199.0/28", "4.196.199.80/28", "4.196.203.160/28", "4.196.205.144/28", "4.196.209.0/28", "4.196.220.0/28", "4.196.220.16/28", "4.196.227.176/28", "4.196.228.96/28", "4.197.19.160/28", "4.197.21.16/28", "4.197.22.64/28", "4.197.29.48/28", "4.197.35.192/28", "4.197.43.176/28", "4.197.43.224/28", "4.197.51.240/28", "4.197.64.80/28", "4.197.80.224/28", "4.197.104.16/28", "4.197.106.96/28", "4.197.120.0/28", "4.197.121.128/28", "4.197.139.144/28", "4.197.140.176/28", "4.197.142.144/28", "4.197.142.160/28", "4.197.142.176/28", "4.197.142.192/28", "4.197.142.224/28", "4.197.142.240/28", "4.197.143.112/28", "4.198.97.16/28", "4.198.97.32/28", "4.198.97.48/28", "4.199.9.16/28", "4.201.88.224/28", "4.201.89.160/28", "4.201.89.192/28", "4.201.89.224/28", "4.201.90.64/28", "4.201.99.0/28", "4.201.99.16/28", "4.201.99.48/28", "4.201.99.128/28", "4.201.100.96/28", "4.201.100.128/28", "4.201.100.160/28", "4.201.108.160/28", "4.201.108.176/28", "4.201.108.192/28", "4.201.232.224/28", "4.201.232.240/28", "4.203.96.160/28", "4.203.96.176/28", "4.203.96.192/28", "4.224.17.208/28", "4.224.26.144/28", "4.224.80.64/28", "4.224.127.48/28", "4.228.83.48/28", "4.228.84.192/28", "4.228.86.64/28", "4.228.88.160/28", "4.228.89.0/28", "4.228.90.240/28", "4.228.98.128/28", "4.228.99.144/28", "4.228.99.160/28", "4.228.136.176/28", "4.228.136.192/28", "4.228.136.208/28", "4.228.136.224/28", "4.228.185.112/28", "4.228.185.128/28", "4.228.185.192/28", "4.228.200.144/28", "4.228.201.80/28", "4.228.202.144/28", "4.228.209.208/28", "4.240.2.0/28", "4.240.6.192/28", "4.246.85.16/28", "4.246.85.112/28", "9.129.0.0/16", "9.160.96.80/28", "9.160.128.128/28", "9.205.8.144/28", "9.205.128.128/28", "9.234.4.192/28", "9.234.4.240/28", "9.234.5.64/28", "9.234.21.48/28", "9.234.99.0/28", "9.234.99.16/28", "9.234.106.160/28", "9.235.32.128/28", "9.235.56.144/28", "9.235.56.176/28", "9.235.56.192/28", "13.70.107.160/28", "13.71.1.16/28", "13.71.5.128/28", "13.71.211.192/28", "13.71.212.64/28", "13.72.231.192/28", "13.75.177.16/28", "13.78.227.176/28", "13.83.235.80/28", "13.83.235.160/28", "13.83.235.224/28", "13.83.236.32/28", "13.83.236.80/28", "13.83.236.96/28", "13.83.236.112/28", "13.83.236.128/28", "13.83.236.160/28", "13.83.236.176/28", "13.83.236.224/28", "13.83.236.240/28", "13.83.237.16/28", "13.83.237.32/28", "13.83.237.64/28", "13.83.237.80/28", "13.83.237.112/28", "13.83.237.128/28", "13.86.109.48/28", "13.86.109.128/28", "13.86.109.144/28", "13.86.109.176/28", "13.86.110.0/28", "13.86.110.80/28", "13.86.243.48/28", "13.86.245.96/28", "13.88.58.80/28", "13.88.58.96/28", "13.88.58.112/28", "13.88.60.0/28", "13.88.85.240/28", "13.88.141.128/28", "13.88.141.144/28", "13.88.142.176/28", "13.89.127.48/28", "13.89.127.64/28", "13.89.127.80/28", "13.89.127.96/28", "13.89.127.112/28", "13.91.144.112/28", "20.3.127.112/28", "20.5.60.112/28", "20.5.67.160/28", "20.5.89.176/28", "20.5.119.240/28", "20.5.211.112/28", "20.5.222.32/28", "20.9.185.80/28", "20.11.249.32/28", "20.11.249.48/28", "20.11.249.64/28", "20.12.223.80/28", "20.14.75.112/28", "20.14.76.176/28", "20.14.77.128/28", "20.14.77.144/28", "20.14.79.128/28", "20.14.79.144/28", "20.14.79.160/28", "20.14.79.176/28", "20.14.79.192/28", "20.14.102.208/28", "20.15.229.0/28", "20.15.230.0/28", "20.15.230.32/28", "20.25.149.240/28", "20.25.181.176/28", "20.28.214.32/28", "20.28.239.224/28", "20.29.37.48/28", "20.29.38.144/28", "20.29.38.160/28", "20.29.38.176/28", "20.29.55.80/28", "20.29.55.96/28", "20.40.215.0/28", "20.40.215.176/28", "20.40.255.128/28", "20.40.255.144/28", "20.40.255.160/28", "20.40.255.192/28", "20.45.181.16/28", "20.45.181.192/28", "20.45.181.208/28", "20.45.181.224/28", "20.45.182.96/28", "20.45.182.160/28", "20.45.182.224/28", "20.45.183.32/28", "20.45.183.80/28", "20.45.183.96/28", "20.45.183.112/28", "20.45.183.128/28", "20.45.183.144/28", "20.45.183.176/28", "20.45.183.192/28", "20.45.183.240/28", "20.46.236.96/28", "20.46.236.192/28", "20.46.237.0/28", "20.51.60.144/28", "20.51.60.240/28", "20.51.61.16/28", "20.51.61.96/28", "20.51.62.192/28", "20.55.158.224/28", "20.55.163.128/28", "20.55.165.16/28", "20.55.167.64/28", "20.55.170.240/28", "20.55.171.32/28", "20.55.171.48/28", "20.55.171.240/28", "20.55.172.112/28", "20.55.173.32/28", "20.55.173.128/28", "20.55.173.144/28", "20.55.173.160/28", "20.55.178.208/28", "20.55.179.32/28", "20.55.179.48/28", "20.55.179.224/28", "20.55.179.240/28", "20.55.180.0/28", "20.55.180.16/28", "20.55.180.32/28", "20.55.180.48/28", "20.55.180.64/28", "20.55.180.80/28", "20.55.185.96/28", "20.55.185.240/28", "20.55.186.0/28", "20.55.186.16/28", "20.55.186.32/28", "20.55.186.80/28", "20.55.186.96/28", "20.55.186.112/28", "20.55.186.176/28", "20.57.215.0/28", "20.57.223.64/28", "20.57.233.80/28", "20.57.233.96/28", "20.57.233.128/28", "20.57.233.144/28", "20.57.235.192/28", "20.57.235.208/28", "20.57.235.224/28", "20.57.235.240/28", "20.57.236.0/28", "20.57.236.32/28", "20.57.236.48/28", "20.57.236.64/28", "20.57.236.80/28", "20.57.236.96/28", "20.57.236.112/28", "20.57.236.128/28", "20.57.236.144/28", "20.57.236.160/28", "20.57.236.176/28", "20.57.236.192/28", "20.57.236.208/28", "20.57.236.224/28", "20.57.236.240/28", "20.57.237.0/28", "20.57.237.16/28", "20.57.237.32/28", "20.57.237.48/28", "20.57.237.64/28", "20.57.237.80/28", "20.57.237.96/28", "20.57.237.112/28", "20.57.237.128/28", "20.57.237.144/28", "20.57.237.160/28", "20.57.237.176/28", "20.57.237.192/28", "20.57.237.208/28", "20.57.237.224/28", "20.58.141.112/28", "20.58.148.224/28", "20.59.200.192/28", "20.59.202.32/28", "20.59.202.128/28", "20.59.203.208/28", "20.59.216.192/28", "20.59.217.112/28", "20.59.217.208/28", "20.59.218.16/28", "20.59.218.64/28", "20.59.218.128/28", "20.59.218.144/28", "20.59.218.176/28", "20.59.218.192/28", "20.59.218.208/28", "20.59.218.224/28", "20.59.218.240/28", "20.59.226.0/28", "20.59.226.16/28", "20.59.226.64/28", "20.59.226.80/28", "20.59.226.96/28", "20.59.226.112/28", "20.59.226.128/28", "20.59.226.144/28", "20.59.227.64/28", "20.59.227.80/28", "20.59.227.96/28", "20.59.233.0/28", "20.59.244.144/28", "20.59.244.160/28", "20.59.244.176/28", "20.59.244.192/28", "20.59.244.208/28", "20.59.244.224/28", "20.59.245.32/28", "20.59.245.128/28", "20.59.245.144/28", "20.59.248.0/28", "20.59.248.48/28", "20.59.248.80/28", "20.59.248.96/28", "20.59.248.160/28", "20.59.248.192/28", "20.59.248.224/28", "20.59.249.0/28", "20.59.249.16/28", "20.59.249.32/28", "20.59.249.48/28", "20.59.249.80/28", "20.59.249.240/28", "20.64.16.64/28", "20.64.16.160/28", "20.64.57.128/28", "20.64.65.0/28", "20.64.65.16/28", "20.64.65.32/28", "20.65.153.176/28", "20.65.153.240/28", "20.65.154.192/28", "20.65.155.0/28", "20.65.226.240/28", "20.65.251.224/28", "20.66.97.240/28", "20.80.109.192/28", "20.80.111.240/28", "20.81.245.144/28", "20.83.37.192/28", "20.83.37.208/28", "20.83.37.224/28", "20.83.37.240/28", "20.84.157.160/28", "20.84.157.176/28", "20.84.157.192/28", "20.84.157.208/28", "20.84.157.224/28", "20.84.245.176/28", "20.84.245.192/28", "20.84.245.208/28", "20.84.246.0/28", "20.84.246.32/28", "20.94.54.48/28", "20.94.54.176/28", "20.94.54.208/28", "20.94.55.64/28", "20.94.55.128/28", "20.94.55.144/28", "20.94.55.160/28", "20.94.55.192/28", "20.94.55.224/28", "20.94.55.240/28", "20.97.62.64/28", "20.98.10.160/28", "20.98.34.80/28", "20.98.133.48/28", "20.98.133.64/28", "20.98.133.112/28", "20.98.133.144/28", "20.98.142.240/28", "20.98.143.0/28", "20.98.143.16/28", "20.98.143.32/28", "20.102.221.0/28", "20.102.221.176/28", "20.102.222.112/28", "20.102.222.224/28", "20.102.223.0/28", "20.102.223.32/28", "20.106.22.64/28", "20.106.22.80/28", "20.117.23.112/28", "20.117.59.128/28", "20.117.60.16/28", "20.117.60.48/28", "20.118.31.96/28", "20.118.31.112/28", "20.118.31.128/28", "20.118.37.0/28", "20.118.37.48/28", "20.118.39.64/28", "20.118.215.128/28", "20.118.215.144/28", "20.118.223.48/28", "20.118.223.128/28", "20.118.223.144/28", "20.118.223.160/28", "20.118.223.176/28", "20.118.229.0/28", "20.118.231.224/28", "20.118.231.240/28", "20.118.239.48/28", "20.118.239.64/28", "20.118.239.80/28", "20.118.239.96/28", "20.118.245.192/28", "20.118.246.192/28", "20.125.128.0/28", "20.125.136.128/28", "20.125.180.128/28", "20.125.180.144/28", "20.125.180.160/28", "20.125.180.176/28", "20.125.181.48/28", "20.125.182.192/28", "20.125.182.208/28", "20.125.183.0/28", "20.125.183.16/28", "20.125.183.96/28", "20.125.183.112/28", "20.125.183.224/28", "20.125.183.240/28", "20.125.188.176/28", "20.125.189.0/28", "20.125.189.16/28", "20.125.189.32/28", "20.125.189.48/28", "20.125.189.64/28", "20.125.189.80/28", "20.125.189.96/28", "20.125.189.112/28", "20.125.189.128/28", "20.125.189.144/28", "20.125.224.112/28", "20.125.224.128/28", "20.125.224.144/28", "20.125.224.160/28", "20.125.224.176/28", "20.125.224.192/28", "20.125.224.208/28", "20.125.224.224/28", "20.125.224.240/28", "20.125.225.0/28", "20.125.225.160/28", "20.125.225.176/28", "20.161.28.240/28", "20.161.29.32/28", "20.163.33.96/28", "20.163.35.0/28", "20.163.35.16/28", "20.163.35.64/28", "20.163.35.80/28", "20.163.35.128/28", "20.163.35.144/28", "20.163.35.160/28", "20.163.35.176/28", "20.163.35.192/28", "20.163.35.208/28", "20.163.41.96/28", "20.163.42.32/28", "20.163.44.48/28", "20.163.45.112/28", "20.163.46.224/28", "20.163.62.64/28", "20.163.62.128/28", "20.163.63.0/28", "20.163.63.16/28", "20.163.63.80/28", "20.163.63.160/28", "20.163.81.16/28", "20.163.91.192/28", "20.163.94.16/28", "20.163.94.176/28", "20.163.95.16/28", "20.163.105.80/28", "20.163.109.128/28", "20.165.25.112/28", "20.165.27.224/28", "20.165.29.64/28", "20.165.55.16/28", "20.165.63.32/28", "20.168.29.64/28", "20.168.52.192/28", "20.168.55.112/28", "20.168.69.0/28", "20.168.69.192/28", "20.168.70.144/28", "20.168.91.208/28", "20.168.92.0/28", "20.168.92.16/28", "20.168.92.48/28", "20.168.92.96/28", "20.168.92.112/28", "20.168.92.128/28", "20.168.92.144/28", "20.168.92.160/28", "20.168.92.176/28", "20.168.92.192/28", "20.168.92.208/28", "20.168.92.224/28", "20.168.95.176/28", "20.168.95.192/28", "20.168.95.208/28", "20.168.95.224/28", "20.168.95.240/28", "20.168.98.96/28", "20.168.98.208/28", "20.168.119.48/28", "20.168.129.48/28", "20.168.129.64/28", "20.168.129.176/28", "20.168.129.192/28", "20.168.137.128/28", "20.168.137.144/28", "20.168.137.160/28", "20.169.2.0/28", "20.169.4.240/28", "20.169.5.16/28", "20.169.17.160/28", "20.169.38.16/28", "20.169.52.80/28", "20.169.52.96/28", "20.169.52.112/28", "20.169.54.112/28", "20.169.54.192/28", "20.169.54.208/28", "20.169.54.224/28", "20.169.54.240/28", "20.169.55.224/28", "20.169.64.0/28", "20.169.64.16/28", "20.169.64.64/28", "20.169.64.96/28", "20.169.64.112/28", "20.169.64.144/28", "20.169.64.160/28", "20.169.72.96/28", "20.169.73.0/28", "20.169.78.32/28", "20.169.86.192/28", "20.169.87.32/28", "20.169.87.48/28", "20.169.87.64/28", "20.169.87.80/28", "20.169.87.176/28", "20.169.87.192/28", "20.169.87.208/28", "20.169.87.224/28", "20.169.87.240/28", "20.169.92.80/28", "20.169.92.96/28", "20.169.92.112/28", "20.169.92.128/28", "20.169.92.192/28", "20.169.92.208/28", "20.169.95.224/28", "20.169.95.240/28", "20.169.96.32/28", "20.169.96.80/28", "20.169.96.96/28", "20.169.96.112/28", "20.169.96.128/28", "20.169.96.144/28", "20.169.96.208/28", "20.169.96.240/28", "20.169.97.96/28", "20.169.97.112/28", "20.169.101.16/28", "20.169.101.32/28", "20.169.101.48/28", "20.169.101.64/28", "20.169.101.80/28", "20.169.101.96/28", "20.169.101.112/28", "20.170.130.160/28", "20.170.184.96/28", "20.170.184.112/28", "20.171.13.48/28", "20.171.13.64/28", "20.171.13.80/28", "20.171.13.128/28", "20.171.13.144/28", "20.171.13.160/28", "20.171.13.176/28", "20.171.14.128/28", "20.171.16.16/28", "20.171.16.64/28", "20.171.16.80/28", "20.171.20.208/28", "20.171.20.224/28", "20.171.20.240/28", "20.171.21.0/28", "20.171.21.16/28", "20.171.21.32/28", "20.171.21.48/28", "20.171.21.64/28", "20.171.21.128/28", "20.171.21.144/28", "20.171.21.160/28", "20.171.21.176/28", "20.171.21.192/28", "20.171.49.64/28", "20.171.126.160/28", "20.171.127.112/28", "20.172.7.0/28", "20.172.24.128/28", "20.172.107.112/28", "20.172.107.160/28", "20.172.108.48/28", "20.172.108.96/28", "20.184.150.112/28", "20.184.151.240/28", "20.189.176.16/28", "20.189.186.160/28", "20.189.188.32/28", "20.189.188.48/28", "20.193.227.32/28", "20.194.0.224/28", "20.194.26.80/28", "20.194.43.96/28", "20.194.43.240/28", "20.194.45.192/28", "20.194.46.16/28", "20.194.46.176/28", "20.194.46.224/28", "20.194.46.240/28", "20.194.47.0/28", "20.194.47.96/28", "20.194.47.160/28", "20.194.47.224/28", "20.194.47.240/28", "20.194.104.96/28", "20.194.104.160/28", "20.194.104.208/28", "20.194.105.32/28", "20.194.105.64/28", "20.194.105.208/28", "20.195.183.16/28", "20.195.183.160/28", "20.195.183.176/28", "20.195.183.192/28", "20.195.188.240/28", "20.195.224.176/28", "20.195.228.160/28", "20.197.185.64/28", "20.197.185.176/28", "20.197.199.240/28", "20.197.251.0/28", "20.197.251.128/28", "20.197.251.176/28", "20.198.112.176/28", "20.204.37.192/28", "20.204.93.192/28", "20.204.94.32/28", "20.204.95.96/28", "20.206.78.160/28", "20.206.79.48/28", "20.206.87.208/28", "20.206.93.112/28", "20.206.95.48/28", "20.206.95.224/28", "20.206.96.112/28", "20.206.96.160/28", "20.206.96.240/28", "20.206.98.64/28", "20.206.99.176/28", "20.206.100.96/28", "20.206.102.64/28", "20.206.116.0/28", "20.206.119.112/28", "20.206.120.128/28", "20.206.136.0/28", "20.206.136.224/28", "20.206.138.80/28", "20.211.173.160/28", "20.215.48.64/28", "20.215.50.32/28", "20.215.50.48/28", "20.215.50.144/28", "20.215.50.160/28", "20.215.51.128/28", "20.215.51.144/28", "20.215.51.160/28", "20.215.51.224/28", "20.215.52.208/28", "20.215.58.64/28", "20.215.58.96/28", "20.215.58.128/28", "20.215.58.160/28", "20.215.68.160/28", "20.215.68.176/28", "20.215.68.192/28", "20.215.68.208/28", "20.215.69.96/28", "20.215.69.112/28", "20.215.69.160/28", "20.215.69.176/28", "20.215.70.16/28", "20.215.70.64/28", "20.215.70.96/28", "20.215.70.208/28", "20.215.71.48/28", "20.215.71.64/28", "20.215.71.128/28", "20.215.71.144/28", "20.215.71.160/28", "20.215.71.176/28", "20.215.71.192/28", "20.215.71.224/28", "20.215.71.240/28", "20.215.188.192/28", "20.215.189.144/28", "20.215.189.176/28", "20.215.189.192/28", "20.215.190.32/28", "20.215.190.80/28", "20.215.191.0/28", "20.215.191.16/28", "20.215.191.96/28", "20.215.191.112/28", "20.215.191.176/28", "20.215.191.192/28", "20.215.191.208/28", "20.215.213.176/28", "20.215.213.192/28", "20.215.213.208/28", "20.215.213.224/28", "20.215.214.0/28", "20.215.214.16/28", "20.215.214.112/28", "20.215.219.176/28", "20.215.220.64/28", "20.215.220.80/28", "20.215.220.96/28", "20.215.220.112/28", "20.215.220.128/28", "20.215.220.144/28", "20.215.220.160/28", "20.215.220.176/28", "20.215.221.80/28", "20.215.221.112/28", "20.219.13.112/28", "20.219.58.32/28", "20.219.147.144/28", "20.221.70.240/28", "20.221.71.16/28", "20.221.71.128/28", "20.221.71.144/28", "20.221.71.160/28", "20.221.71.176/28", "20.221.71.192/28", "20.221.71.208/28", "20.221.76.176/28", "20.221.76.224/28", "20.221.77.0/28", "20.221.77.16/28", "20.225.92.240/28", "20.225.162.16/28", "20.225.167.144/28", "20.225.229.224/28", "20.226.13.192/28", "20.226.32.80/28", "20.226.48.176/28", "20.226.54.96/28", "20.226.63.224/28", "20.226.69.144/28", "20.226.71.192/28", "20.226.74.96/28", "20.226.75.32/28", "20.226.75.240/28", "20.226.83.160/28", "20.226.99.240/28", "20.226.104.160/28", "20.226.109.32/28", "20.226.110.0/28", "20.226.110.32/28", "20.226.110.80/28", "20.226.111.144/28", "20.226.113.128/28", "20.226.116.112/28", "20.226.117.96/28", "20.226.117.112/28", "20.226.122.16/28", "20.226.248.64/28", "20.226.249.96/28", "20.226.251.128/28", "20.226.251.144/28", "20.226.252.96/28", "20.235.77.240/28", "20.235.123.224/28", "20.248.179.176/28", "20.249.62.0/28", "20.254.200.112/28", "23.98.143.16/28", "23.100.232.0/28", "23.100.232.16/28", "23.100.232.32/28", "23.100.232.48/28", "23.100.232.64/28", "23.100.237.176/28", "23.100.238.160/28", "23.100.239.32/28", "40.67.183.96/28", "40.67.183.192/28", "40.74.225.32/28", "40.74.230.80/28", "40.74.233.0/28", "40.75.75.16/28", "40.75.132.112/28", "40.77.93.176/28", "40.78.162.32/28", "40.78.162.48/28", "40.78.171.176/28", "40.78.180.240/28", "40.80.211.128/28", "40.81.37.96/28", "40.82.193.64/28", "40.82.193.80/28", "40.84.222.0/28", "40.116.67.64/28", "40.116.94.128/28", "40.119.35.0/28", "40.119.35.208/28", "40.119.36.224/28", "40.122.245.16/28", "40.124.25.48/28", "40.124.77.176/28", "40.124.101.64/28", "40.124.101.128/28", "40.124.101.176/28", "40.124.161.48/28", "40.124.161.192/28", "40.124.216.16/28", "40.124.224.64/28", "40.124.225.0/28", "40.124.225.80/28", "40.124.232.96/28", "40.124.232.112/28", "40.124.248.192/28", "48.193.64.16/28", "48.193.64.32/28", "48.211.213.80/28", "48.214.149.224/28", "48.214.149.240/28", "48.214.150.80/28", "48.214.150.96/28", "48.217.163.128/28", "48.217.163.160/28", "48.217.163.208/28", "48.217.163.224/28", "48.217.164.32/28", "48.217.164.48/28", "48.217.164.64/28", "48.217.164.80/28", "48.217.164.144/28", "48.217.164.160/28", "48.217.164.176/28", "48.217.165.16/28", "48.217.165.32/28", "48.217.165.64/28", "48.217.165.96/28", "48.217.165.112/28", "48.221.153.80/28", "48.221.153.112/28", "48.221.153.128/28", "51.56.33.0/28", "51.56.48.240/28", "51.57.0.64/28", "51.57.9.48/28", "51.57.33.80/28", "51.59.24.224/28", "51.59.32.192/28", "51.59.32.208/28", "51.59.32.224/28", "51.59.32.240/28", "51.59.40.208/28", "52.141.71.0/28", "52.152.110.240/28", "52.153.227.64/28", "52.153.229.48/28", "52.154.17.48/28", "52.154.130.64/28", "52.155.34.160/28", "52.155.35.96/28", "52.155.36.48/28", "52.155.40.0/28", "52.155.41.144/28", "52.155.43.80/28", "52.155.58.128/28", "52.157.34.0/28", "52.157.34.16/28", "52.157.34.112/28", "52.157.34.144/28", "52.157.34.160/28", "52.159.141.80/28", "52.159.162.0/28", "52.159.255.0/28", "52.160.162.0/28", "52.160.164.112/28", "52.160.167.176/28", "52.160.197.128/28", "52.160.197.208/28", "52.160.225.0/28", "52.160.225.16/28", "52.160.225.64/28", "52.160.225.96/28", "52.160.225.112/28", "52.160.225.144/28", "52.160.232.64/28", "52.160.232.80/28", "52.160.232.144/28", "52.161.2.64/28", "52.161.70.80/28", "52.161.76.32/28", "52.161.209.0/28", "52.161.209.16/28", "52.161.209.32/28", "52.161.209.48/28", "52.161.209.64/28", "52.161.209.80/28", "52.161.209.128/28", "52.161.209.144/28", "52.161.210.0/28", "52.161.210.16/28", "52.161.210.32/28", "52.161.210.48/28", "52.161.210.64/28", "52.161.210.80/28", "52.161.210.96/28", "52.161.210.112/28", "52.161.210.128/28", "52.161.210.144/28", "52.161.210.160/28", "52.161.210.176/28", "52.161.210.192/28", "52.161.210.208/28", "52.161.210.224/28", "52.161.210.240/28", "52.161.211.0/28", "52.161.211.16/28", "52.161.211.32/28", "52.161.214.16/28", "52.161.214.32/28", "52.161.214.48/28", "52.161.214.128/28", "52.161.214.144/28", "52.161.214.160/28", "52.161.214.176/28", "52.161.214.192/28", "52.161.214.208/28", "52.161.214.224/28", "52.161.214.240/28", "52.161.215.128/28", "52.161.215.144/28", "52.161.215.176/28", "52.161.215.208/28", "52.162.4.208/28", "52.162.81.48/28", "52.162.83.0/28", "52.165.58.128/28", "52.165.58.144/28", "52.165.61.112/28", "52.165.61.128/28", "52.165.61.144/28", "52.165.61.160/28", "52.165.61.192/28", "52.165.61.208/28", "52.165.61.224/28", "52.165.61.240/28", "52.165.100.224/28", "52.165.101.176/28", "52.165.211.96/28", "52.165.211.192/28", "52.165.212.16/28", "52.165.212.32/28", "52.165.212.64/28", "52.165.212.112/28", "52.165.213.16/28", "52.165.213.192/28", "52.172.135.208/28", "52.173.10.160/28", "52.173.11.176/28", "52.173.11.192/28", "52.173.11.224/28", "52.173.13.128/28", "52.173.109.224/28", "52.173.122.112/28", "52.173.124.0/28", "52.173.124.32/28", "52.173.124.48/28", "52.173.124.80/28", "52.173.124.96/28", "52.173.124.144/28", "52.173.124.160/28", "52.173.124.208/28", "52.173.124.240/28", "52.173.125.16/28", "52.173.125.64/28", "52.173.125.80/28", "52.173.125.112/28", "52.173.125.128/28", "52.173.125.144/28", "52.173.125.176/28", "52.173.125.192/28", "52.173.125.208/28", "52.173.125.224/28", "52.173.126.176/28", "52.173.161.16/28", "52.173.161.144/28", "52.173.164.0/28", "52.173.164.48/28", "52.173.164.160/28", "52.173.164.176/28", "52.173.164.208/28", "52.173.164.224/28", "52.173.164.240/28", "52.173.178.48/28", "52.173.178.64/28", "52.173.178.80/28", "52.173.181.0/28", "52.173.218.128/28", "52.173.234.144/28", "52.176.18.16/28", "52.176.18.128/28", "52.176.18.144/28", "52.176.21.144/28", "52.176.33.80/28", "52.176.33.176/28", "52.176.35.16/28", "52.176.35.128/28", "52.176.35.144/28", "52.176.35.160/28", "52.176.35.176/28", "52.176.36.32/28", "52.176.36.96/28", "52.176.36.176/28", "52.176.37.16/28", "52.176.38.96/28", "52.176.38.112/28", "52.176.67.224/28", "52.176.69.32/28", "52.176.69.144/28", "52.176.120.224/28", "52.176.123.64/28", "52.176.124.32/28", "52.176.124.48/28", "52.176.124.64/28", "52.176.124.128/28", "52.176.124.160/28", "52.176.124.240/28", "52.176.125.64/28", "52.176.137.32/28", "52.176.137.96/28", "52.176.137.112/28", "52.176.137.144/28", "52.176.137.192/28", "52.176.138.16/28", "52.176.139.112/28", "52.176.139.144/28", "52.176.139.160/28", "52.176.139.192/28", "52.176.139.240/28", "52.176.140.112/28", "52.176.140.160/28", "52.176.140.176/28", "52.176.141.0/28", "52.176.141.48/28", "52.176.141.64/28", "52.176.141.112/28", "52.176.141.224/28", "52.176.141.240/28", "52.180.148.112/28", "52.180.148.192/28", "52.183.217.160/28", "52.187.244.192/28", "52.187.252.128/28", "52.190.138.128/28", "52.190.139.96/28", "52.190.141.128/28", "52.190.141.144/28", "52.225.35.128/28", "52.225.35.240/28", "52.225.36.16/28", "52.225.73.160/28", "52.225.96.96/28", "52.225.97.160/28", "52.225.97.176/28", "52.225.104.32/28", "52.225.104.48/28", "52.225.104.96/28", "52.225.104.112/28", "52.225.112.48/28", "52.225.112.64/28", "52.225.112.80/28", "52.225.113.96/28", "52.225.113.112/28", "52.230.165.0/28", "52.230.179.16/28", "52.230.179.32/28", "52.230.179.48/28", "52.234.2.0/28", "52.234.2.192/28", "52.234.37.192/28", "52.234.37.240/28", "52.234.40.160/28", "52.234.107.208/28", "52.234.110.224/28", "52.238.197.128/28", "52.240.171.48/28", "52.240.171.64/28", "52.240.171.80/28", "52.240.171.96/28", "52.240.171.112/28", "52.240.171.128/28", "52.240.171.144/28", "52.240.171.160/28", "52.240.171.176/28", "52.240.172.16/28", "52.240.172.32/28", "52.240.172.48/28", "52.240.172.64/28", "52.240.179.0/28", "52.240.179.16/28", "52.240.179.32/28", "52.240.179.48/28", "52.240.179.64/28", "52.240.179.80/28", "52.240.179.96/28", "52.240.179.112/28", "52.240.179.128/28", "52.240.180.80/28", "52.240.180.96/28", "52.240.180.112/28", "52.240.190.144/28", "52.240.190.160/28", "52.240.190.176/28", "52.240.190.192/28", "52.240.190.208/28", "52.240.190.224/28", "52.240.190.240/28", "52.240.191.0/28", "52.240.191.16/28", "52.240.191.32/28", "52.240.191.160/28", "52.240.208.112/28", "52.241.148.32/28", "52.241.148.48/28", "52.242.133.176/28", "52.242.197.32/28", "52.242.197.80/28", "52.242.197.192/28", "52.242.217.96/28", "52.242.218.96/28", "52.242.218.128/28", "52.242.218.144/28", "52.242.218.160/28", "52.242.218.176/28", "52.242.220.64/28", "52.242.220.80/28", "52.242.220.128/28", "52.242.220.176/28", "52.242.220.192/28", "52.242.243.144/28", "52.242.243.176/28", "52.242.243.208/28", "52.242.243.240/28", "52.242.244.128/28", "52.242.244.144/28", "52.242.244.160/28", "52.242.244.176/28", "52.242.245.32/28", "52.242.245.80/28", "52.248.41.48/28", "52.250.240.240/28", "52.250.243.224/28", "52.250.243.240/28", "52.250.244.0/28", "52.250.244.16/28", "52.250.248.224/28", "52.250.248.240/28", "52.250.249.0/28", "52.250.249.16/28", "52.254.129.128/28", "52.254.129.144/28", "52.254.129.160/28", "57.154.1.0/28", "57.154.1.96/28", "57.154.1.112/28", "57.154.1.192/28", "57.154.1.240/28", "57.154.2.0/28", "57.154.2.32/28", "57.154.2.48/28", "57.154.2.64/28", "57.154.2.80/28", "57.154.2.96/28", "57.154.2.112/28", "57.154.6.144/28", "57.154.6.176/28", "57.154.7.0/28", "57.154.7.16/28", "57.154.8.224/28", "57.154.8.240/28", "57.154.9.80/28", "57.154.9.144/28", "57.154.9.176/28", "57.154.9.224/28", "57.154.9.240/28", "57.154.10.16/28", "57.154.14.192/28", "57.154.15.32/28", "57.154.20.160/28", "57.154.22.112/28", "57.154.23.0/28", "57.154.24.96/28", "57.154.26.48/28", "57.154.26.128/28", "57.154.26.176/28", "57.154.27.0/28", "57.154.33.144/28", "57.154.35.160/28", "57.154.37.240/28", "64.236.255.32/28", "68.218.43.240/28", "68.218.77.224/28", "68.218.108.240/28", "70.37.53.32/28", "70.37.67.0/28", "70.37.74.208/28", "70.37.76.208/28", "70.153.32.48/28", "74.248.16.240/28", "74.248.21.96/28", "74.248.21.112/28", "74.248.21.176/28", "74.248.21.224/28", "74.248.21.240/28", "74.248.22.0/28", "74.248.24.32/28", "74.248.25.128/28", "74.248.25.144/28", "74.248.25.208/28", "74.248.25.224/28", "74.248.26.144/28", "74.248.26.176/28", "74.248.26.192/28", "74.248.26.208/28", "74.248.26.240/28", "74.248.27.0/28", "74.248.27.96/28", "74.248.27.112/28", "74.248.27.192/28", "74.248.27.208/28", "74.248.28.32/28", "74.248.28.48/28", "74.248.28.96/28", "74.248.38.112/28", "74.248.38.128/28", "74.248.38.144/28", "74.248.38.160/28", "74.248.38.176/28", "74.248.39.0/28", "74.248.39.16/28", "74.248.39.32/28", "74.248.39.48/28", "74.248.39.64/28", "74.248.99.96/28", "74.248.99.112/28", "74.248.112.208/28", "74.248.115.64/28", "74.248.115.128/28", "74.248.116.176/28", "74.248.121.80/28", "74.248.121.128/28", "74.248.121.144/28", "74.248.122.0/28", "74.248.122.128/28", "74.248.122.192/28", "74.248.122.208/28", "74.248.123.32/28", "74.248.134.112/28", "74.248.134.160/28", "74.248.135.64/28", "74.248.143.112/28", "74.248.158.208/28", "74.248.159.96/28", "74.249.134.160/28", "74.249.134.176/28", "74.249.134.192/28", "74.249.134.208/28", "74.249.190.224/28", "74.249.190.240/28", "74.249.191.0/28", "74.249.191.16/28", "74.249.191.32/28", "74.249.191.240/28", "104.209.10.32/28", "104.210.74.96/28", "104.210.78.80/28", "104.211.73.128/28", "104.215.112.96/28", "104.215.122.224/28", "128.24.248.176/28", "128.203.187.112/28", "128.203.197.192/28", "130.131.15.80/28", "130.131.39.48/28", "130.131.88.64/28", "130.131.88.80/28", "130.131.88.96/28", "130.131.88.112/28", "130.131.88.128/28", "130.131.88.144/28", "130.131.88.160/28", "130.131.88.176/28", "130.131.88.192/28", "130.131.88.208/28", "130.131.90.0/28", "130.131.90.16/28", "130.131.90.32/28", "130.131.96.240/28", "130.131.97.0/28", "130.131.97.16/28", "130.131.97.32/28", "130.131.97.80/28", "130.131.98.32/28", "130.131.98.80/28", "130.131.98.96/28", "130.131.98.112/28", "130.131.104.48/28", "130.131.104.208/28", "130.131.104.224/28", "130.131.104.240/28", "130.131.105.0/28", "130.131.105.16/28", "130.131.105.32/28", "130.131.105.48/28", "130.131.105.64/28", "130.131.105.80/28", "130.131.105.96/28", "130.131.105.112/28", "130.131.105.128/28", "130.131.105.144/28", "130.131.106.208/28", "130.131.107.0/28", "130.131.107.16/28", "130.131.107.32/28", "130.131.107.64/28", "130.131.112.16/28", "130.131.112.32/28", "130.131.112.48/28", "130.131.112.64/28", "130.131.112.80/28", "130.131.112.96/28", "130.131.114.0/28", "130.131.114.80/28", "130.131.195.160/28", "130.131.195.176/28", "130.131.204.16/28", "130.131.204.32/28", "130.131.204.48/28", "130.131.204.64/28", "130.131.230.0/28", "130.131.238.160/28", "130.131.238.176/28", "130.131.239.0/28", "130.131.245.112/28", "130.131.245.128/28", "130.213.40.240/28", "130.213.41.0/28", "130.213.41.16/28", "130.213.41.32/28", "130.213.41.48/28", "130.213.41.64/28", "130.213.41.80/28", "130.213.41.96/28", "130.213.41.112/28", "130.213.42.0/28", "130.213.42.64/28", "130.213.42.112/28", "130.213.81.64/28", "130.213.81.80/28", "130.213.81.112/28", "130.213.81.128/28", "130.213.81.144/28", "130.213.81.160/28", "130.213.89.48/28", "130.213.89.160/28", "130.213.89.176/28", "130.213.89.192/28", "130.213.97.80/28", "130.213.97.96/28", "130.213.97.112/28", "130.213.97.128/28", "130.213.104.192/28", "130.213.104.208/28", "130.213.104.224/28", "132.196.18.144/28", "132.196.19.80/28", "132.196.20.128/28", "132.196.20.224/28", "132.196.20.240/28", "132.196.21.0/28", "132.196.21.16/28", "132.196.21.48/28", "132.196.21.64/28", "132.196.21.96/28", "132.196.21.144/28", "132.196.21.160/28", "132.196.21.192/28", "132.196.21.208/28", "132.196.21.224/28", "132.196.21.240/28", "132.196.22.0/28", "132.196.22.64/28", "132.196.22.96/28", "132.196.22.112/28", "132.196.22.144/28", "132.196.22.192/28", "132.196.23.16/28", "132.196.23.32/28", "132.196.23.48/28", "132.196.23.96/28", "132.196.23.128/28", "132.196.23.160/28", "132.196.23.176/28", "132.196.23.224/28", "132.196.29.128/28", "132.196.29.144/28", "132.196.32.96/28", "132.196.32.128/28", "132.196.32.144/28", "132.196.32.160/28", "132.196.32.192/28", "132.196.32.208/28", "132.196.32.224/28", "132.196.32.240/28", "132.196.33.0/28", "132.196.33.16/28", "132.196.37.96/28", "132.196.37.112/28", "132.196.37.128/28", "132.196.55.32/28", "132.196.63.64/28", "132.196.85.64/28", "132.196.85.80/28", "132.196.85.96/28", "132.196.85.112/28", "132.196.95.48/28", "132.196.95.96/28", "132.196.101.96/28", "132.196.103.96/28", "132.196.103.128/28", "134.33.68.176/28", "134.33.69.0/28", "134.33.69.16/28", "134.33.69.32/28", "134.33.69.48/28", "134.33.69.64/28", "134.33.69.112/28", "134.33.74.160/28", "134.33.120.112/28", "134.112.88.64/28", "134.112.88.80/28", "134.112.96.32/28", "134.112.104.48/28", "134.112.104.80/28", "134.112.112.80/28", "134.112.120.32/28", "134.112.120.48/28", "134.138.52.208/28", "134.138.52.240/28", "134.149.233.240/28", "135.119.22.160/28", "135.119.22.176/28", "135.119.77.112/28", "135.119.79.64/28", "135.119.95.80/28", "135.119.95.144/28", "135.119.95.160/28", "135.119.95.176/28", "135.119.98.96/28", "135.119.98.240/28", "135.220.233.240/28", "135.232.191.176/28", "135.232.221.160/28", "135.232.223.176/28", "135.232.223.192/28", "135.232.223.208/28", "135.232.223.224/28", "135.232.223.240/28", "135.232.238.240/28", "135.232.239.0/28", "135.232.239.16/28", "135.232.239.32/28", "135.232.239.48/28", "135.232.239.64/28", "135.232.239.80/28", "135.232.239.96/28", "135.232.239.112/28", "135.232.241.96/28", "135.232.245.240/28", "135.232.246.0/28", "135.232.246.16/28", "135.232.246.32/28", "135.232.246.48/28", "135.232.246.64/28", "135.232.246.80/28", "135.232.246.96/28", "135.232.246.112/28", "135.232.246.128/28", "135.232.246.144/28", "135.232.246.160/28", "135.232.246.176/28", "135.232.246.192/28", "135.232.246.208/28", "135.232.246.224/28", "135.232.246.240/28", "135.232.247.0/28", "135.232.247.16/28", "135.232.247.32/28", "135.232.247.48/28", "135.232.247.64/28", "145.132.176.192/28", "157.55.187.160/28", "172.169.251.80/28", "172.169.251.192/28", "172.169.251.208/28", "172.169.252.80/28", "172.169.254.48/28", "172.169.254.64/28", "172.170.10.80/28", "172.170.10.96/28", "172.170.10.112/28", "172.170.226.48/28", "172.170.242.80/28", "172.171.4.96/28", "172.171.13.80/28", "172.171.13.96/28", "172.171.13.112/28", "172.178.140.160/28", "172.182.214.48/28", "172.182.214.64/28", "172.182.214.80/28", "172.182.217.96/28", "172.182.217.144/28", "172.182.217.160/28", "172.182.217.176/28", "172.182.217.192/28", "172.182.217.208/28", "172.182.217.224/28", "172.182.233.32/28", "172.182.233.48/28", "172.182.233.64/28", "172.182.233.80/28", "172.182.233.96/28", "172.182.240.208/28", "172.182.240.224/28", "172.182.241.0/28", "172.182.241.16/28", "172.182.241.32/28", "172.182.245.32/28", "172.182.245.48/28", "172.182.251.224/28", "172.182.253.48/28", "172.182.253.64/28", "172.182.253.96/28", "172.182.253.112/28", "172.182.253.160/28", "172.182.253.176/28", "172.182.253.208/28", "172.182.254.16/28", "172.182.254.128/28", "172.182.254.192/28", "172.182.254.208/28", "172.182.255.160/28", "172.183.51.64/28", "172.183.52.64/28", "172.183.201.48/28", "172.183.201.64/28", "172.183.201.96/28", "172.184.135.192/28", "172.184.188.240/28", "172.184.239.96/28", "172.184.239.112/28", "172.184.247.240/28", "172.185.55.240/28", "172.185.216.176/28", "172.185.216.192/28", "172.202.15.0/28", "172.202.53.224/28", "172.202.53.240/28", "172.202.54.0/28", "172.202.54.16/28", "172.202.54.32/28", "172.202.54.64/28", "172.202.61.208/28", "172.202.79.96/28", "172.202.91.224/28", "172.202.102.0/28", "172.202.102.16/28", "172.202.116.32/28", "172.202.116.48/28", "172.202.116.64/28", "172.202.116.80/28", "172.202.116.96/28", "172.202.116.112/28", "172.202.116.128/28", "172.202.125.224/28", "172.202.125.240/28", "172.202.126.32/28", "172.202.126.48/28", "172.202.126.64/28", "172.202.126.80/28", "172.202.174.16/28", "172.203.181.80/28", "172.203.181.192/28", "172.203.181.240/28", "172.203.182.0/28", "172.203.182.80/28", "172.203.182.128/28", "172.203.182.144/28", "172.203.182.160/28", "172.203.182.176/28", "172.203.182.192/28", "172.203.182.208/28", "172.203.182.224/28", "172.203.183.32/28", "172.203.183.48/28", "172.203.183.64/28", "172.203.190.208/28", "172.203.205.160/28", "172.203.206.48/28", "172.203.206.96/28", "172.203.206.112/28", "172.203.206.128/28", "172.203.206.144/28", "172.203.206.192/28", "172.203.206.224/28", "172.203.206.240/28", "172.210.133.192/28", "172.210.134.96/28", "172.210.134.128/28", "172.210.134.208/28", "172.210.134.224/28", "172.210.134.240/28", "172.210.135.80/28", "172.210.135.96/28", "172.212.140.176/28", "172.212.145.48/28", "172.212.150.48/28", "172.213.3.208/28", "172.213.4.80/28", "172.213.4.96/28", "172.213.4.128/28", "172.213.4.144/28", "172.213.4.176/28", "172.213.4.192/28", "172.213.4.208/28", "172.213.4.240/28", "172.213.5.16/28", "172.213.5.32/28", "172.213.5.48/28", "172.213.5.64/28", "172.213.5.112/28", "172.213.5.128/28", "172.213.5.144/28", "172.213.5.176/28", "172.213.5.208/28", "172.213.5.224/28", "172.213.10.80/28", "172.213.19.224/28", "172.214.42.224/28", "172.214.159.80/28", "172.214.159.96/28", "172.214.159.128/28", "172.214.159.144/28", "172.214.159.160/28", "172.214.159.176/28", "172.214.159.192/28", "172.214.159.208/28", "172.214.159.224/28", "172.214.159.240/28", "191.232.236.80/28", "191.233.137.192/28", "191.233.142.144/28", "191.233.193.128/28", "191.233.194.16/28", "191.233.194.80/28", "191.233.194.240/28", "191.233.198.80/28", "191.233.198.96/28", "191.233.198.208/28", "191.233.199.48/28", "191.234.160.96/28", "191.234.161.224/28", "191.234.162.16/28", "191.234.167.128/28", "191.234.168.192/28", "191.234.168.208/28", "191.234.168.224/28", "191.234.173.96/28", "191.235.97.128/28", "191.235.101.128/28", "191.235.105.0/28", "191.239.242.48/28", "191.239.245.16/28", "191.239.246.16/28", "191.239.248.80/28", "191.239.249.144/28" ], "note": "OpenAI publishes these as the \"ChatGPT agents\" ranges and lists them for Codex cloud, used for \"Connections from Codex cloud to services such as GitHub\". Not linked to an agent: OpenAI documents no user-agent token for Codex cloud, and the guide does not say whether ChatGPT agent's browser uses these ranges.", "sources": [ { "url": "https://developers.openai.com/api/docs/guides/ip-addresses", "title": "IP egress ranges", "fetched_at": "2026-10-28T17:15:33.800Z" } ] } ], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "ChatGPT-User", "coverage": "researched", "user_agent_tokens": [ "ChatGPT-User" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": "When users ask ChatGPT or a CustomGPT a question, it may visit a web page with a ChatGPT-User agent.", "search_engine": false, "robots_token": null, "robots_policy": "may_ignore", "robots_quote": "Because these actions are initiated by a user, robots.txt rules may not apply.", "ip_list_agent": "chatgpt-user", "signs": false, "note": "Also used for GPT Actions. OpenAI names only OAI-SearchBot and GPTBot as robots.txt tags, so robots_token is null. OpenAI says ChatGPT-User does not decide whether content appears in search; OAI-SearchBot does. OpenAI's IP egress guide lists GPT Actions under the separate chatgpt-connectors ranges, which are not linked here (see that list's note).", "sources": [ { "url": "https://developers.openai.com/api/docs/bots", "title": "Overview of OpenAI Crawlers", "fetched_at": "2026-10-28T17:15:31.412Z" } ] }, { "name": "GPTBot", "coverage": "researched", "user_agent_tokens": [ "GPTBot" ], "aliases": [], "purpose": "training", "purpose_quote": "It is used to crawl content that may be used in training our generative AI foundation models.", "search_engine": false, "robots_token": "GPTBot", "robots_policy": "honors", "robots_quote": "OpenAI uses OAI-SearchBot and GPTBot robots.txt tags to enable webmasters to manage how their sites and content work with AI.", "ip_list_agent": "gptbot", "signs": false, "note": "OpenAI frames a GPTBot disallow as a signal that content should not be used for training, and says that when a site allows both GPTBot and OAI-SearchBot it may use one crawl for both purposes. When fetching robots.txt the User-Agent may carry an extra \"robots.txt\" marker.", "sources": [ { "url": "https://developers.openai.com/api/docs/bots", "title": "Overview of OpenAI Crawlers", "fetched_at": "2026-10-28T17:15:31.412Z" } ] }, { "name": "OAI-AdsBot", "coverage": "researched", "user_agent_tokens": [ "OAI-AdsBot" ], "aliases": [], "purpose": "ads", "purpose_quote": "OAI-AdsBot is used to validate the safety of web pages submitted as ads on ChatGPT.", "search_engine": false, "robots_token": null, "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": "adsbot", "signs": false, "note": "OpenAI's page says nothing about robots.txt for OAI-AdsBot; its robots.txt sentence names only OAI-SearchBot and GPTBot, so robots_token is null. The page says it only visits landing pages submitted as ads and that its data is not used to train foundation models.", "sources": [ { "url": "https://developers.openai.com/api/docs/bots", "title": "Overview of OpenAI Crawlers", "fetched_at": "2026-10-28T17:15:31.412Z" } ] }, { "name": "OAI-SearchBot", "coverage": "researched", "user_agent_tokens": [ "OAI-SearchBot" ], "aliases": [], "purpose": "search", "purpose_quote": "OAI-SearchBot is used to surface websites in search results in ChatGPT’s search features.", "search_engine": false, "robots_token": "OAI-SearchBot", "robots_policy": "honors", "robots_quote": "OpenAI uses OAI-SearchBot and GPTBot robots.txt tags to enable webmasters to manage how their sites and content work with AI.", "ip_list_agent": "searchbot", "signs": false, "note": "The documented User-Agent is a Chrome-like string with the OAI-SearchBot token appended. When fetching robots.txt it may carry an extra \"robots.txt\" marker. OpenAI says robots.txt changes can take about 24 hours to apply to search.", "sources": [ { "url": "https://developers.openai.com/api/docs/bots", "title": "Overview of OpenAI Crawlers", "fetched_at": "2026-10-28T17:15:31.412Z" } ] }, { "name": "ChatGPT Agent", "coverage": "community", "user_agent_tokens": [ "ChatGPT Agent" ], "aliases": [], "purpose": "agent", "purpose_quote": null, "search_engine": false, "robots_token": "ChatGPT Agent", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Agents\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "Codex", "coverage": "community", "user_agent_tokens": [ "Codex" ], "aliases": [], "purpose": "agent", "purpose_quote": null, "search_engine": false, "robots_token": "Codex", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Its operator is named in its ai.robots.txt description. Function: \"AI Coding Agents\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "Dot", "coverage": "community", "user_agent_tokens": [ "Dot" ], "aliases": [], "purpose": "agent", "purpose_quote": null, "search_engine": false, "robots_token": "Dot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Its operator is named in its ai.robots.txt description. Function: \"AI Agents\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "OpenAI", "coverage": "community", "user_agent_tokens": [ "OpenAI" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "OpenAI", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Unclear at this time.\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "Operator", "coverage": "community", "user_agent_tokens": [ "Operator" ], "aliases": [], "purpose": "agent", "purpose_quote": null, "search_engine": false, "robots_token": "Operator", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Its operator is named in its ai.robots.txt description. Function: \"AI Agents\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://developers.openai.com/api/docs/bots", "title": "Overview of OpenAI Crawlers", "fetched_at": "2026-10-28T17:15:31.412Z" }, { "url": "https://blog.castle.io/how-to-authenticate-openai-operator-requests-using-http-message-signatures/", "title": "How to authenticate OpenAI Operator requests using HTTP message signatures (Castle, third party)", "fetched_at": "2026-10-28T17:15:32.581Z" }, { "url": "https://developers.openai.com/api/docs/guides/ip-addresses", "title": "IP egress ranges", "fetched_at": "2026-10-28T17:15:33.800Z" } ] }, { "slug": "panscient", "name": "Panscient", "website": "https://panscient.com", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Panscient", "coverage": "community", "user_agent_tokens": [ "Panscient" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "Panscient", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Data collection and analysis using machine learning and AI.\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "panscient.com", "coverage": "community", "user_agent_tokens": [ "panscient.com" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "panscient.com", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Data collection and analysis using machine learning and AI.\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "parallel", "name": "Parallel", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt lists the operator as unclear; it is named in the agent's own description there.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Shap-User", "coverage": "community", "user_agent_tokens": [ "Shap-User" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": null, "search_engine": false, "robots_token": "Shap-User", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Its operator is named in its ai.robots.txt description. Function: \"AI Assistants\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "ShapBot", "coverage": "community", "user_agent_tokens": [ "ShapBot" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "ShapBot", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Data Providers\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "payhawk", "name": "Payhawk", "website": null, "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Payhawk Invoice Fetching Agent", "coverage": "community", "user_agent_tokens": [ "Payhawk Invoice Fetching Agent" ], "aliases": [], "purpose": "agent", "purpose_quote": null, "search_engine": false, "robots_token": "Payhawk Invoice Fetching Agent", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Agents\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "perplexity", "name": "Perplexity", "website": "https://www.perplexity.ai/", "docs_url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers", "coverage": "researched", "note": null, "verification": { "signing": { "status": "none_found", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": "Perplexity's crawler page does not mention request signing, and no third-party report was found. On 2026-10-26 /.well-known/http-message-signatures-directory returned 404 on www.perplexity.ai (perplexity.ai and perplexity.com redirect there). No key directory found in the operator's documentation.", "sources": [] }, "ip_lists": [ { "agent": "perplexitybot", "url": "https://www.perplexity.com/perplexitybot.json", "format": "ipranges-json", "prefix_count": 8, "fetched_at": "2026-10-28T17:15:33.900Z", "sha256": "2688b50802b42611b412f8ac3bc614248d0eab00d74f4ceda41cb78526bde6b2", "prefixes": [ "3.211.124.183/32", "3.222.232.239/32", "3.224.62.45/32", "3.231.139.107/32", "18.97.1.228/30", "18.97.9.96/29", "18.210.92.235/32", "107.20.236.150/32" ], "note": null, "sources": [ { "url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers", "title": "Perplexity Crawlers", "fetched_at": "2026-10-28T17:15:37.239Z" } ] }, { "agent": "perplexity-user", "url": "https://www.perplexity.com/perplexity-user.json", "format": "ipranges-json", "prefix_count": 4, "fetched_at": "2026-10-28T17:15:36.257Z", "sha256": "23e55350c44ac51a8a6e7ed786d861db8bc62886fd1cfc228dfa467127223e48", "prefixes": [ "18.97.21.0/30", "18.97.43.80/29", "34.193.163.52/32", "44.208.221.197/32" ], "note": null, "sources": [ { "url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers", "title": "Perplexity Crawlers", "fetched_at": "2026-10-28T17:15:37.239Z" } ] } ], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Perplexity-User", "coverage": "researched", "user_agent_tokens": [ "Perplexity-User" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": "When users ask Perplexity a question, it might visit a web page to help provide an accurate answer and include a link to the page in its response.", "search_engine": false, "robots_token": "Perplexity-User", "robots_policy": "may_ignore", "robots_quote": "Since a user requested the fetch, this fetcher generally ignores robots.txt rules.", "ip_list_agent": "perplexity-user", "signs": false, "note": "The same page also lists Perplexity-User among its robots.txt tags and says it \"controls which sites these user requests can access\", but states that the fetcher generally ignores robots.txt rules.", "sources": [ { "url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers", "title": "Perplexity Crawlers", "fetched_at": "2026-10-28T17:15:37.239Z" } ] }, { "name": "PerplexityBot", "coverage": "researched", "user_agent_tokens": [ "PerplexityBot" ], "aliases": [], "purpose": "search", "purpose_quote": "PerplexityBot is designed to surface and link websites in search results on Perplexity.", "search_engine": false, "robots_token": "PerplexityBot", "robots_policy": "honors", "robots_quote": "Webmasters can use the following robots.txt tags to manage how their sites and content interact with Perplexity.", "ip_list_agent": "perplexitybot", "signs": false, "note": "Perplexity says PerplexityBot is not used to crawl content for AI foundation models, and that robots.txt changes may take up to 24 hours to apply.", "sources": [ { "url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers", "title": "Perplexity Crawlers", "fetched_at": "2026-10-28T17:15:37.239Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers", "title": "Perplexity Crawlers", "fetched_at": "2026-10-28T17:15:37.239Z" } ] }, { "slug": "phind", "name": "phind", "website": "https://www.phind.com/", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "PhindBot", "coverage": "community", "user_agent_tokens": [ "PhindBot" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": null, "search_engine": false, "robots_token": "PhindBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Assistants\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "poggio", "name": "Poggio", "website": null, "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Poggio-Citations", "coverage": "community", "user_agent_tokens": [ "Poggio-Citations" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": null, "search_engine": false, "robots_token": "Poggio-Citations", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Assistants\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "poseidon-research", "name": "Poseidon Research", "website": "https://www.poseidonresearch.com", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Poseidon Research Crawler", "coverage": "community", "user_agent_tokens": [ "Poseidon Research Crawler" ], "aliases": [], "purpose": "search", "purpose_quote": null, "search_engine": false, "robots_token": "Poseidon Research Crawler", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI research crawler\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "posterna", "name": "Posterna", "website": "https://posterna.com/", "docs_url": "https://github.com/posternahq/posterna/blob/main/docs/SPEC.md", "coverage": "researched", "note": "The specification is the documentation. The hosted probe at posterna.com signs its requests with the key in posterna.com's key directory; `posterna doctor` signs with a throwaway key that no directory lists.", "verification": { "signing": { "status": "live", "directory_url": "https://posterna.com/.well-known/http-message-signatures-directory", "kid_convention": "thumbprint", "purpose_field": null, "covers": [ "PosternaProbe" ], "note": "Requests from the hosted probe are signed with this key. Requests from `posterna doctor` are signed with a throwaway key and do not verify against it.", "sources": [ { "url": "https://github.com/posternahq/posterna/blob/main/docs/SPEC.md", "title": "Posterna v1 specification", "fetched_at": "2026-10-28T17:15:39.583Z" }, { "url": "https://posterna.com/.well-known/http-message-signatures-directory", "title": "Key directory", "fetched_at": "2026-10-28T17:15:38.440Z" } ] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "PosternaDirectoryBuilder", "coverage": "researched", "user_agent_tokens": [ "PosternaDirectoryBuilder" ], "aliases": [], "purpose": "other", "purpose_quote": "downloads robots.json, every cited page, IP list and key directory", "search_engine": false, "robots_token": "PosternaDirectoryBuilder", "robots_policy": "honors", "robots_quote": "honors robots.txt (RFC 9309)", "ip_list_agent": null, "signs": false, "note": "Builds this directory once a day from GitHub Actions. It fetches only the pages, IP lists and key directories that operators publish and this directory cites, and does not sign its requests.", "sources": [ { "url": "https://github.com/posternahq/posterna/blob/main/docs/SPEC.md", "title": "Posterna v1 specification", "fetched_at": "2026-10-28T17:15:39.583Z" } ] }, { "name": "PosternaProbe", "coverage": "researched", "user_agent_tokens": [ "PosternaProbe" ], "aliases": [], "purpose": "other", "purpose_quote": "send a request signed with an ephemeral key to the site's debug route", "search_engine": false, "robots_token": null, "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": true, "note": "Sent by the hosted probe at posterna.com/probe, signed with the key in Posterna's key directory, and by `posterna doctor`, which a site owner runs against their own site with a throwaway key.", "sources": [ { "url": "https://github.com/posternahq/posterna/blob/main/docs/SPEC.md", "title": "Posterna v1 specification", "fetched_at": "2026-10-28T17:15:39.583Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://github.com/posternahq/posterna/blob/main/docs/SPEC.md", "title": "Posterna v1 specification", "fetched_at": "2026-10-28T17:15:39.583Z" } ] }, { "slug": "qoder", "name": "Qoder", "website": null, "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "qodercli", "coverage": "community", "user_agent_tokens": [ "qodercli" ], "aliases": [], "purpose": "agent", "purpose_quote": null, "search_engine": false, "robots_token": "qodercli", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Coding Agents\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "qualified", "name": "Qualified", "website": "https://www.qualified.com", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "QualifiedBot", "coverage": "community", "user_agent_tokens": [ "QualifiedBot" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": null, "search_engine": false, "robots_token": "QualifiedBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Assistants\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "quantumcloud", "name": "QuantumCloud", "website": "https://www.quantumcloud.com", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "wpbot", "coverage": "community", "user_agent_tokens": [ "wpbot" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "wpbot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Live chat support and lead generation.\". Respects robots.txt: \"Unclear at this time; opt out provided via Google Form\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "querit", "name": "Querit", "website": null, "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Querit-SearchBot", "coverage": "community", "user_agent_tokens": [ "Querit-SearchBot" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "Querit-SearchBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Data Providers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "QueritBot", "coverage": "community", "user_agent_tokens": [ "QueritBot" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "QueritBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Data Providers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "quillbot", "name": "Quillbot", "website": "https://quillbot.com", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "QuillBot", "coverage": "community", "user_agent_tokens": [ "QuillBot" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "QuillBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Company offers AI detection, writing tools and other services.\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "quillbot.com", "coverage": "community", "user_agent_tokens": [ "quillbot.com" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "quillbot.com", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Company offers AI detection, writing tools and other services.\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "reflection", "name": "Reflection", "website": "https://reflection.ai/", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Reflectionbot", "coverage": "community", "user_agent_tokens": [ "Reflectionbot" ], "aliases": [], "purpose": "agent", "purpose_quote": null, "search_engine": false, "robots_token": "Reflectionbot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Undocumented AI Agents\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "rois", "name": "ROIS", "website": null, "docs_url": "https://ds.rois.ac.jp/en_center8/en_crawler/", "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Cotoyogi", "coverage": "community", "user_agent_tokens": [ "Cotoyogi" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "Cotoyogi", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI LLM Scraper.\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "sb-intuitions", "name": "SB Intuitions", "website": null, "docs_url": "https://www.sbintuitions.co.jp/en/", "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "SBIntuitionsBot", "coverage": "community", "user_agent_tokens": [ "SBIntuitionsBot" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "SBIntuitionsBot", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Uses data gathered in AI development and information analysis.\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "semrush", "name": "Semrush", "website": "https://www.semrush.com/", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "SemrushBot-OCOB", "coverage": "community", "user_agent_tokens": [ "SemrushBot-OCOB" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "SemrushBot-OCOB", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Crawls your site for ContentShake AI tool.\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "SemrushBot-SWA", "coverage": "community", "user_agent_tokens": [ "SemrushBot-SWA" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": null, "search_engine": false, "robots_token": "SemrushBot-SWA", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Checks URLs on your site for SEO Writing Assistant.\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "sidetrade", "name": "Sidetrade", "website": "https://www.sidetrade.com", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Sidetrade indexer bot", "coverage": "community", "user_agent_tokens": [ "Sidetrade indexer bot" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "Sidetrade indexer bot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Extracts data for a variety of uses including training AI.\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "tavily", "name": "Tavily", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt lists the operator as unclear; it is named in the agent's own description there.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "TavilyBot", "coverage": "community", "user_agent_tokens": [ "TavilyBot" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "TavilyBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Its operator is named in its ai.robots.txt description. Function: \"AI Data Providers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "tencent", "name": "Tencent", "website": "https://hunyuan.tencent.com", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Hunyuan", "coverage": "community", "user_agent_tokens": [ "Hunyuan" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "Hunyuan", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Hunyuan is a training crawler operated by Tencent.\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "the-agent-times", "name": "The Agent Times", "website": null, "docs_url": "https://theagenttimes.com/about", "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "AgentTimes", "coverage": "community", "user_agent_tokens": [ "AgentTimes" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "AgentTimes", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Data Scraper from RSS Feeds.\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "thinkbot", "name": "Thinkbot", "website": "https://www.thinkbot.agency", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Thinkbot", "coverage": "community", "user_agent_tokens": [ "Thinkbot" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "Thinkbot", "robots_policy": "may_ignore", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Insights on AI integration and automation.\". Respects robots.txt: \"No\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "timpi", "name": "Timpi", "website": "https://timpi.io", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Timpibot", "coverage": "community", "user_agent_tokens": [ "Timpibot" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "Timpibot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Scrapes data for use in training LLMs.\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "twin", "name": "Twin", "website": null, "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "TwinAgent", "coverage": "community", "user_agent_tokens": [ "TwinAgent" ], "aliases": [], "purpose": "agent", "purpose_quote": null, "search_engine": false, "robots_token": "TwinAgent", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Agents\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "unknown-aranet-searchbot", "name": "Unknown operator (Aranet-SearchBot)", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt does not name the operator of this agent.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Aranet-SearchBot", "coverage": "community", "user_agent_tokens": [ "Aranet-SearchBot" ], "aliases": [], "purpose": "agent", "purpose_quote": null, "search_engine": false, "robots_token": "Aranet-SearchBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Undocumented AI Agents\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "unknown-azureai-searchbot", "name": "Unknown operator (AzureAI-SearchBot)", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt does not name the operator of this agent.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "AzureAI-SearchBot", "coverage": "community", "user_agent_tokens": [ "AzureAI-SearchBot" ], "aliases": [], "purpose": "search", "purpose_quote": null, "search_engine": false, "robots_token": "AzureAI-SearchBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Search Crawlers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "unknown-channel3bot", "name": "Unknown operator (Channel3Bot)", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt does not name the operator of this agent.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Channel3Bot", "coverage": "community", "user_agent_tokens": [ "Channel3Bot" ], "aliases": [], "purpose": "search", "purpose_quote": null, "search_engine": false, "robots_token": "Channel3Bot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Search Crawlers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "unknown-chatglm-spider", "name": "Unknown operator (ChatGLM-Spider)", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt does not name the operator of this agent.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "ChatGLM-Spider", "coverage": "community", "user_agent_tokens": [ "ChatGLM-Spider" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "ChatGLM-Spider", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Data Scrapers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "unknown-code", "name": "Unknown operator (Code)", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt does not name the operator of this agent.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Code", "coverage": "community", "user_agent_tokens": [ "Code" ], "aliases": [], "purpose": "agent", "purpose_quote": null, "search_engine": false, "robots_token": "Code", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Coding Agents\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "unknown-crawl4ai", "name": "Unknown operator (Crawl4AI)", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt does not name the operator of this agent.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Crawl4AI", "coverage": "community", "user_agent_tokens": [ "Crawl4AI" ], "aliases": [], "purpose": "agent", "purpose_quote": null, "search_engine": false, "robots_token": "Crawl4AI", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Undocumented AI Agents\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "unknown-cursor", "name": "Unknown operator (Cursor)", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt does not name the operator of this agent.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Cursor", "coverage": "community", "user_agent_tokens": [ "Cursor" ], "aliases": [], "purpose": "agent", "purpose_quote": null, "search_engine": false, "robots_token": "Cursor", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Coding Agents\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "unknown-friendlycrawler", "name": "Unknown operator (FriendlyCrawler)", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt does not name the operator of this agent.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "FriendlyCrawler", "coverage": "community", "user_agent_tokens": [ "FriendlyCrawler" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "FriendlyCrawler", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"We are using the data from the crawler to build datasets for machine learning experiments.\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "unknown-iaskbot", "name": "Unknown operator (iAskBot)", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt does not name the operator of this agent.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "iAskBot", "coverage": "community", "user_agent_tokens": [ "iAskBot" ], "aliases": [], "purpose": "agent", "purpose_quote": null, "search_engine": false, "robots_token": "iAskBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Undocumented AI Agents\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "unknown-imagespider", "name": "Unknown operator (imageSpider)", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt does not name the operator of this agent.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "imageSpider", "coverage": "community", "user_agent_tokens": [ "imageSpider" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "imageSpider", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Data Scrapers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "unknown-instinct", "name": "Unknown operator (Instinct)", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt does not name the operator of this agent.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Instinct", "coverage": "community", "user_agent_tokens": [ "Instinct" ], "aliases": [], "purpose": "agent", "purpose_quote": null, "search_engine": false, "robots_token": "Instinct", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Agents\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "unknown-kunatocrawler", "name": "Unknown operator (KunatoCrawler)", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt does not name the operator of this agent.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "KunatoCrawler", "coverage": "community", "user_agent_tokens": [ "KunatoCrawler" ], "aliases": [], "purpose": "agent", "purpose_quote": null, "search_engine": false, "robots_token": "KunatoCrawler", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Undocumented AI Agents\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "unknown-laion-huggingface-processor", "name": "Unknown operator (laion-huggingface-processor)", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt does not name the operator of this agent.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "laion-huggingface-processor", "coverage": "community", "user_agent_tokens": [ "laion-huggingface-processor" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "laion-huggingface-processor", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Data Scrapers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "unknown-lcc", "name": "Unknown operator (LCC)", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt does not name the operator of this agent.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "LCC", "coverage": "community", "user_agent_tokens": [ "LCC" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "LCC", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Data Scrapers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "unknown-linkupbot", "name": "Unknown operator (LinkupBot)", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt does not name the operator of this agent.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "LinkupBot", "coverage": "community", "user_agent_tokens": [ "LinkupBot" ], "aliases": [], "purpose": "search", "purpose_quote": null, "search_engine": false, "robots_token": "LinkupBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Search Crawlers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "unknown-mycentralaiscraperbot", "name": "Unknown operator (MyCentralAIScraperBot)", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt does not name the operator of this agent.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "MyCentralAIScraperBot", "coverage": "community", "user_agent_tokens": [ "MyCentralAIScraperBot" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "MyCentralAIScraperBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI data scraper\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "unknown-newsai", "name": "Unknown operator (newsai)", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt does not name the operator of this agent.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "newsai", "coverage": "community", "user_agent_tokens": [ "newsai" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "newsai", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI data scraper\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "unknown-opencode", "name": "Unknown operator (opencode)", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt does not name the operator of this agent.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "opencode", "coverage": "community", "user_agent_tokens": [ "opencode" ], "aliases": [], "purpose": "agent", "purpose_quote": null, "search_engine": false, "robots_token": "opencode", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Coding Agents\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "unknown-spider", "name": "Unknown operator (Spider)", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt does not name the operator of this agent.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Spider", "coverage": "community", "user_agent_tokens": [ "Spider" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "Spider", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Data Scrapers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "unknown-useai", "name": "Unknown operator (UseAI)", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt does not name the operator of this agent.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "UseAI", "coverage": "community", "user_agent_tokens": [ "UseAI" ], "aliases": [], "purpose": "user_fetch", "purpose_quote": null, "search_engine": false, "robots_token": "UseAI", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Assistants\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "unknown-wrtnbot", "name": "Unknown operator (WRTNBot)", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt does not name the operator of this agent.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "WRTNBot", "coverage": "community", "user_agent_tokens": [ "WRTNBot" ], "aliases": [], "purpose": "agent", "purpose_quote": null, "search_engine": false, "robots_token": "WRTNBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Undocumented AI Agents\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "unknown-zanistabot", "name": "Unknown operator (ZanistaBot)", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt does not name the operator of this agent.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "ZanistaBot", "coverage": "community", "user_agent_tokens": [ "ZanistaBot" ], "aliases": [], "purpose": "search", "purpose_quote": null, "search_engine": false, "robots_token": "ZanistaBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Search Crawlers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "valyu", "name": "Valyu", "website": null, "docs_url": null, "coverage": "community", "note": "ai.robots.txt lists the operator as unclear; it is named in the agent's own description there.", "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "HenkBot", "coverage": "community", "user_agent_tokens": [ "HenkBot" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "HenkBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Its operator is named in its ai.robots.txt description. Function: \"AI Data Providers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "velen-crawler", "name": "Velen Crawler", "website": "https://velen.io", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "VelenPublicWebCrawler", "coverage": "community", "user_agent_tokens": [ "VelenPublicWebCrawler" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "VelenPublicWebCrawler", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Scrapes data for business data sets and machine learning models.\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "webspark", "name": "WEBSPARK", "website": null, "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "WARDBot", "coverage": "community", "user_agent_tokens": [ "WARDBot" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "WARDBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Data Scrapers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "webz-io", "name": "Webz.io", "website": "https://webz.io/", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "omgili", "coverage": "community", "user_agent_tokens": [ "omgili" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "omgili", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Data is sold.\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "omgilibot", "coverage": "community", "user_agent_tokens": [ "omgilibot" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "omgilibot", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Data is sold.\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "Webzio-Extended", "coverage": "community", "user_agent_tokens": [ "Webzio-Extended" ], "aliases": [ "webzio-extended" ], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "Webzio-Extended", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Its operator is named in its ai.robots.txt description. Function: \"AI Data Scrapers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "xai", "name": "xAI", "website": "https://x.ai/", "docs_url": "https://docs.x.ai/grok-bot/overview", "coverage": "researched", "note": null, "verification": { "signing": { "status": "none_found", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": "No xAI page mentions Web Bot Auth or HTTP message signatures. On 2026-10-26 /.well-known/http-message-signatures-directory returned 404 on x.ai, grok.com, docs.x.ai, api.x.ai and accounts.x.ai; on www.x.ai it redirects to the x.ai home page. No third-party report of an xAI key directory was found. No key directory found in the operator's documentation.", "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Grok API web search", "coverage": "researched", "user_agent_tokens": [], "aliases": [], "purpose": "user_fetch", "purpose_quote": "The Web Search tool enables Grok to search the web in real-time and browse web pages to find information.", "search_engine": false, "robots_token": null, "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "The server-side web_search tool of the xAI API, run when an API caller enables it. The docs do not say what User-Agent, IP addresses or robots.txt policy its page fetches use, so its requests cannot be told apart from other traffic. How the consumer Grok apps (grok.com, iOS, Android) fetch pages is not documented either.", "sources": [ { "url": "https://docs.x.ai/developers/tools/web-search", "title": "Web Search", "fetched_at": "2026-10-28T17:15:43.575Z" } ] }, { "name": "Grok Bot", "coverage": "researched", "user_agent_tokens": [], "aliases": [], "purpose": "agent", "purpose_quote": "Each Bot works on a persistent cloud computer with a browser, filesystem, and terminal, so tasks finish in your real tools instead of as chat drafts.", "search_engine": false, "robots_token": null, "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "xAI's docs give no User-Agent string or token for Grok Bot's browser and say nothing about robots.txt. On sites that resist automation the FAQ says \"The Bot should hand those steps to you rather than bypassing them.\" (source grok-bot-faq). Hosted computers use shared static egress ranges, but these are not published: \"Current ranges are available from your account team.\" (source grok-bot-security-faq). Members can also route a computer's traffic through their own desktop or network, which then shows that network's IP address.", "sources": [ { "url": "https://docs.x.ai/grok-bot/overview", "title": "Grok Bot", "fetched_at": "2026-10-28T17:15:40.310Z" } ] }, { "name": "grok-agent", "coverage": "researched", "user_agent_tokens": [ "grok-agent" ], "aliases": [ "web_fetch" ], "purpose": "user_fetch", "purpose_quote": "Fetch the content of a specific URL and return it as markdown.", "search_engine": false, "robots_token": null, "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "The web_fetch tool of the Grok Build coding CLI (xai-org/grok-build). Its source sets the User-Agent to \"Mozilla/5.0 (compatible; grok-agent/1.0; +https://x.ai)\" (USER_AGENT_STRING in config.rs, source grok-build-webfetch-config). The token comes from xAI's own source code, not from a docs page. The tool runs on the user's own machine (or an egress proxy the user sets), so there is no xAI IP range. It is off unless the user turns it on (GROK_WEB_FETCH, \"Off by default for security.\", source grok-build-settings) and by default only fetches a built-in allowlist of documentation domains. No robots.txt check appears in the web_fetch source files read on 2026-10-26, and no docs page states a robots.txt policy.", "sources": [ { "url": "https://raw.githubusercontent.com/xai-org/grok-build/main/crates/codegen/xai-grok-tools/src/implementations/grok_build/web_fetch/mod.rs", "title": "Grok Build web_fetch tool (source, xai-org/grok-build)", "fetched_at": "2026-10-28T17:15:45.817Z" }, { "url": "https://raw.githubusercontent.com/xai-org/grok-build/main/crates/codegen/xai-grok-tools/src/implementations/grok_build/web_fetch/config.rs", "title": "Grok Build web_fetch configuration (source, xai-org/grok-build)", "fetched_at": "2026-10-28T17:15:46.113Z" } ] }, { "name": "GrokAppAndroid", "coverage": "community", "user_agent_tokens": [ "GrokAppAndroid" ], "aliases": [], "purpose": "search", "purpose_quote": null, "search_engine": false, "robots_token": "GrokAppAndroid", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Search Crawlers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "GrokBot", "coverage": "community", "user_agent_tokens": [ "GrokBot" ], "aliases": [], "purpose": "search", "purpose_quote": null, "search_engine": false, "robots_token": "GrokBot", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"AI Search Crawlers\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "xAI-Grok", "coverage": "community", "user_agent_tokens": [ "xAI-Grok" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "xAI-Grok", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"xAI-Grok is an xAI crawler token associated with Grok content collection.\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://docs.x.ai/grok-bot/overview", "title": "Grok Bot", "fetched_at": "2026-10-28T17:15:40.310Z" }, { "url": "https://docs.x.ai/grok-bot/faq", "title": "Grok Bot FAQ", "fetched_at": "2026-10-28T17:15:41.547Z" }, { "url": "https://docs.x.ai/grok-bot/security-faq", "title": "Grok Bot security FAQ", "fetched_at": "2026-10-28T17:15:42.506Z" }, { "url": "https://docs.x.ai/developers/tools/web-search", "title": "Web Search", "fetched_at": "2026-10-28T17:15:43.575Z" }, { "url": "https://docs.x.ai/build/settings/reference", "title": "Grok Build settings reference", "fetched_at": "2026-10-28T17:15:44.638Z" }, { "url": "https://raw.githubusercontent.com/xai-org/grok-build/main/crates/codegen/xai-grok-tools/src/implementations/grok_build/web_fetch/mod.rs", "title": "Grok Build web_fetch tool (source, xai-org/grok-build)", "fetched_at": "2026-10-28T17:15:45.817Z" }, { "url": "https://raw.githubusercontent.com/xai-org/grok-build/main/crates/codegen/xai-grok-tools/src/implementations/grok_build/web_fetch/config.rs", "title": "Grok Build web_fetch configuration (source, xai-org/grok-build)", "fetched_at": "2026-10-28T17:15:46.113Z" } ] }, { "slug": "yandex", "name": "Yandex", "website": "https://yandex.ru", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "YandexAdditional", "coverage": "community", "user_agent_tokens": [ "YandexAdditional" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "YandexAdditional", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Scrapes/analyzes data for the YandexGPT LLM.\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "name": "YandexAdditionalBot", "coverage": "community", "user_agent_tokens": [ "YandexAdditionalBot" ], "aliases": [], "purpose": "other", "purpose_quote": null, "search_engine": false, "robots_token": "YandexAdditionalBot", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Scrapes/analyzes data for the YandexGPT LLM.\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "you", "name": "You", "website": null, "docs_url": "https://about.you.com/youchat/", "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "YouBot", "coverage": "community", "user_agent_tokens": [ "YouBot" ], "aliases": [], "purpose": "search", "purpose_quote": null, "search_engine": false, "robots_token": "YouBot", "robots_policy": "honors", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Scrapes data for search engine and LLMs.\". Respects robots.txt: \"Yes\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] }, { "slug": "zyte", "name": "Zyte", "website": "https://www.zyte.com", "docs_url": null, "coverage": "community", "note": null, "verification": { "signing": { "status": "not_probed", "directory_url": null, "kid_convention": null, "purpose_field": null, "covers": [], "note": null, "sources": [] }, "ip_lists": [], "reverse_dns": null, "asn": [] }, "agents": [ { "name": "Scrapy", "coverage": "community", "user_agent_tokens": [ "Scrapy" ], "aliases": [], "purpose": "training", "purpose_quote": null, "search_engine": false, "robots_token": "Scrapy", "robots_policy": "unstated", "robots_quote": null, "ip_list_agent": null, "signs": false, "note": "Community record from ai.robots.txt. Function: \"Scrapes data for a variety of uses including training AI.\". Respects robots.txt: \"Unclear at this time.\".", "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ], "observed": {}, "enrollment": { "status": "none" }, "identity_anchors": [], "sources": [ { "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json", "title": "ai.robots.txt robots.json", "fetched_at": "2026-10-28T17:13:56.511Z" } ] } ] }