{
  "name": "vantaweb.io AI crawler request log summary, 19-26 July 2026",
  "publisher": "VantaWeb",
  "author": "Hunter Spence",
  "license": "https://creativecommons.org/licenses/by/4.0/",
  "attribution": "VantaWeb, https://vantaweb.io/data/",
  "study": "https://vantaweb.io/blog/ai-crawler-traffic-audit/",
  "period": "2026-07-19/2026-07-26",
  "source": "Cloudflare GraphQL Analytics API, httpRequestsAdaptiveGroups, dimensions userAgent and clientRequestPath",
  "columns": [
    "table",
    "item",
    "metric",
    "value",
    "unit",
    "note"
  ],
  "rows": [
    {
      "table": "funnel",
      "item": "Requests carrying a crawler user agent",
      "metric": "requests",
      "value": "3727",
      "unit": "requests",
      "note": "User agent claimed to be an AI crawler or a search-engine crawler; vantaweb.io, 19-26 July 2026"
    },
    {
      "table": "funnel",
      "item": "Requests for unpublished paths",
      "metric": "requests",
      "value": "3109",
      "unit": "requests",
      "note": "Path is not among the 105 sitemap URLs or the infrastructure-file whitelist, so counted as spoofed"
    },
    {
      "table": "funnel",
      "item": "Requests for published paths",
      "metric": "requests",
      "value": "618",
      "unit": "requests",
      "note": "3,727 minus 3,109"
    },
    {
      "table": "funnel",
      "item": "Requests for unpublished paths",
      "metric": "share of crawler-user-agent requests",
      "value": "83.4",
      "unit": "percent",
      "note": "3,109 of 3,727"
    },
    {
      "table": "funnel",
      "item": "User-agent count vs real crawler activity",
      "metric": "overstatement factor",
      "value": "6.03",
      "unit": "ratio",
      "note": "3,727 divided by 618; AI and search crawlers combined"
    },
    {
      "table": "legitimate_fetches_per_day",
      "item": "ClaudeBot (Anthropic)",
      "metric": "fetches per day",
      "value": "43",
      "unit": "fetches/day",
      "note": "After removing requests for unpublished paths"
    },
    {
      "table": "legitimate_fetches_per_day",
      "item": "OAI-SearchBot (OpenAI)",
      "metric": "fetches per day",
      "value": "13",
      "unit": "fetches/day",
      "note": "After removing requests for unpublished paths"
    },
    {
      "table": "legitimate_fetches_per_day",
      "item": "ChatGPT-User (OpenAI)",
      "metric": "fetches per day",
      "value": "8",
      "unit": "fetches/day",
      "note": "After removing requests for unpublished paths"
    },
    {
      "table": "legitimate_fetches_per_day",
      "item": "Googlebot (Google)",
      "metric": "fetches per day",
      "value": "13",
      "unit": "fetches/day",
      "note": "After removing requests for unpublished paths"
    },
    {
      "table": "ai_vs_search",
      "item": "Legitimate AI crawler fetches",
      "metric": "fetches per day",
      "value": "64",
      "unit": "fetches/day",
      "note": "About 64 per day (ClaudeBot, OAI-SearchBot, ChatGPT-User)"
    },
    {
      "table": "ai_vs_search",
      "item": "Legitimate search crawler fetches",
      "metric": "fetches per day",
      "value": "13",
      "unit": "fetches/day",
      "note": "About 13 per day"
    },
    {
      "table": "ai_vs_search",
      "item": "Legitimate AI to search ratio",
      "metric": "ratio",
      "value": "4.9",
      "unit": "ratio to 1",
      "note": "Roughly 4.9 to 1 after cleaning"
    },
    {
      "table": "ai_vs_search",
      "item": "Raw AI to search ratio",
      "metric": "ratio",
      "value": "5.14",
      "unit": "ratio to 1",
      "note": "Before removing requests for unpublished paths, across a much larger volume"
    }
  ]
}