CDN 2025 SQL (#4300)

* Pushing 2024 queries into 2025

* Updated queries to pull from latest crawl and dataset.

* Fixed sanList array parsing for tls time.

* Adding client hint, early hint, and image format changes (#4295)

* Removed old early hints queries.

* SQLFluff fix

* Manual formatting

* More linting

---------

Co-authored-by: Joe Viggiano <jviggian@amazon.com>
Co-authored-by: Nick McCord <102171522+nick-mccord@users.noreply.github.com>
Co-authored-by: Barry Pollard <barrypollard@google.com>

Author: 4 people

sql/2025/cdn/cdn_usage_by_site_rank.sql

@@ -0,0 +1,38 @@
+#standardSQL
+# cdn_usage_by_site_rank.sql : Distribution of HTML pages served by CDN vs Origin by rank
+
+WITH requests AS (
+  SELECT
+    client,
+    resp.rank, -- Need to validate this should be resp.rank and not pages.rank
+    IF(IFNULL(NULLIF(REGEXP_EXTRACT(JSON_EXTRACT_SCALAR(resp.summary, '$._cdn_provider'), r'^([^,]*).*'), ''), '') = '', 'ORIGIN', 'CDN') AS cdn
+  FROM
+    `httparchive.crawl.requests` AS resp
+  INNER JOIN
+    `httparchive.crawl.pages`
+  USING (page, client, date)
+  WHERE
+    date = '2025-07-01' AND
+    is_main_document
+)
+
+SELECT
+  client,
+  nested_rank,
+  cdn,
+  COUNT(0) AS num_requests,
+  SUM(COUNT(0)) OVER (PARTITION BY client, nested_rank) AS total,
+  COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client, nested_rank) AS pct_requests
+FROM
+  requests,
+  UNNEST([1000, 10000, 100000, 1000000, 10000000, 100000000]) AS nested_rank
+WHERE
+  rank <= nested_rank
+GROUP BY
+  client,
+  cdn,
+  nested_rank
+ORDER BY
+  client,
+  nested_rank,
+  cdn

sql/2025/cdn/client_hints/baseline.sql

@@ -0,0 +1,103 @@
+#standardSQL
+-- Baseline Client Hints adoption metrics for context
+-- Measures both server-side (Accept-CH) and client-side (Sec-CH-*) adoption
+DECLARE d DATE DEFAULT '2025-07-01';
+
+WITH
+-- Total requests baseline
+total_requests AS (
+  SELECT
+    client,
+    COUNT(0) AS total_requests,
+    COUNT(DISTINCT page) AS total_pages
+  FROM
+    `httparchive.crawl.requests`
+  WHERE
+    date = d
+  GROUP BY
+    client
+),
+
+-- Requests with Accept-CH header (servers requesting hints)
+accept_ch_stats AS (
+  SELECT
+    r.client,
+    COUNT(0) AS requests_with_accept_ch,
+    COUNT(DISTINCT r.page) AS pages_with_accept_ch
+  FROM
+    `httparchive.crawl.requests` r,
+    UNNEST(r.response_headers) AS h
+  WHERE
+    r.date = d AND
+    LOWER(h.name) = 'accept-ch' AND
+    h.value IS NOT NULL AND
+    h.value != ''
+  GROUP BY
+    r.client
+),
+
+-- Requests with any Sec-CH-* header (clients sending hints)
+-- Use DISTINCT to count each request only once, even if it has multiple hint headers
+client_hints_sent AS (
+  SELECT
+    client,
+    COUNT(0) AS requests_with_hints,
+    COUNT(DISTINCT page) AS pages_sending_hints
+  FROM (
+    SELECT DISTINCT
+      r.client,
+      r.page,
+      r.url  -- Use url to uniquely identify each request
+    FROM
+      `httparchive.crawl.requests` r,
+      UNNEST(r.request_headers) AS h
+    WHERE
+      r.date = d AND
+      LOWER(h.name) LIKE 'sec-ch-%'
+  )
+  GROUP BY
+    client
+),
+
+-- Count distinct hint types being sent
+hint_diversity AS (
+  SELECT
+    r.client,
+    COUNT(DISTINCT LOWER(h.name)) AS distinct_hint_types
+  FROM
+    `httparchive.crawl.requests` r,
+    UNNEST(r.request_headers) AS h
+  WHERE
+    r.date = d AND
+    LOWER(h.name) LIKE 'sec-ch-%'
+  GROUP BY
+    r.client
+)
+
+SELECT
+  t.client AS `Client`,
+  t.total_requests AS `Total_Requests`,
+  t.total_pages AS `Total_Pages`,
+
+  -- Server-side adoption (Accept-CH)
+  IFNULL(a.requests_with_accept_ch, 0) AS `Requests_with_Accept-CH`,
+  IFNULL(a.pages_with_accept_ch, 0) AS `Pages_with_Accept-CH`,
+  ROUND(IFNULL(a.requests_with_accept_ch, 0) / t.total_requests * 100, 2) AS `Pct_Requests_with_Accept-CH`,
+  ROUND(IFNULL(a.pages_with_accept_ch, 0) / t.total_pages * 100, 2) AS `Pct_Pages_with_Accept-CH`,
+
+  -- Client-side adoption (Sec-CH-*)
+  IFNULL(c.requests_with_hints, 0) AS `Requests_Sending_Hints`,
+  IFNULL(c.pages_sending_hints, 0) AS `Pages_Sending_Hints`,
+  ROUND(IFNULL(c.requests_with_hints, 0) / t.total_requests * 100, 2) AS `Pct_Requests_Sending_Hints`,
+  ROUND(IFNULL(c.pages_sending_hints, 0) / t.total_pages * 100, 2) AS `Pct_Pages_Sending_Hints`,
+
+  -- Diversity
+  IFNULL(h.distinct_hint_types, 0) AS `Distinct_Hint_Types_in_Use`
+
+FROM
+  total_requests t
+LEFT JOIN accept_ch_stats a ON t.client = a.client
+LEFT JOIN client_hints_sent c ON t.client = c.client
+LEFT JOIN hint_diversity h ON t.client = h.client
+ORDER BY
+  t.client;

sql/2025/cdn/client_hints/client_hints_by_cdn.sql

@@ -0,0 +1,70 @@
+#standardSQL
+-- Analyze Client Hints usage by specific CDN provider
+DECLARE d DATE DEFAULT '2025-07-01';
+
+WITH
+-- Extract CDN provider and check for Accept-CH header
+cdn_requests AS (
+  SELECT
+    client,
+    IFNULL(
+      NULLIF(
+        REGEXP_EXTRACT(JSON_EXTRACT_SCALAR(summary, '$._cdn_provider'), r'^([^,]*).*'),
+        ''
+      ),
+      'Origin'
+    ) AS cdn_provider,
+    response_headers
+  FROM
+    `httparchive.crawl.requests`
+  WHERE
+    date = d
+),
+
+-- Count total requests per CDN
+total_by_cdn AS (
+  SELECT
+    client,
+    cdn_provider,
+    COUNT(0) AS total_requests
+  FROM
+    cdn_requests
+  GROUP BY
+    client,
+    cdn_provider
+),
+
+-- Count requests with Accept-CH header per CDN
+accept_ch_by_cdn AS (
+  SELECT
+    r.client,
+    r.cdn_provider,
+    COUNT(0) AS requests_with_accept_ch
+  FROM
+    cdn_requests r,
+    UNNEST(r.response_headers) AS h
+  WHERE
+    LOWER(h.name) = 'accept-ch' AND
+    h.value IS NOT NULL AND
+    h.value != ''
+  GROUP BY
+    r.client,
+    r.cdn_provider
+)
+
+SELECT
+  t.cdn_provider AS `CDN_Provider`,
+  t.client AS `Client`,
+  t.total_requests AS `Total_Requests`,
+  IFNULL(a.requests_with_accept_ch, 0) AS `Requests_with_Accept-CH`,
+  ROUND(IFNULL(a.requests_with_accept_ch, 0) / t.total_requests * 100, 2) AS `Pct_with_Accept-CH`
+FROM
+  total_by_cdn t
+LEFT JOIN accept_ch_by_cdn a ON t.client = a.client AND t.cdn_provider = a.cdn_provider
+WHERE
+  t.total_requests >= 1000  -- Filter to CDNs with meaningful sample size
+ORDER BY
+  t.total_requests DESC,
+  t.cdn_provider,
+  t.client
+LIMIT 50;

sql/2025/cdn/client_hints/client_hints_by_header.sql

@@ -0,0 +1,71 @@
+#standardSQL
+DECLARE d DATE DEFAULT '2025-07-01';
+
+WITH
+-- Count requests per Client Hint and client (mobile/desktop)
+hint_counts AS (
+  SELECT
+    LOWER(h.name) AS client_hint,
+    r.client,
+    COUNT(0) AS requests
+  FROM
+    `httparchive.crawl.requests` r,
+    UNNEST(r.request_headers) AS h
+  WHERE
+    r.date = d AND
+    LOWER(h.name) LIKE 'sec-ch-%'
+  GROUP BY
+    client_hint,
+    r.client
+),
+
+-- Pivot to mobile/desktop columns
+hint_rollup AS (
+  SELECT
+    client_hint,
+    SUM(CASE WHEN client = 'mobile' THEN requests ELSE 0 END) AS mobile_requests,
+    SUM(CASE WHEN client = 'desktop' THEN requests ELSE 0 END) AS desktop_requests
+  FROM
+    hint_counts
+  GROUP BY
+    client_hint
+),
+
+-- Dictionary for Category/Entropy
+dict AS (
+  SELECT *
+  FROM UNNEST([
+    STRUCT('sec-ch-ua' AS client_hint, 'User-Agent' AS category, 'Low' AS entropy, 1 AS ord),
+    ('sec-ch-ua-mobile', 'User-Agent', 'Low', 2),
+    ('sec-ch-ua-platform', 'User-Agent', 'Low', 3),
+    ('sec-ch-ua-platform-version', 'User-Agent', 'High', 4),
+    ('sec-ch-ua-model', 'Device', 'High', 5),
+    ('sec-ch-ua-full-version-list', 'User-Agent', 'High', 6),
+    ('sec-ch-ua-arch', 'Device', 'High', 7),
+    ('sec-ch-ua-bitness', 'Device', 'High', 8),
+    ('sec-ch-ua-wow64', 'Device', 'High', 9),
+    ('sec-ch-ua-full-version', 'User-Agent', 'High', 10),
+    ('sec-ch-prefers-color-scheme', 'User Preference', 'Low', 11),
+    ('sec-ch-viewport-width', 'Viewport', 'High', 12),
+    ('sec-ch-dpr', 'Device', 'High', 13),
+    ('sec-ch-device-memory', 'Device', 'High', 14),
+    ('sec-ch-ua-form-factors', 'Device', 'High', 15),
+    ('sec-ch-viewport-height', 'Viewport', 'High', 16),
+    ('sec-ch-prefers-reduced-motion', 'User Preference', 'Low', 17),
+    ('sec-ch-prefers-reduced-transparency', 'User Preference', 'Low', 18),
+    ('sec-ch-width', 'Viewport', 'High', 19)
+  ])
+)
+
+SELECT
+  d.client_hint AS `Client_Hint`,
+  d.category AS `Category`,
+  d.entropy AS `Entropy`,
+  IFNULL(r.mobile_requests, 0) AS `Mobile_Requests`,
+  IFNULL(r.desktop_requests, 0) AS `Desktop_Requests`,
+  IFNULL(r.mobile_requests, 0) + IFNULL(r.desktop_requests, 0) AS `Total_Requests`
+FROM
+  dict d
+LEFT JOIN hint_rollup r USING (client_hint)
+ORDER BY
+  d.ord;

sql/2025/cdn/client_hints/client_hints_cdn_vs_origin.sql

@@ -0,0 +1,64 @@
+#standardSQL
+-- Compare Client Hints adoption between CDN and origin requests
+DECLARE d DATE DEFAULT '2025-07-01';
+
+WITH
+-- Classify requests as CDN or origin
+requests_classified AS (
+  SELECT
+    client,
+    CASE
+      WHEN IFNULL(NULLIF(REGEXP_EXTRACT(JSON_EXTRACT_SCALAR(summary, '$._cdn_provider'), r'^([^,]*).*'), ''), '') = ''
+        THEN 'Origin'
+      ELSE 'CDN'
+    END AS source_type,
+    response_headers
+  FROM
+    `httparchive.crawl.requests`
+  WHERE
+    date = d
+),
+
+-- Count total requests
+total_requests AS (
+  SELECT
+    client,
+    source_type,
+    COUNT(0) AS total_requests
+  FROM
+    requests_classified
+  GROUP BY
+    client,
+    source_type
+),
+
+-- Count requests with Accept-CH header (server requesting hints)
+accept_ch_requests AS (
+  SELECT
+    r.client,
+    r.source_type,
+    COUNT(0) AS requests_with_accept_ch
+  FROM
+    requests_classified r,
+    UNNEST(r.response_headers) AS h
+  WHERE
+    LOWER(h.name) = 'accept-ch' AND
+    h.value IS NOT NULL AND
+    h.value != ''
+  GROUP BY
+    r.client,
+    r.source_type
+)
+
+SELECT
+  t.client AS `Client`,
+  t.source_type AS `Source`,
+  t.total_requests AS `Total_Requests`,
+  IFNULL(a.requests_with_accept_ch, 0) AS `Requests_with_Accept-CH`,
+  ROUND(IFNULL(a.requests_with_accept_ch, 0) / t.total_requests * 100, 2) AS `Pct_with_Accept-CH`
+FROM
+  total_requests t
+LEFT JOIN accept_ch_requests a ON t.client = a.client AND t.source_type = a.source_type
+ORDER BY
+  t.client,
+  t.source_type;