updated metrics

Author: max-ostapenko

sql/2025/privacy/bounce_domains_top.sql

@@ -17,19 +17,19 @@ WITH redirect_requests AS (
     url,
     index,
     NET.REG_DOMAIN(header.value) AS location_domain,
-    page
+    root_page
 ),
 
 -- Find the first navigation redirect
 navigation_redirect AS (
   FROM redirect_requests
   |> WHERE
     index = 1 AND
-    NET.REG_DOMAIN(page) = NET.REG_DOMAIN(url) AND
+    NET.REG_DOMAIN(root_page) = NET.REG_DOMAIN(url) AND
     NET.REG_DOMAIN(url) != location_domain
   |> SELECT
     client,
-    page,
+    root_page,
     location_domain AS bounce_domain
 ),
 
@@ -38,12 +38,12 @@ bounce_redirect AS (
   FROM redirect_requests
   |> WHERE
     index = 2 AND
-    NET.REG_DOMAIN(page) != NET.REG_DOMAIN(url) AND
+    NET.REG_DOMAIN(root_page) != NET.REG_DOMAIN(url) AND
     NET.REG_DOMAIN(url) != location_domain
   |> SELECT
     client,
     url,
-    page,
+    root_page,
     location_domain AS bounce_redirect_location_domain
 ),
 
@@ -53,26 +53,26 @@ bounce_sequences AS (
   |> JOIN bounce_redirect AS bounce
   ON
     nav.client = bounce.client AND
-    nav.page = bounce.page
-  |> AGGREGATE COUNT(DISTINCT nav.page) AS pages_count
+    nav.root_page = bounce.root_page
+  |> AGGREGATE COUNT(DISTINCT nav.root_page) AS websites_count
   GROUP BY nav.client, bounce_domain
 ),
 
-pages_total AS (
+websites_total AS (
   FROM `httparchive.crawl.pages`
   |> WHERE date = '2025-07-01' --AND rank = 1000
-  |> AGGREGATE COUNT(DISTINCT page) AS total_pages GROUP BY client
+  |> AGGREGATE COUNT(DISTINCT root_page) AS total_websites GROUP BY client
 )
 
 FROM bounce_sequences
-|> JOIN pages_total USING (client)
-|> EXTEND pages_count / total_pages AS pages_pct
-|> DROP total_pages
+|> JOIN websites_total USING (client)
+|> EXTEND websites_count / total_websites AS websites_pct
+|> DROP total_websites
 |> PIVOT(
-  ANY_VALUE(pages_count) AS cnt,
-  ANY_VALUE(pages_pct) AS pages_pct
+  ANY_VALUE(websites_count) AS cnt,
+  ANY_VALUE(websites_pct) AS pct
   FOR client IN ('desktop', 'mobile')
 )
-|> RENAME cnt_mobile AS mobile, cnt_desktop AS desktop
-|> ORDER BY mobile + desktop DESC
+|> RENAME pct_mobile AS mobile, pct_desktop AS desktop, cnt_mobile AS mobile_count, cnt_desktop AS desktop_count
+|> ORDER BY mobile_count + desktop_count DESC
 |> LIMIT 100

sql/2025/privacy/client_hints_top.sql

@@ -12,7 +12,7 @@ headers AS (
   |> JOIN UNNEST(response_headers) AS header
   |> WHERE LOWER(header.name) = 'accept-ch'
   |> LEFT JOIN UNNEST(SPLIT(LOWER(header.value), ',')) AS header_value
-  |> SELECT client, page, header_value
+  |> SELECT client, root_page, header_value
 
 ),
 
@@ -21,28 +21,24 @@ meta_tags AS (
   FROM `httparchive.crawl.pages`
   |> WHERE date = '2025-07-01' AND is_root_page --AND rank = 1000
   |> JOIN UNNEST(JSON_QUERY_ARRAY(custom_metrics.other.almanac.`meta-nodes`.nodes)) AS meta_node
-  |> EXTEND
-  LOWER(SAFE.STRING(meta_node.`http-equiv`)) AS tag_name,
+  |> EXTEND LOWER(SAFE.STRING(meta_node.`http-equiv`)) AS tag_name
   |> WHERE tag_name = 'accept-ch'
   |> LEFT JOIN UNNEST(SPLIT(LOWER(SAFE.STRING(meta_node.content)), ',')) AS tag_value
-  |> SELECT client, page, tag_value
+  |> SELECT client, root_page, tag_value
 )
 
 FROM headers
-|> FULL OUTER JOIN meta_tags USING (client, page)
+|> FULL OUTER JOIN meta_tags USING (client, root_page)
 |> JOIN totals USING (client)
 |> EXTEND TRIM(COALESCE(header_value, tag_value)) AS value
 |> AGGREGATE
-COUNT(DISTINCT page) AS number_of_pages,
-COUNT(DISTINCT page) / ANY_VALUE(total_websites) AS pct_pages
+COUNT(DISTINCT root_page) AS number_of_websites,
+COUNT(DISTINCT root_page) / ANY_VALUE(total_websites) AS pct_websites
 GROUP BY client, value
 |> PIVOT(
-  ANY_VALUE(number_of_pages) AS pages_count,
-  ANY_VALUE(pct_pages) AS pct
+  ANY_VALUE(number_of_websites) AS websites_count,
+  ANY_VALUE(pct_websites) AS pct
   FOR client IN ('desktop', 'mobile')
 )
-|> RENAME
-pct_mobile AS mobile,
-pct_desktop AS desktop
-|> ORDER BY pages_count_mobile + pages_count_desktop DESC
-|> LIMIT 200
+|> RENAME pct_mobile AS mobile, pct_desktop AS desktop
+|> ORDER BY websites_count_desktop + websites_count_mobile DESC

sql/2025/privacy/iab_tcf_v2_cmps_top.sql

@@ -1,3 +1,4 @@
+-- noqa: disable=PRS
 -- Counts of CMPs using IAB Transparency & Consent Framework
 -- cf. https://github.com/InteractiveAdvertisingBureau/GDPR-Transparency-and-Consent-Framework/blob/master/TCFv2/IAB%20Tech%20Lab%20-%20CMP%20API%20v2.md--tcdata
 -- CMP vendor list: https://iabeurope.eu/cmp-list/
@@ -6,15 +7,15 @@ FROM `httparchive.crawl.pages`
 |> WHERE date = '2025-07-01' --AND rank = 1000
 |> EXTEND
 SAFE.INT64(custom_metrics.privacy.iab_tcf_v2.data.cmpId) AS cmpId,
-COUNT(DISTINCT page) OVER (PARTITION BY client) AS total_pages
+COUNT(DISTINCT root_page) OVER (PARTITION BY client) AS total_websites
 |> AGGREGATE
-COUNT(0) AS number_of_pages,
-COUNT(0) / ANY_VALUE(total_pages) AS pct_pages
+COUNT(DISTINCT root_page) AS number_of_websites,
+COUNT(DISTINCT root_page) / ANY_VALUE(total_websites) AS pct_websites
 GROUP BY client, cmpId
 |> PIVOT (
-  ANY_VALUE(number_of_pages) AS pages_count,
-  ANY_VALUE(pct_pages) AS pct
+  ANY_VALUE(number_of_websites) AS websites_count,
+  ANY_VALUE(pct_websites) AS pct
   FOR client IN ('desktop', 'mobile')
 )
 |> RENAME pct_mobile AS mobile, pct_desktop AS desktop
-|> ORDER BY pages_count_mobile + pages_count_desktop DESC
+|> ORDER BY websites_count_desktop + websites_count_mobile DESC

sql/2025/privacy/iab_tcf_v2_countries_top.sql

@@ -26,16 +26,14 @@ base_data AS (
 )
 
 FROM base_data
-|> AGGREGATE
-  COUNT(DISTINCT root_page) AS number_of_pages
-GROUP BY client, publisherCC
+|> AGGREGATE COUNT(DISTINCT root_page) AS number_of_websites GROUP BY client, publisherCC
 |> JOIN base_totals USING (client)
-|> EXTEND number_of_pages / total_websites AS pct_of_pages
+|> EXTEND number_of_websites / total_websites AS pct_of_websites
 |> DROP total_websites
 |> PIVOT(
-  ANY_VALUE(number_of_pages) AS pages_count,
-  ANY_VALUE(pct_of_pages) AS pct
+  ANY_VALUE(number_of_websites) AS websites_count,
+  ANY_VALUE(pct_of_websites) AS pct
   FOR client IN ('desktop', 'mobile')
 )
 |> RENAME pct_mobile AS mobile, pct_desktop AS desktop
-|> ORDER BY pages_count_mobile + pages_count_desktop DESC
+|> ORDER BY websites_count_desktop + websites_count_mobile DESC