Fix queries for 2025

Author: tunetheweb

sql/2025/ecommerce/core_web_vitals_by_platform.sql

@@ -48,20 +48,23 @@ FROM
   `chrome-ux-report.materialized.device_summary`
 JOIN (
   SELECT
-    _TABLE_SUFFIX AS client,
-    url,
-    app AS ecomm
+    client,
+    page,
+    tech.technology AS ecomm
   FROM
-    `httparchive.technologies.2025_06_01_*`
+    `httparchive.crawl.pages`,
+    UNNEST(technologies) AS tech,
+    UNNEST(tech.categories) AS category
   WHERE
+    date = '2025-07-01' AND
     category = 'Ecommerce' AND
     (
-      app != 'Cart Functionality' AND
-      app != 'Google Analytics Enhanced eCommerce'
+      tech.technology != 'Cart Functionality' AND
+      tech.technology != 'Google Analytics Enhanced eCommerce'
     )
 )
 ON
-  CONCAT(origin, '/') = url AND
+  CONCAT(origin, '/') = page AND
   IF(device = 'desktop', 'desktop', 'mobile') = client
 WHERE
   date = '2025-07-01'

sql/2025/ecommerce/median_lighthouse_score_ecommsites.sql

@@ -7,23 +7,23 @@ WITH technologies AS (
     rank,
     lighthouse,
     COUNT(DISTINCT page) OVER (PARTITION BY client) AS total_websites
-  FROM `httparchive.all.pages`,
+  FROM `httparchive.crawl.pages`,
     UNNEST(technologies) AS tech,
     UNNEST(categories) AS category
   WHERE
     date = '2025-07-01' AND
-    is_root_page = TRUE
+    is_root_page
 )
 
 SELECT
   client,
   rank,
   technology,
   ARRAY_AGG(DISTINCT category) AS categories,
-  APPROX_QUANTILES(CAST(JSON_EXTRACT_SCALAR(lighthouse, '$.categories.performance.score') AS NUMERIC), 1000)[OFFSET(500)] AS median_performance,
-  APPROX_QUANTILES(CAST(JSON_EXTRACT_SCALAR(lighthouse, '$.categories.accessibility.score') AS NUMERIC), 1000)[OFFSET(500)] AS median_accessibility,
-  APPROX_QUANTILES(CAST(JSON_EXTRACT_SCALAR(lighthouse, '$.categories.seo.score') AS NUMERIC), 1000)[OFFSET(500)] AS median_seo,
-  APPROX_QUANTILES(CAST(JSON_EXTRACT_SCALAR(lighthouse, '$.categories.best-practices.score') AS NUMERIC), 1000)[OFFSET(500)] AS median_best_practices,
+  APPROX_QUANTILES(CAST(JSON_VALUE(lighthouse.categories.performance.score) AS NUMERIC), 1000)[OFFSET(500)] AS median_performance,
+  APPROX_QUANTILES(CAST(JSON_VALUE(lighthouse.categories.accessibility.score) AS NUMERIC), 1000)[OFFSET(500)] AS median_accessibility,
+  APPROX_QUANTILES(CAST(JSON_VALUE(lighthouse.categories.seo.score) AS NUMERIC), 1000)[OFFSET(500)] AS median_seo,
+  APPROX_QUANTILES(CAST(JSON_VALUE(lighthouse.categories.`best-practices`.score) AS NUMERIC), 1000)[OFFSET(500)] AS median_best_practices,
   ANY_VALUE(total_websites) AS total_websites,
   COUNT(DISTINCT page) AS number_of_websites,
   COUNT(DISTINCT page) / ANY_VALUE(total_websites) AS percent_of_websites

sql/2025/ecommerce/top_ecommerce.sql

@@ -9,15 +9,15 @@ SELECT
   total,
   COUNT(DISTINCT page) / total AS pct
 FROM
-  `httparchive.all.pages`,
+  `httparchive.crawl.pages`,
   UNNEST(technologies) AS technologies,
   UNNEST(technologies.categories) AS cats
 JOIN (
   SELECT
     client,
     COUNT(0) AS total
   FROM
-    `httparchive.all.pages`
+    `httparchive.crawl.pages`
   WHERE
     date = '2025-07-01' AND
     is_root_page
@@ -43,15 +43,15 @@ SELECT
   total,
   COUNT(DISTINCT page) / total AS pct
 FROM
-  `httparchive.all.pages`,
+  `httparchive.crawl.pages`,
   UNNEST(technologies) AS technologies,
   UNNEST(technologies.categories) AS cats
 JOIN (
   SELECT
     client,
     COUNT(0) AS total
   FROM
-    `httparchive.all.pages`
+    `httparchive.crawl.pages`
   WHERE
     date = '2024-06-01' AND
     is_root_page
@@ -111,15 +111,15 @@ SELECT
   total,
   COUNT(DISTINCT page) / total AS pct
 FROM
-  `httparchive.all.pages`,
+  `httparchive.crawl.pages`,
   UNNEST(technologies) AS technologies,
   UNNEST(technologies.categories) AS cats
 JOIN (
   SELECT
     client,
     COUNT(0) AS total
   FROM
-    `httparchive.all.pages`
+    `httparchive.crawl.pages`
   WHERE
     date = '2022-08-01' AND -- noqa: CV09
     is_root_page
@@ -145,15 +145,15 @@ SELECT
   total,
   COUNT(DISTINCT page) / total AS pct
 FROM
-  `httparchive.all.pages`,
+  `httparchive.crawl.pages`,
   UNNEST(technologies) AS technologies,
   UNNEST(technologies.categories) AS cats
 JOIN (
   SELECT
     client,
     COUNT(0) AS total
   FROM
-    `httparchive.all.pages`
+    `httparchive.crawl.pages`
   WHERE
     date = '2021-07-01' AND
     is_root_page

sql/2025/ecommerce/top_payment_provider_by_geo.sql

@@ -5,22 +5,22 @@ WITH geo_summary AS (
   SELECT
     `chrome-ux-report`.experimental.GET_COUNTRY(country_code) AS geo,
     IF(device = 'desktop', 'desktop', 'mobile') AS client,
-    origin,
+    CONCAT(origin, '/') AS root_page,
     COUNT(DISTINCT origin) OVER (PARTITION BY country_code, IF(device = 'desktop', 'desktop', 'mobile')) AS total
   FROM
     `chrome-ux-report.materialized.country_summary`
   WHERE
-    yyyymm = 202506
+    yyyymm = 202507
   UNION ALL
   SELECT
     'ALL' AS geo,
     IF(device = 'desktop', 'desktop', 'mobile') AS client,
-    origin,
+    CONCAT(origin, '/') AS root_page,
     COUNT(DISTINCT origin) OVER (PARTITION BY IF(device = 'desktop', 'desktop', 'mobile')) AS total
   FROM
     `chrome-ux-report.materialized.device_summary`
   WHERE
-    yyyymm = 202506
+    yyyymm = 202507
 )
 
 SELECT
@@ -30,40 +30,34 @@ FROM (
     client,
     geo,
     payment,
-    COUNT(0) AS pages,
+    COUNT(DISTINCT root_page) AS sites,
     ANY_VALUE(total) AS total,
-    COUNT(DISTINCT url) / ANY_VALUE(total) AS pct
-  FROM (
-    SELECT DISTINCT
-      geo,
-      client,
-      CONCAT(origin, '/') AS url,
-      total
-    FROM
-      geo_summary
-  ) JOIN (
+    COUNT(DISTINCT root_page) / ANY_VALUE(total) AS pct
+  FROM
+    geo_summary
+  JOIN (
     SELECT DISTINCT
       client,
       cats,
       technologies.technology AS payment,
-      page AS url
+      page,
+      root_page
     FROM
-      `httparchive.all.pages`,
+      `httparchive.crawl.pages`,
       UNNEST(technologies) AS technologies,
       UNNEST(technologies.categories) AS cats
     WHERE
       technologies.technology IS NOT NULL AND
       cats = 'Payment processors' AND
       technologies.technology != '' AND
-      date = '2025-07-01' AND
-      is_root_page
-  ) USING (client, url)
+      date = '2025-07-01'
+  ) USING (client, root_page)
   GROUP BY
     client,
     geo,
     payment
 )
 WHERE
-  pages > 1000
+  sites > 1000
 ORDER BY
-  pages DESC
+  sites DESC

sql/2025/ecommerce/top_payment_providers.sql

@@ -9,15 +9,15 @@ SELECT
   total,
   COUNT(DISTINCT page) / total AS pct
 FROM
-  `httparchive.all.pages`,
+  `httparchive.crawl.pages`,
   UNNEST(technologies) AS technologies,
   UNNEST(technologies.categories) AS cats
 JOIN (
   SELECT
     client,
     COUNT(0) AS total
   FROM
-    `httparchive.all.pages`
+    `httparchive.crawl.pages`
   WHERE
     date = '2025-07-01' AND
     is_root_page
@@ -42,15 +42,15 @@ SELECT
   total,
   COUNT(DISTINCT page) / total AS pct
 FROM
-  `httparchive.all.pages`,
+  `httparchive.crawl.pages`,
   UNNEST(technologies) AS technologies,
   UNNEST(technologies.categories) AS cats
 JOIN (
   SELECT
     client,
     COUNT(0) AS total
   FROM
-    `httparchive.all.pages`
+    `httparchive.crawl.pages`
   WHERE
     date = '2024-06-01' AND
     is_root_page
@@ -75,15 +75,15 @@ SELECT
   total,
   COUNT(DISTINCT page) / total AS pct
 FROM
-  `httparchive.all.pages`,
+  `httparchive.crawl.pages`,
   UNNEST(technologies) AS technologies,
   UNNEST(technologies.categories) AS cats
 JOIN (
   SELECT
     client,
     COUNT(0) AS total
   FROM
-    `httparchive.all.pages`
+    `httparchive.crawl.pages`
   WHERE
     date = '2023-06-01' AND
     is_root_page
@@ -108,25 +108,25 @@ SELECT
   total,
   COUNT(DISTINCT page) / total AS pct
 FROM
-  `httparchive.all.pages`,
+  `httparchive.crawl.pages`,
   UNNEST(technologies) AS technologies,
   UNNEST(technologies.categories) AS cats
 JOIN (
   SELECT
     client,
     COUNT(0) AS total
   FROM
-    `httparchive.all.pages`
+    `httparchive.crawl.pages`
   WHERE
-    date = '2022-08-01' AND -- noqa: CV09
+    date = '2022-06-01' AND
     is_root_page
   GROUP BY
     client
 )
 USING (client)
 WHERE
   cats = 'Payment processors' AND
-  date = '2022-08-01' AND -- noqa: CV09
+  date = '2022-06-01' AND
   is_root_page
 GROUP BY
   client,
@@ -141,15 +141,15 @@ SELECT
   total,
   COUNT(DISTINCT page) / total AS pct
 FROM
-  `httparchive.all.pages`,
+  `httparchive.crawl.pages`,
   UNNEST(technologies) AS technologies,
   UNNEST(technologies.categories) AS cats
 JOIN (
   SELECT
     client,
     COUNT(0) AS total
   FROM
-    `httparchive.all.pages`
+    `httparchive.crawl.pages`
   WHERE
     date = '2021-07-01' AND
     is_root_page

sql/2025/ecommerce/top_shopsystem_by_geo.sql

@@ -5,22 +5,22 @@ WITH geo_summary AS (
   SELECT
     `chrome-ux-report`.experimental.GET_COUNTRY(country_code) AS geo,
     IF(device = 'desktop', 'desktop', 'mobile') AS client,
-    origin,
+    CONCAT(origin, '/') AS root_page,
     COUNT(DISTINCT origin) OVER (PARTITION BY country_code, IF(device = 'desktop', 'desktop', 'mobile')) AS total
   FROM
     `chrome-ux-report.materialized.country_summary`
   WHERE
-    yyyymm = 202506
+    yyyymm = 202507
   UNION ALL
   SELECT
     'ALL' AS geo,
     IF(device = 'desktop', 'desktop', 'mobile') AS client,
-    origin,
+    CONCAT(origin, '/') AS root_page,
     COUNT(DISTINCT origin) OVER (PARTITION BY IF(device = 'desktop', 'desktop', 'mobile')) AS total
   FROM
     `chrome-ux-report.materialized.device_summary`
   WHERE
-    yyyymm = 202506
+    yyyymm = 202507
 )
 
 SELECT
@@ -30,25 +30,20 @@ FROM (
     client,
     geo,
     app,
-    COUNT(0) AS pages,
+    COUNT(DISTINCT root_page) AS sites,
     ANY_VALUE(total) AS total,
-    COUNT(DISTINCT url) / ANY_VALUE(total) AS pct
-  FROM (
-    SELECT DISTINCT
-      geo,
-      client,
-      CONCAT(origin, '/') AS url,
-      total
-    FROM
-      geo_summary
-  ) JOIN (
+    COUNT(DISTINCT root_page) / ANY_VALUE(total) AS pct
+  FROM
+    geo_summary
+  JOIN (
     SELECT DISTINCT
       client,
       cats,
       technologies.technology AS app,
-      page AS url
+      page,
+      root_page
     FROM
-      `httparchive.all.pages`,
+      `httparchive.crawl.pages`,
       UNNEST(technologies) AS technologies,
       UNNEST(technologies.categories) AS cats
     WHERE
@@ -57,15 +52,14 @@ FROM (
       technologies.technology != 'Cart Functionality' AND
       technologies.technology != 'Google Analytics Enhanced eCommerce' AND
       technologies.technology != '' AND
-      date = '2025-07-01' AND
-      is_root_page
-  ) USING (client, url)
+      date = '2025-07-01'
+  ) USING (client, root_page)
   GROUP BY
     client,
     geo,
     app
 )
 WHERE
-  pages > 1000
+  sites > 1000
 ORDER BY
-  pages DESC
+  sites DESC