Fix not working SQL queries

Author: burakguneli

sql/2025/sustainability/favicons.sql

@@ -4,16 +4,59 @@ CREATE TEMPORARY FUNCTION GETFAVICONIMAGE(payload STRING)
 RETURNS STRING LANGUAGE js AS '''
 var result = 'NO_DATA';
 try {
-    var almanac = JSON.parse(payload);
+    var parsed = JSON.parse(payload);
+    
+    // If wrapped, unwrap _almanac
+    if (parsed && typeof parsed === 'object' && parsed._almanac && typeof parsed._almanac === 'object') {
+      parsed = parsed._almanac;
+    }
 
-    if (Array.isArray(almanac) || typeof almanac != 'object') return result;
+    // Deep search for any array of link-like nodes anywhere in the object
+    function findLinkNodes(obj) {
+      if (!obj) return [];
+      var stack = [obj];
+      while (stack.length) {
+        var current = stack.pop();
+        if (!current) continue;
+        if (Array.isArray(current)) {
+          // If array of objects with rel/href, treat as nodes
+          if (
+            current.length && typeof current[0] === 'object' && current.some(function(it){return it && (it.href || it.rel);})
+          ) {
+            return current;
+          }
+          for (var i = 0; i < current.length; i++) stack.push(current[i]);
+        } else if (typeof current === 'object') {
+          // Common patterns: {nodes: [...]} wrappers
+          if (current.nodes && Array.isArray(current.nodes)) {
+            var n = current.nodes;
+            if (n.length && typeof n[0] === 'object' && n.some(function(it){return it && (it.href || it.rel);})){return n;}
+          }
+          for (var k in current) if (Object.prototype.hasOwnProperty.call(current, k)) stack.push(current[k]);
+        }
+      }
+      return [];
+    }
 
-    if (almanac["link-nodes"] && almanac["link-nodes"].nodes && almanac["link-nodes"].nodes.find) {
-      var faviconNode = almanac["link-nodes"].nodes.find(n => n.rel && n.rel.split(' ').find(r => r.trim().toLowerCase() == 'icon'));
+    var nodes = findLinkNodes(parsed);
+    if (!nodes || !nodes.length) return result;
+
+    if (nodes && nodes.find) {
+      var faviconNode = nodes.find(function(n) {
+        if (!n || !('rel' in n)) return false;
+        var rels = Array.isArray(n.rel) ? n.rel : String(n.rel).split(' ');
+        for (var j = 0; j < rels.length; j++) {
+          var r = String(rels[j]).trim().toLowerCase();
+          if (r === 'icon' || r === 'shortcut icon' || r === 'apple-touch-icon' || r === 'apple-touch-icon-precomposed') {
+            return true;
+          }
+        }
+        return false;
+      });
 
       if (faviconNode) {
         if (faviconNode.href) {
-          var temp = faviconNode.href;
+          var temp = String(faviconNode.href);
 
           if (temp.includes('?')) {
             temp = temp.substring(0, temp.indexOf('?'));
@@ -43,29 +86,98 @@ try {
 return result;
 ''';
 
-# Main query to analyze favicon image extensions with sampling
-WITH favicons AS (
+# Main query to analyze favicon image extensions using requests heuristics
+WITH pages AS (
   SELECT
     client,
-    GETFAVICONIMAGE(
-      JSON_EXTRACT_SCALAR(payload, '$._almanac')
-    ) AS image_type_extension,
-    COUNT(0) AS freq,
-    SUM(COUNT(0)) OVER (PARTITION BY client) AS total,
-    COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client) AS percentage_of_total
+    page
   FROM
     `httparchive.crawl.pages`
   WHERE
-    date = '2025-06-01'
+    date = '2025-06-01' AND
+    is_root_page
+),
+
+reqs AS (
+  SELECT
+    client,
+    page,
+    url,
+    response_headers
+  FROM
+    `httparchive.crawl.requests`
+  WHERE
+    date = '2025-06-01' AND
+    is_root_page
+),
+
+candidates AS (
+  SELECT
+    r.client,
+    r.page,
+    r.url,
+    LOWER(
+      REGEXP_EXTRACT(
+        REGEXP_REPLACE(SPLIT(r.url, '?')[SAFE_OFFSET(0)], r'/+$', ''),
+        r'\.([A-Za-z0-9]+)$'
+      )
+    ) AS url_ext,
+    (
+      SELECT LOWER(value)
+      FROM UNNEST(r.response_headers)
+      WHERE LOWER(name) = 'content-type'
+      LIMIT 1
+    ) AS content_type
+  FROM
+    reqs r
+  JOIN
+    pages p
+  USING (client, page)
+  WHERE
+    REGEXP_CONTAINS(LOWER(r.url), r'favicon|apple-touch-icon|android-chrome|mstile|safari-pinned-tab')
+),
+
+resolved AS (
+  SELECT
+    client,
+    page,
+    url,
+    COALESCE(
+      NULLIF(url_ext, ''),
+      CASE
+        WHEN content_type LIKE 'image/svg%' THEN 'svg'
+        WHEN content_type LIKE 'image/png%' THEN 'png'
+        WHEN content_type LIKE 'image/webp%' THEN 'webp'
+        WHEN content_type LIKE 'image/jpeg%' OR content_type LIKE 'image/jpg%' THEN 'jpg'
+        WHEN content_type LIKE 'image/x-icon%' OR content_type LIKE 'image/vnd.microsoft.icon%' THEN 'ico'
+        ELSE 'unknown'
+      END
+    ) AS image_type_extension
+  FROM
+    candidates
+),
+
+rollup_data AS (
+  SELECT
+    client,
+    image_type_extension,
+    COUNT(DISTINCT page) AS pages
+  FROM
+    resolved
   GROUP BY
     client,
     image_type_extension
 )
 
 SELECT
-  *,
-  percentage_of_total AS pct
+  client,
+  image_type_extension,
+  pages AS count,
+  SUM(pages) OVER (PARTITION BY client) AS total,
+  ROUND(100 * SAFE_DIVIDE(pages, SUM(pages) OVER (PARTITION BY client)), 2) AS pct
 FROM
-  favicons
+  rollup_data
 ORDER BY
-  pct DESC
+  client ASC,
+  count DESC,
+  image_type_extension ASC

sql/2025/sustainability/green_third_party_requests.sql

@@ -1,11 +1,23 @@
 #standardSQL
 # Median third-parties & green third-party requests per websites by rank
 
-WITH requests AS (
+WITH third_party_date AS (
+  SELECT MAX(date) AS date
+  FROM `httparchive.almanac.third_parties`
+  WHERE date <= '2025-06-01'
+),
+
+gwf_date AS (
+  SELECT MAX(date) AS date
+  FROM `httparchive.almanac.green_web_foundation`
+  WHERE date <= '2025-06-01'
+),
+
+requests AS (
   SELECT
     client,
     url,
-    CAST(JSON_VALUE(summary, '$.pageid') AS INT64) AS page
+    page
   FROM
     `httparchive.crawl.requests`
   WHERE
@@ -17,20 +29,21 @@ green AS (
     TRUE AS is_green,
     NET.HOST(url) AS host
   FROM
-    `httparchive.almanac.green_web_foundation`
-  WHERE
-    date = '2025-09-01'
+    `httparchive.almanac.green_web_foundation` g
+  JOIN gwf_date d
+  ON g.date = d.date
 ),
 
 pages AS (
   SELECT
     client,
     rank,
-    CAST(JSON_VALUE(summary, '$.pageid') AS INT64) AS page
+    page
   FROM
     `httparchive.crawl.pages`
   WHERE
-    date = '2025-06-01'
+    date = '2025-06-01' AND
+    is_root_page
 ),
 
 third_party AS (
@@ -41,9 +54,9 @@ third_party AS (
     `httparchive.almanac.third_parties` AS tp
   INNER JOIN
     requests AS r
-  ON NET.HOST(r.url) = NET.HOST(tp.domain)
+  ON NET.REG_DOMAIN(r.url) = NET.REG_DOMAIN(tp.domain)
   WHERE
-    tp.date = '2025-06-01' AND
+    tp.date = (SELECT date FROM third_party_date) AND
     tp.category NOT IN ('hosting')
   GROUP BY
     tp.domain
@@ -57,9 +70,9 @@ green_tp AS (
     `httparchive.almanac.third_parties` AS tp
   INNER JOIN
     green AS g
-  ON NET.HOST(g.host) = NET.HOST(tp.domain)
+  ON NET.REG_DOMAIN(g.host) = NET.REG_DOMAIN(tp.domain)
   WHERE
-    tp.date = '2025-06-01' AND
+    tp.date = (SELECT date FROM third_party_date) AND
     tp.category NOT IN ('hosting')
   GROUP BY
     tp.domain
@@ -70,13 +83,13 @@ base AS (
     r.client,
     r.page,
     p.rank,
-    COUNT(tp.domain) AS third_parties_per_page
+    COUNT(DISTINCT tp.domain) AS third_parties_per_page
   FROM
     requests AS r
   LEFT JOIN
     third_party AS tp
   ON
-    NET.HOST(r.url) = NET.HOST(tp.domain)
+    NET.REG_DOMAIN(r.url) = NET.REG_DOMAIN(tp.domain)
   INNER JOIN
     pages AS p
   ON r.client = p.client AND r.page = p.page
@@ -91,13 +104,13 @@ base_green AS (
     r.client,
     r.page,
     p.rank,
-    COUNT(gtp.domain) AS green_third_parties_per_page
+    COUNT(DISTINCT gtp.domain) AS green_third_parties_per_page
   FROM
     requests AS r
   LEFT JOIN
     green_tp AS gtp
   ON
-    NET.HOST(r.url) = NET.HOST(gtp.domain)
+    NET.REG_DOMAIN(r.url) = NET.REG_DOMAIN(gtp.domain)
   INNER JOIN
     pages AS p
   ON r.client = p.client AND r.page = p.page
@@ -126,7 +139,7 @@ SELECT
       bg.green_third_parties_per_page,
       b.third_parties_per_page
     ), 1000
-  )[OFFSET(500)] AS pct_green
+  )[OFFSET(500)] * 100 AS pct_green
 FROM
   base AS b,
   UNNEST([1000, 10000, 100000, 1000000, 10000000, 100000000]) AS rank_grouping

sql/2025/sustainability/green_web_hosting.sql

@@ -1,14 +1,21 @@
 # standardSQL
 # What percentage of URLs are hosted on a known green web hosting provider?
 
-WITH green AS (
+WITH gwf_date AS (
+  SELECT MAX(date) AS date
+  FROM `httparchive.almanac.green_web_foundation`
+  WHERE date <= '2025-06-01'
+),
+
+green AS (
   SELECT
     TRUE AS is_green,
     NET.HOST(url) AS host
   FROM
-    `httparchive.almanac.green_web_foundation`
-  WHERE
-    date = '2025-09-01'
+    `httparchive.almanac.green_web_foundation` g
+  JOIN
+    gwf_date d
+  ON g.date = d.date
 ),
 
 pages AS (
@@ -34,7 +41,7 @@ SELECT
   END AS ranking,
   COUNTIF(is_green) AS total_green,
   COUNT(0) AS total_sites,
-  SAFE_DIVIDE(COUNTIF(is_green), COUNT(0)) AS pct_green
+  ROUND(100 * SAFE_DIVIDE(COUNTIF(is_green), COUNT(0)), 2) AS pct_green
 FROM (
   -- Left join green hosting information
   SELECT