Skip to content

Commit 7e1f76d

Browse files
tunethewebtunetheweb
authored
Merge branch 'main' into patch-1
2 parents 79b4f7e + b92fa4d commit 7e1f76d

9 files changed

+988
-0
lines changed

‎sql/2025/cdn/alt_svc_protocol_analysis.sql‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
#standardSQL
2+
# Alt-Svc Protocol Analysis - Deep dive into Alternative Service headers
3+
# Analyzes what protocols are advertised vs actually used
4+
5+
WITH alt_svc_analysis AS (
6+
SELECT
7+
client,
8+
page,
9+
url,
10+
is_main_document,
11+
12+
-- CDN detection
13+
IFNULL(
14+
NULLIF(REGEXP_EXTRACT(JSON_EXTRACT_SCALAR(summary, '$._cdn_provider'), r'^([^,]*).*'), ''),
15+
'ORIGIN'
16+
) AS cdn,
17+
18+
-- Current protocol used
19+
UPPER(IFNULL(
20+
JSON_EXTRACT_SCALAR(payload, '$._protocol'),
21+
IFNULL(
22+
NULLIF(JSON_EXTRACT_SCALAR(payload, '$._tls_next_proto'), 'unknown'),
23+
NULLIF(CONCAT('HTTP/', JSON_EXTRACT_SCALAR(payload, '$.response.httpVersion')), 'HTTP/')
24+
)
25+
)) AS current_protocol,
26+
27+
-- Extract Alt-Svc header value
28+
(
29+
SELECT h.value
30+
FROM UNNEST(response_headers) AS h
31+
WHERE LOWER(h.name) = 'alt-svc'
32+
LIMIT 1 -- noqa: AM09
33+
) AS alt_svc_header,
34+
35+
-- Check if Alt-Svc exists
36+
EXISTS(
37+
SELECT 1
38+
FROM UNNEST(response_headers) AS h
39+
WHERE LOWER(h.name) = 'alt-svc'
40+
) AS has_alt_svc
41+
42+
FROM `httparchive.crawl.requests`
43+
WHERE date = '2025-07-01' AND
44+
EXISTS (
45+
SELECT 1
46+
FROM UNNEST(response_headers) AS h
47+
WHERE LOWER(h.name) = 'alt-svc'
48+
)
49+
),
50+
51+
protocol_extraction AS (
52+
SELECT
53+
*,
54+
-- Extract HTTP/3 variants from Alt-Svc
55+
REGEXP_CONTAINS(LOWER(alt_svc_header), r'h3[^=]*=') AS advertises_h3,
56+
REGEXP_CONTAINS(LOWER(alt_svc_header), r'h3-\d+[^=]*=') AS advertises_h3_draft,
57+
REGEXP_CONTAINS(LOWER(alt_svc_header), r'h2[^=]*=') AS advertises_h2,
58+
REGEXP_CONTAINS(LOWER(alt_svc_header), r'http/1\.1[^=]*=') AS advertises_h1,
59+
60+
-- Extract max-age values
61+
SAFE_CAST(REGEXP_EXTRACT(alt_svc_header, r'ma=(\d+)') AS INT64) AS max_age_seconds,
62+
63+
-- Check for clear directive
64+
REGEXP_CONTAINS(LOWER(alt_svc_header), r'clear') AS has_clear_directive
65+
66+
FROM alt_svc_analysis
67+
)
68+
69+
SELECT
70+
client,
71+
cdn,
72+
is_main_document,
73+
current_protocol,
74+
COUNT(0) AS total_requests,
75+
76+
-- Protocol advertising analysis
77+
COUNTIF(advertises_h3) AS advertises_h3_count,
78+
ROUND(SAFE_DIVIDE(COUNTIF(advertises_h3), COUNT(0)) * 100, 2) AS advertises_h3_pct,
79+
80+
COUNTIF(advertises_h3_draft) AS advertises_h3_draft_count,
81+
ROUND(SAFE_DIVIDE(COUNTIF(advertises_h3_draft), COUNT(0)) * 100, 2) AS advertises_h3_draft_pct,
82+
83+
COUNTIF(advertises_h2) AS advertises_h2_count,
84+
ROUND(SAFE_DIVIDE(COUNTIF(advertises_h2), COUNT(0)) * 100, 2) AS advertises_h2_pct,
85+
86+
COUNTIF(advertises_h1) AS advertises_h1_count,
87+
ROUND(SAFE_DIVIDE(COUNTIF(advertises_h1), COUNT(0)) * 100, 2) AS advertises_h1_pct,
88+
89+
-- Protocol mismatch analysis
90+
COUNTIF(advertises_h3 AND current_protocol NOT LIKE '%H3%' AND current_protocol != 'HTTP/3') AS h3_advertised_not_used,
91+
COUNTIF(current_protocol LIKE '%H3%' OR current_protocol = 'HTTP/3') AS currently_using_h3,
92+
93+
-- Max-age statistics
94+
APPROX_QUANTILES(max_age_seconds, 100)[OFFSET(50)] AS median_max_age_seconds,
95+
APPROX_QUANTILES(max_age_seconds, 100)[OFFSET(90)] AS p90_max_age_seconds,
96+
97+
-- Clear directive usage
98+
COUNTIF(has_clear_directive) AS clear_directive_count,
99+
ROUND(SAFE_DIVIDE(COUNTIF(has_clear_directive), COUNT(0)) * 100, 2) AS clear_directive_pct,
100+
101+
-- Sample Alt-Svc headers for analysis
102+
ARRAY_AGG(alt_svc_header IGNORE NULLS LIMIT 5) AS sample_alt_svc_headers
103+
104+
FROM protocol_extraction
105+
GROUP BY
106+
client,
107+
cdn,
108+
is_main_document,
109+
current_protocol
110+
HAVING
111+
total_requests >= 100
112+
ORDER BY
113+
client DESC,
114+
total_requests DESC,
115+
advertises_h3_pct DESC

‎sql/2025/cdn/brotli_compression_adoption.sql‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
#standardSQL
2+
# 20_brotli_compression_adoption.sql: Brotli and modern compression adoption by CDN
3+
# Analyzes compression algorithm usage patterns and efficiency across CDN providers
4+
#
5+
# Rationale: Brotli compression offers 20-30% better compression than gzip for text resources.
6+
# As it becomes more widely supported, we want to track which CDNs are leading adoption
7+
# and how much bandwidth savings are being achieved. This is critical for performance
8+
# and sustainability metrics.
9+
10+
WITH compression_analysis AS (
11+
SELECT
12+
client,
13+
14+
-- CDN detection
15+
IFNULL(
16+
NULLIF(REGEXP_EXTRACT(JSON_EXTRACT_SCALAR(summary, '$._cdn_provider'), r'^([^,]*).*'), ''),
17+
'ORIGIN'
18+
) AS cdn,
19+
20+
-- Page and resource info
21+
page,
22+
url,
23+
is_main_document,
24+
25+
-- Content type classification
26+
CASE
27+
WHEN REGEXP_CONTAINS(LOWER(url), r'\.(js|mjs)($|\?)') THEN 'JavaScript'
28+
WHEN REGEXP_CONTAINS(LOWER(url), r'\.css($|\?)') THEN 'CSS'
29+
WHEN REGEXP_CONTAINS(LOWER(url), r'\.(html|htm)($|\?)') OR is_main_document THEN 'HTML'
30+
WHEN REGEXP_CONTAINS(LOWER(url), r'\.(json)($|\?)') THEN 'JSON'
31+
WHEN REGEXP_CONTAINS(LOWER(url), r'\.(svg)($|\?)') THEN 'SVG'
32+
WHEN REGEXP_CONTAINS(LOWER(url), r'\.(woff2?|ttf|otf|eot)($|\?)') THEN 'Fonts'
33+
WHEN REGEXP_CONTAINS(LOWER(url), r'\.(jpg|jpeg|png|gif|webp|avif)($|\?)') THEN 'Images'
34+
ELSE 'Other'
35+
END AS content_type,
36+
37+
-- Compression detection from Content-Encoding header
38+
(
39+
SELECT LOWER(h.value)
40+
FROM UNNEST(response_headers) AS h
41+
WHERE LOWER(h.name) = 'content-encoding'
42+
LIMIT 1 -- noqa: AM09
43+
) AS content_encoding,
44+
45+
-- Vary header check (indicates dynamic compression support)
46+
EXISTS(
47+
SELECT 1 FROM UNNEST(response_headers) AS h
48+
WHERE LOWER(h.name) = 'vary' AND LOWER(h.value) LIKE '%accept-encoding%'
49+
) AS supports_dynamic_compression,
50+
51+
-- Response size metrics
52+
SAFE_CAST(JSON_EXTRACT_SCALAR(payload, '$.response.bodySize') AS INT64) AS response_body_size,
53+
SAFE_CAST(JSON_EXTRACT_SCALAR(payload, '$.response.bodySize') AS INT64) AS uncompressed_size,
54+
55+
-- Transfer size (actual bytes transferred)
56+
SAFE_CAST(JSON_EXTRACT_SCALAR(payload, '$.response._transferSize') AS INT64) AS transfer_size
57+
FROM `httparchive.crawl.requests`
58+
WHERE date = '2025-07-01' AND
59+
-- Focus on compressible content types
60+
REGEXP_CONTAINS(LOWER(url), r'\.(js|mjs|css|html|htm|json|svg|xml|txt)($|\?)')
61+
)
62+
63+
SELECT
64+
client,
65+
cdn,
66+
content_type,
67+
68+
-- Volume metrics
69+
COUNT(DISTINCT page) AS total_pages,
70+
COUNT(0) AS total_requests,
71+
72+
-- Compression type distribution
73+
COUNTIF(content_encoding = 'br') AS brotli_requests,
74+
COUNTIF(content_encoding = 'gzip') AS gzip_requests,
75+
COUNTIF(content_encoding = 'deflate') AS deflate_requests,
76+
COUNTIF(content_encoding IS NULL OR content_encoding = '') AS uncompressed_requests,
77+
COUNTIF(content_encoding NOT IN ('br', 'gzip', 'deflate', '') AND content_encoding IS NOT NULL) AS other_compression,
78+
79+
-- Compression percentages
80+
ROUND(SAFE_DIVIDE(COUNTIF(content_encoding = 'br'), COUNT(0)) * 100, 2) AS brotli_pct,
81+
ROUND(SAFE_DIVIDE(COUNTIF(content_encoding = 'gzip'), COUNT(0)) * 100, 2) AS gzip_pct,
82+
ROUND(SAFE_DIVIDE(COUNTIF(content_encoding = 'deflate'), COUNT(0)) * 100, 2) AS deflate_pct,
83+
ROUND(SAFE_DIVIDE(COUNTIF(content_encoding IS NULL OR content_encoding = ''), COUNT(0)) * 100, 2) AS uncompressed_pct,
84+
85+
-- Dynamic compression support
86+
COUNTIF(supports_dynamic_compression) AS dynamic_compression_count,
87+
ROUND(SAFE_DIVIDE(COUNTIF(supports_dynamic_compression), COUNT(0)) * 100, 2) AS dynamic_compression_pct,
88+
89+
-- Size metrics (in KB)
90+
ROUND(AVG(response_body_size) / 1024, 2) AS avg_response_size_kb,
91+
ROUND(AVG(CASE WHEN content_encoding = 'br' THEN response_body_size END) / 1024, 2) AS avg_brotli_size_kb,
92+
ROUND(AVG(CASE WHEN content_encoding = 'gzip' THEN response_body_size END) / 1024, 2) AS avg_gzip_size_kb,
93+
ROUND(AVG(CASE WHEN content_encoding IS NULL OR content_encoding = '' THEN response_body_size END) / 1024, 2) AS avg_uncompressed_size_kb,
94+
95+
-- Compression efficiency comparison
96+
ROUND(
97+
SAFE_DIVIDE(
98+
AVG(CASE WHEN content_encoding = 'gzip' THEN response_body_size END) -
99+
AVG(CASE WHEN content_encoding = 'br' THEN response_body_size END),
100+
AVG(CASE WHEN content_encoding = 'gzip' THEN response_body_size END)
101+
) * 100, 2
102+
) AS brotli_vs_gzip_savings_pct,
103+
104+
-- Total data transfer metrics
105+
ROUND(SUM(response_body_size) / (1024 * 1024 * 1024), 2) AS total_gb_transferred,
106+
ROUND(SUM(CASE WHEN content_encoding = 'br' THEN response_body_size END) / (1024 * 1024 * 1024), 2) AS total_gb_brotli,
107+
ROUND(SUM(CASE WHEN content_encoding = 'gzip' THEN response_body_size END) / (1024 * 1024 * 1024), 2) AS total_gb_gzip
108+
FROM compression_analysis
109+
GROUP BY
110+
client,
111+
cdn,
112+
content_type
113+
HAVING
114+
total_requests >= 100 -- Minimum threshold for statistical relevance
115+
ORDER BY
116+
client DESC,
117+
brotli_pct DESC,
118+
total_requests DESC

0 commit comments

Comments
 (0)