Improve CI workflow verification and caching

.github/dependabot.yml

@@ -0,0 +1,11 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+
+  - package-ecosystem: "vcpkg"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/scripts/generate-tiles.ps1

@@ -0,0 +1,69 @@
+param(
+	[Parameter(Mandatory = $true)]
+	[string] $Tilemaker,
+
+	[Parameter(Mandatory = $true)]
+	[string] $Area,
+
+	[Parameter(Mandatory = $true)]
+	[AllowEmptyString()]
+	[string] $OutputSuffix,
+
+	[Parameter(Mandatory = $true)]
+	[string] $StoreRoot
+)
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = "Stop"
+
+function Invoke-Tilemaker {
+	param(
+		[Parameter(Mandatory = $true)]
+		[string] $Output,
+
+		[Parameter(Mandatory = $true)]
+		[string[]] $Arguments
+	)
+
+	Write-Host "::group::Build $Output"
+	& $Tilemaker @Arguments
+	$status = $LASTEXITCODE
+	Write-Host "::endgroup::"
+	if ($status -ne 0) {
+		Write-Error "tilemaker failed while writing $Output with exit code $status"
+		exit $status
+	}
+	if (!(Test-Path -LiteralPath $Output) -or (Get-Item -LiteralPath $Output).Length -eq 0) {
+		Write-Error "tilemaker did not create $Output"
+		exit 1
+	}
+}
+
+$inputPbf = "${Area}.osm.pbf"
+$common = @(
+	$inputPbf,
+	"--config=resources/config-openmaptiles.json",
+	"--process=resources/process-openmaptiles.lua"
+)
+
+Invoke-Tilemaker "${Area}${OutputSuffix}.pmtiles" `
+	($common + @("--output=${Area}${OutputSuffix}.pmtiles", "--verbose"))
+
+Invoke-Tilemaker "${Area}${OutputSuffix}-repeat.pmtiles" `
+	($common + @("--output=${Area}${OutputSuffix}-repeat.pmtiles", "--verbose"))
+
+Invoke-Tilemaker "${Area}${OutputSuffix}.mbtiles" `
+	($common + @(
+		"--output=${Area}${OutputSuffix}.mbtiles",
+		"--store",
+		(Join-Path $StoreRoot "store${OutputSuffix}"),
+		"--verbose"
+	))
+
+Invoke-Tilemaker "${Area}${OutputSuffix}-repeat.mbtiles" `
+	($common + @(
+		"--output=${Area}${OutputSuffix}-repeat.mbtiles",
+		"--store",
+		(Join-Path $StoreRoot "store${OutputSuffix}-repeat"),
+		"--verbose"
+	))

.github/scripts/generate-tiles.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [ "$#" -ne 4 ]; then
+	echo "usage: $0 <tilemaker> <area> <output-suffix> <store-root>" >&2
+	exit 2
+fi
+
+tilemaker="$1"
+area="$2"
+suffix="$3"
+store_root="$4"
+
+run_tilemaker() {
+	output="$1"
+	shift
+	echo "::group::Build ${output}"
+	if "$@"; then
+		echo "::endgroup::"
+	else
+		status="$?"
+		echo "::endgroup::"
+		echo "::error title=Tile generation failed::tilemaker failed while writing ${output} with exit code ${status}"
+		exit "${status}"
+	fi
+	if [ ! -s "${output}" ]; then
+		echo "::error title=Tile output missing::tilemaker did not create ${output}"
+		exit 1
+	fi
+}
+
+common=(
+	"${area}.osm.pbf"
+	--config=resources/config-openmaptiles.json
+	--process=resources/process-openmaptiles.lua
+)
+
+run_tilemaker "${area}${suffix}.pmtiles" \
+	"${tilemaker}" "${common[@]}" \
+	--output="${area}${suffix}.pmtiles" --verbose
+
+run_tilemaker "${area}${suffix}-repeat.pmtiles" \
+	"${tilemaker}" "${common[@]}" \
+	--output="${area}${suffix}-repeat.pmtiles" --verbose
+
+run_tilemaker "${area}${suffix}.mbtiles" \
+	"${tilemaker}" "${common[@]}" \
+	--output="${area}${suffix}.mbtiles" \
+	--verbose --store "${store_root}/store${suffix}"
+
+run_tilemaker "${area}${suffix}-repeat.mbtiles" \
+	"${tilemaker}" "${common[@]}" \
+	--output="${area}${suffix}-repeat.mbtiles" \
+	--verbose --store "${store_root}/store${suffix}-repeat"

.github/scripts/verify-generated-tiles.py

@@ -36,6 +36,12 @@ def error(path, message):
     status = 1
 
 
+def workflow_error(message):
+    global status
+    print(f"::error::{message}")
+    status = 1
+
+
 def notice(path, message):
     label = archive_label(path)
     print(f"::notice title={label}::{label}: {message}")
@@ -90,10 +96,15 @@ def build_label(path):
         return str(path.parent)
 
     label = artifact.removeprefix("tile-outputs-")
+    no_clip_cache = label.startswith("no-clip-cache-")
+    if no_clip_cache:
+        label = label.removeprefix("no-clip-cache-")
+
+    mode = ", no clip cache" if no_clip_cache else ""
     if label.endswith("-cmake"):
-        return f"{runner_label(label.removesuffix('-cmake'))} (CMake)"
+        return f"{runner_label(label.removesuffix('-cmake'))} (CMake{mode})"
     if label.endswith("-makefile"):
-        return f"{runner_label(label.removesuffix('-makefile'))} (Makefile)"
+        return f"{runner_label(label.removesuffix('-makefile'))} (Makefile{mode})"
     return artifact
 
 
@@ -115,6 +126,10 @@ def should_validate_archive(path):
     return path.parent.name != "tile-outputs-github-action"
 
 
+def should_semantic_compare_archive(path):
+    return path.parent.name.startswith("tile-outputs-no-clip-cache-")
+
+
 def decompress_payload(data):
     for wbits in (16 + zlib.MAX_WBITS, zlib.MAX_WBITS):
         try:
@@ -846,10 +861,24 @@ def main():
         "PMTiles archive failed verification",
     )
 
-    check_repeat(mbtiles, "mbtiles", mbtiles_decode_semantic_tiles, "MBTiles archive failed verification")
-    check_repeat(pmtiles, "pmtiles", pmtiles_decode_semantic_tiles, "PMTiles archive failed verification")
-    check_cross_runner(mbtiles, "mbtiles", mbtiles_decode_semantic_tiles, "MBTiles archive failed verification")
-    check_cross_runner(pmtiles, "pmtiles", pmtiles_decode_semantic_tiles, "PMTiles archive failed verification")
+    semantic_mbtiles = {
+        path: content for path, content in mbtiles.items()
+        if should_semantic_compare_archive(path)
+    }
+    semantic_pmtiles = {
+        path: content for path, content in pmtiles.items()
+        if should_semantic_compare_archive(path)
+    }
+
+    if not semantic_mbtiles:
+        workflow_error("no no-clip-cache MBTiles artifacts found for semantic verification")
+    if not semantic_pmtiles:
+        workflow_error("no no-clip-cache PMTiles artifacts found for semantic verification")
+
+    check_repeat(semantic_mbtiles, "mbtiles", mbtiles_decode_semantic_tiles, "MBTiles archive failed verification")
+    check_repeat(semantic_pmtiles, "pmtiles", pmtiles_decode_semantic_tiles, "PMTiles archive failed verification")
+    check_cross_runner(semantic_mbtiles, "mbtiles", mbtiles_decode_semantic_tiles, "MBTiles archive failed verification")
+    check_cross_runner(semantic_pmtiles, "pmtiles", pmtiles_decode_semantic_tiles, "PMTiles archive failed verification")
     return status