From 08333356be90b407d8a43262e05985abd68c9378 Mon Sep 17 00:00:00 2001 From: Symmetricity <184246+Symmetricity@users.noreply.github.com> Date: Thu, 4 Jun 2026 18:21:40 +0200 Subject: [PATCH 1/6] Add no-cache semantic tile verification The clip cache can make generated tile semantics depend on which ancestor clips happen to be present during parallel tile generation. That is useful for normal runtime, but it makes CI semantic checks sensitive to cache scheduling instead of the underlying output path being verified. Add a compile-time CMake switch that replaces ClipCache with always-miss no-op methods, then build a second CI binary with that switch enabled. Normal tile outputs are still generated and archive-validated with the cache enabled, while repeat and cross-runner semantic comparisons use only the no-cache artifacts. Co-authored-by: Codex --- .github/scripts/verify-generated-tiles.py | 41 ++++++- .github/workflows/ci.yml | 127 ++++++++++++++++++++++ CMakeLists.txt | 4 + include/clip_cache.h | 12 ++ 4 files changed, 178 insertions(+), 6 deletions(-) diff --git a/.github/scripts/verify-generated-tiles.py b/.github/scripts/verify-generated-tiles.py index 15dde652..4aef81f4 100644 --- a/.github/scripts/verify-generated-tiles.py +++ b/.github/scripts/verify-generated-tiles.py @@ -36,6 +36,12 @@ def error(path, message): status = 1 +def workflow_error(message): + global status + print(f"::error::{message}") + status = 1 + + def notice(path, message): label = archive_label(path) print(f"::notice title={label}::{label}: {message}") @@ -90,10 +96,15 @@ def build_label(path): return str(path.parent) label = artifact.removeprefix("tile-outputs-") + no_clip_cache = label.startswith("no-clip-cache-") + if no_clip_cache: + label = label.removeprefix("no-clip-cache-") + + mode = ", no clip cache" if no_clip_cache else "" if label.endswith("-cmake"): - return f"{runner_label(label.removesuffix('-cmake'))} (CMake)" + return f"{runner_label(label.removesuffix('-cmake'))} (CMake{mode})" if label.endswith("-makefile"): - return f"{runner_label(label.removesuffix('-makefile'))} (Makefile)" + return f"{runner_label(label.removesuffix('-makefile'))} (Makefile{mode})" return artifact @@ -115,6 +126,10 @@ def should_validate_archive(path): return path.parent.name != "tile-outputs-github-action" +def should_semantic_compare_archive(path): + return path.parent.name.startswith("tile-outputs-no-clip-cache-") + + def decompress_payload(data): for wbits in (16 + zlib.MAX_WBITS, zlib.MAX_WBITS): try: @@ -846,10 +861,24 @@ def main(): "PMTiles archive failed verification", ) - check_repeat(mbtiles, "mbtiles", mbtiles_decode_semantic_tiles, "MBTiles archive failed verification") - check_repeat(pmtiles, "pmtiles", pmtiles_decode_semantic_tiles, "PMTiles archive failed verification") - check_cross_runner(mbtiles, "mbtiles", mbtiles_decode_semantic_tiles, "MBTiles archive failed verification") - check_cross_runner(pmtiles, "pmtiles", pmtiles_decode_semantic_tiles, "PMTiles archive failed verification") + semantic_mbtiles = { + path: content for path, content in mbtiles.items() + if should_semantic_compare_archive(path) + } + semantic_pmtiles = { + path: content for path, content in pmtiles.items() + if should_semantic_compare_archive(path) + } + + if not semantic_mbtiles: + workflow_error("no no-clip-cache MBTiles artifacts found for semantic verification") + if not semantic_pmtiles: + workflow_error("no no-clip-cache PMTiles artifacts found for semantic verification") + + check_repeat(semantic_mbtiles, "mbtiles", mbtiles_decode_semantic_tiles, "MBTiles archive failed verification") + check_repeat(semantic_pmtiles, "pmtiles", pmtiles_decode_semantic_tiles, "PMTiles archive failed verification") + check_cross_runner(semantic_mbtiles, "mbtiles", mbtiles_decode_semantic_tiles, "MBTiles archive failed verification") + check_cross_runner(semantic_pmtiles, "pmtiles", pmtiles_decode_semantic_tiles, "PMTiles archive failed verification") return status diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d6fda88..ffc06322 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -318,6 +318,12 @@ jobs: cd ${{ github.workspace }}\build && cmake -DTILEMAKER_BUILD_STATIC=ON -DVCPKG_TARGET_TRIPLET="x64-windows-static-md" -DCMAKE_TOOLCHAIN_FILE="${env:VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake" .. cd ${{ github.workspace }}\build && cmake --build . --config RelWithDebInfo + - name: Build tilemaker without clip cache + run: | + mkdir ${{ github.workspace }}\build-no-clip-cache + cd ${{ github.workspace }}\build-no-clip-cache && cmake -DTILEMAKER_BUILD_STATIC=ON -DTILEMAKER_DISABLE_CLIP_CACHE=ON -DVCPKG_TARGET_TRIPLET="x64-windows-static-md" -DCMAKE_TOOLCHAIN_FILE="${env:VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake" .. + cd ${{ github.workspace }}\build-no-clip-cache && cmake --build . --config RelWithDebInfo + - name: Build openmaptiles-compatible mbtiles files of Liechtenstein run: | $tilemaker = "${{ github.workspace }}\build\RelWithDebInfo\tilemaker.exe" @@ -353,6 +359,41 @@ jobs: ${{ env.AREA }}-repeat.mbtiles ${{ env.AREA }}-repeat.pmtiles + - name: Build semantic-verification tiles without clip cache + run: | + $tilemaker = "${{ github.workspace }}\build-no-clip-cache\RelWithDebInfo\tilemaker.exe" + function Invoke-Tilemaker { + param([string] $Output, [string[]] $Arguments) + Write-Host "::group::Build $Output" + & $tilemaker @Arguments + $status = $LASTEXITCODE + Write-Host "::endgroup::" + if ($status -ne 0) { + Write-Error "tilemaker failed while writing $Output with exit code $status" + exit $status + } + if (!(Test-Path -LiteralPath $Output) -or (Get-Item -LiteralPath $Output).Length -eq 0) { + Write-Error "tilemaker did not create $Output" + exit 1 + } + } + + Invoke-Tilemaker "${{ env.AREA }}-no-clip-cache.pmtiles" @("${{ env.AREA }}.osm.pbf", "--config=resources/config-openmaptiles.json", "--process=resources/process-openmaptiles.lua", "--output=${{ env.AREA }}-no-clip-cache.pmtiles", "--verbose") + Invoke-Tilemaker "${{ env.AREA }}-no-clip-cache-repeat.pmtiles" @("${{ env.AREA }}.osm.pbf", "--config=resources/config-openmaptiles.json", "--process=resources/process-openmaptiles.lua", "--output=${{ env.AREA }}-no-clip-cache-repeat.pmtiles", "--verbose") + Invoke-Tilemaker "${{ env.AREA }}-no-clip-cache.mbtiles" @("${{ env.AREA }}.osm.pbf", "--config=resources/config-openmaptiles.json", "--process=resources/process-openmaptiles.lua", "--output=${{ env.AREA }}-no-clip-cache.mbtiles", "--store", "${{ runner.temp }}\osm_store_no_clip_cache", "--verbose") + Invoke-Tilemaker "${{ env.AREA }}-no-clip-cache-repeat.mbtiles" @("${{ env.AREA }}.osm.pbf", "--config=resources/config-openmaptiles.json", "--process=resources/process-openmaptiles.lua", "--output=${{ env.AREA }}-no-clip-cache-repeat.mbtiles", "--store", "${{ runner.temp }}\osm_store_no_clip_cache_repeat", "--verbose") + + - name: Upload semantic-verification tiles + uses: actions/upload-artifact@v7 + with: + name: tile-outputs-no-clip-cache-windows-cmake + retention-days: 14 + path: | + ${{ env.AREA }}-no-clip-cache.mbtiles + ${{ env.AREA }}-no-clip-cache.pmtiles + ${{ env.AREA }}-no-clip-cache-repeat.mbtiles + ${{ env.AREA }}-no-clip-cache-repeat.pmtiles + - name: 'Upload compiled executable' uses: actions/upload-artifact@v7 with: @@ -412,6 +453,14 @@ jobs: cmake --build . strip tilemaker + - name: Build tilemaker without clip cache + run: | + mkdir build-no-clip-cache + cd build-no-clip-cache + cmake -DTILEMAKER_BUILD_STATIC=ON -DTILEMAKER_DISABLE_CLIP_CACHE=ON -DCMAKE_BUILD_TYPE=Release -DVCPKG_TARGET_TRIPLET="${{ matrix.triplet }}" -DCMAKE_TOOLCHAIN_FILE="$VCPKG_INSTALLATION_ROOT/scripts/buildsystems/vcpkg.cmake" -DCMAKE_CXX_COMPILER=g++ .. + cmake --build . + strip tilemaker + - name: Build openmaptiles-compatible mbtiles files of Liechtenstein run: | run_tilemaker() { @@ -449,6 +498,43 @@ jobs: ${{ env.AREA }}-repeat.mbtiles ${{ env.AREA }}-repeat.pmtiles + - name: Build semantic-verification tiles without clip cache + run: | + run_tilemaker() { + output="$1" + shift + echo "::group::Build ${output}" + if "$@"; then + echo "::endgroup::" + else + status="$?" + echo "::endgroup::" + echo "::error title=Tile generation failed::tilemaker failed while writing ${output} with exit code ${status}" + exit "${status}" + fi + if [ ! -s "${output}" ]; then + echo "::error title=Tile output missing::tilemaker did not create ${output}" + exit 1 + fi + } + + tilemaker="${{ github.workspace }}/build-no-clip-cache/${{ matrix.executable }}" + run_tilemaker "${{ env.AREA }}-no-clip-cache.pmtiles" "${tilemaker}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-no-clip-cache.pmtiles" --verbose + run_tilemaker "${{ env.AREA }}-no-clip-cache-repeat.pmtiles" "${tilemaker}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-no-clip-cache-repeat.pmtiles" --verbose + run_tilemaker "${{ env.AREA }}-no-clip-cache.mbtiles" "${tilemaker}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-no-clip-cache.mbtiles" --verbose --store "${{ runner.temp }}/store-no-clip-cache" + run_tilemaker "${{ env.AREA }}-no-clip-cache-repeat.mbtiles" "${tilemaker}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-no-clip-cache-repeat.mbtiles" --verbose --store "${{ runner.temp }}/store-no-clip-cache-repeat" + + - name: Upload semantic-verification tiles + uses: actions/upload-artifact@v7 + with: + name: tile-outputs-no-clip-cache-${{ matrix.os }}-cmake + retention-days: 14 + path: | + ${{ env.AREA }}-no-clip-cache.mbtiles + ${{ env.AREA }}-no-clip-cache.pmtiles + ${{ env.AREA }}-no-clip-cache-repeat.mbtiles + ${{ env.AREA }}-no-clip-cache-repeat.pmtiles + - name: 'Upload compiled executable' uses: actions/upload-artifact@v7 with: @@ -533,6 +619,47 @@ jobs: ${{ env.AREA }}-repeat.mbtiles ${{ env.AREA }}-repeat.pmtiles + - name: Build tilemaker without clip cache + run: | + make clean + make -j4 CONFIG=-DTILEMAKER_DISABLE_CLIP_CACHE tilemaker + + - name: Build semantic-verification tiles without clip cache + run: | + run_tilemaker() { + output="$1" + shift + echo "::group::Build ${output}" + if "$@"; then + echo "::endgroup::" + else + status="$?" + echo "::endgroup::" + echo "::error title=Tile generation failed::tilemaker failed while writing ${output} with exit code ${status}" + exit "${status}" + fi + if [ ! -s "${output}" ]; then + echo "::error title=Tile output missing::tilemaker did not create ${output}" + exit 1 + fi + } + + run_tilemaker "${{ env.AREA }}-no-clip-cache.pmtiles" ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-no-clip-cache.pmtiles" --verbose + run_tilemaker "${{ env.AREA }}-no-clip-cache-repeat.pmtiles" ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-no-clip-cache-repeat.pmtiles" --verbose + run_tilemaker "${{ env.AREA }}-no-clip-cache.mbtiles" ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-no-clip-cache.mbtiles" --verbose --store "${{ runner.temp }}/store-no-clip-cache" + run_tilemaker "${{ env.AREA }}-no-clip-cache-repeat.mbtiles" ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-no-clip-cache-repeat.mbtiles" --verbose --store "${{ runner.temp }}/store-no-clip-cache-repeat" + + - name: Upload semantic-verification tiles + uses: actions/upload-artifact@v7 + with: + name: tile-outputs-no-clip-cache-${{ matrix.os }}-makefile + retention-days: 14 + path: | + ${{ env.AREA }}-no-clip-cache.mbtiles + ${{ env.AREA }}-no-clip-cache.pmtiles + ${{ env.AREA }}-no-clip-cache-repeat.mbtiles + ${{ env.AREA }}-no-clip-cache-repeat.pmtiles + Github-Action: name: Generate mbtiles with Github Action diff --git a/CMakeLists.txt b/CMakeLists.txt index c67b73ad..ffadcb96 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,6 +7,7 @@ endif() project(tilemaker) OPTION(TILEMAKER_BUILD_STATIC "Attempt to link dependencies static" OFF) +OPTION(TILEMAKER_DISABLE_CLIP_CACHE "Disable geometry clip cache" OFF) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") @@ -144,6 +145,9 @@ file(GLOB tilemaker_src_files src/way_stores.cpp ) add_executable(tilemaker ${tilemaker_src_files}) +IF (TILEMAKER_DISABLE_CLIP_CACHE) + target_compile_definitions(tilemaker PRIVATE TILEMAKER_DISABLE_CLIP_CACHE) +ENDIF () target_include_directories(tilemaker PRIVATE include) target_include_directories(tilemaker PRIVATE ${CMAKE_BINARY_DIR}) # for generated files target_link_libraries(tilemaker diff --git a/include/clip_cache.h b/include/clip_cache.h index 353356dd..bad7c246 100644 --- a/include/clip_cache.h +++ b/include/clip_cache.h @@ -11,6 +11,17 @@ class TileBbox; template class ClipCache { public: +#ifdef TILEMAKER_DISABLE_CLIP_CACHE + ClipCache(size_t, unsigned int) { + } + + const std::shared_ptr get(uint, TileCoordinate, TileCoordinate, NodeID) const{ + return nullptr; + } + + void add(const TileBbox&, const NodeID, const T&) { + } +#else ClipCache(size_t threadNum, unsigned int baseZoom): baseZoom(baseZoom), clipCache(threadNum * 16), @@ -74,6 +85,7 @@ class ClipCache { std::vector, std::shared_ptr>> clipCache; mutable std::vector clipCacheMutex; std::vector clipCacheSize; +#endif }; #endif From f06e6fad8c48dcf246c9cccd8c8d21f020af56ab Mon Sep 17 00:00:00 2001 From: Symmetricity <184246+Symmetricity@users.noreply.github.com> Date: Thu, 4 Jun 2026 18:27:49 +0200 Subject: [PATCH 2/6] Use a vcpkg manifest for CI dependencies The CMake jobs listed the same vcpkg packages directly in the workflow, which made dependency changes harder to review and made the binary cache depend on unrelated workflow edits. Move the dependency list into vcpkg.json with a pinned builtin baseline, use manifest-mode installs in the Windows and Linux vcpkg jobs, and key the binary cache on the manifest files. Add Dependabot coverage for the vcpkg baseline so dependency refreshes are proposed as focused changes. Co-authored-by: Codex --- .github/dependabot.yml | 6 ++++++ .github/workflows/ci.yml | 19 +++++++++++-------- vcpkg.json | 27 +++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 8 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 vcpkg.json diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..654d43b2 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "vcpkg" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ffc06322..278c1ae1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,6 +62,9 @@ jobs: CMakeLists.txt|Makefile|Dockerfile|action.yml) runtime=true ;; + vcpkg.json|vcpkg-configuration.json) + runtime=true + ;; get-coastline.sh|get-landcover.sh|get-monaco.sh) runtime=true ;; @@ -305,23 +308,23 @@ jobs: uses: actions/cache@v5 with: path: ${{ github.workspace }}\vcpkg-binary-cache - key: vcpkg-binary-${{ runner.os }}-${{ steps.runner-image.outputs.image-os }}-${{ steps.runner-image.outputs.image-version }}-x64-windows-static-md-${{ hashFiles('.github/workflows/ci.yml') }} + key: vcpkg-binary-${{ runner.os }}-${{ steps.runner-image.outputs.image-os }}-${{ steps.runner-image.outputs.image-version }}-x64-windows-static-md-${{ hashFiles('vcpkg.json', 'vcpkg-configuration.json') }} restore-keys: vcpkg-binary-${{ runner.os }}-${{ steps.runner-image.outputs.image-os }}-${{ steps.runner-image.outputs.image-version }}-x64-windows-static-md- - name: Build dependencies run: | - vcpkg install --triplet=x64-windows-static-md lua shapelib zlib protobuf[zlib] sqlite3 boost-program-options boost-filesystem boost-geometry boost-system boost-asio boost-interprocess boost-iostreams boost-sort rapidjson + vcpkg install --triplet=x64-windows-static-md - name: Build tilemaker run: | mkdir ${{ github.workspace }}\build - cd ${{ github.workspace }}\build && cmake -DTILEMAKER_BUILD_STATIC=ON -DVCPKG_TARGET_TRIPLET="x64-windows-static-md" -DCMAKE_TOOLCHAIN_FILE="${env:VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake" .. + cd ${{ github.workspace }}\build && cmake -DTILEMAKER_BUILD_STATIC=ON -DVCPKG_TARGET_TRIPLET="x64-windows-static-md" -DVCPKG_INSTALLED_DIR="${{ github.workspace }}\vcpkg_installed" -DCMAKE_TOOLCHAIN_FILE="${env:VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake" .. cd ${{ github.workspace }}\build && cmake --build . --config RelWithDebInfo - name: Build tilemaker without clip cache run: | mkdir ${{ github.workspace }}\build-no-clip-cache - cd ${{ github.workspace }}\build-no-clip-cache && cmake -DTILEMAKER_BUILD_STATIC=ON -DTILEMAKER_DISABLE_CLIP_CACHE=ON -DVCPKG_TARGET_TRIPLET="x64-windows-static-md" -DCMAKE_TOOLCHAIN_FILE="${env:VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake" .. + cd ${{ github.workspace }}\build-no-clip-cache && cmake -DTILEMAKER_BUILD_STATIC=ON -DTILEMAKER_DISABLE_CLIP_CACHE=ON -DVCPKG_TARGET_TRIPLET="x64-windows-static-md" -DVCPKG_INSTALLED_DIR="${{ github.workspace }}\vcpkg_installed" -DCMAKE_TOOLCHAIN_FILE="${env:VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake" .. cd ${{ github.workspace }}\build-no-clip-cache && cmake --build . --config RelWithDebInfo - name: Build openmaptiles-compatible mbtiles files of Liechtenstein @@ -438,18 +441,18 @@ jobs: uses: actions/cache@v5 with: path: ${{ github.workspace }}/vcpkg-binary-cache - key: vcpkg-binary-${{ runner.os }}-${{ steps.runner-image.outputs.image-os }}-${{ steps.runner-image.outputs.image-version }}-${{ matrix.triplet }}-${{ hashFiles('.github/workflows/ci.yml') }} + key: vcpkg-binary-${{ runner.os }}-${{ steps.runner-image.outputs.image-os }}-${{ steps.runner-image.outputs.image-version }}-${{ matrix.triplet }}-${{ hashFiles('vcpkg.json', 'vcpkg-configuration.json') }} restore-keys: vcpkg-binary-${{ runner.os }}-${{ steps.runner-image.outputs.image-os }}-${{ steps.runner-image.outputs.image-version }}-${{ matrix.triplet }}- - name: Build dependencies run: | - vcpkg install --triplet="${{ matrix.triplet }}" lua shapelib zlib protobuf[zlib] sqlite3 boost-program-options boost-filesystem boost-geometry boost-system boost-asio boost-interprocess boost-iostreams boost-sort rapidjson + vcpkg install --triplet="${{ matrix.triplet }}" - name: Build tilemaker run: | mkdir build cd build - cmake -DTILEMAKER_BUILD_STATIC=ON -DCMAKE_BUILD_TYPE=Release -DVCPKG_TARGET_TRIPLET="${{ matrix.triplet }}" -DCMAKE_TOOLCHAIN_FILE="$VCPKG_INSTALLATION_ROOT/scripts/buildsystems/vcpkg.cmake" -DCMAKE_CXX_COMPILER=g++ .. + cmake -DTILEMAKER_BUILD_STATIC=ON -DCMAKE_BUILD_TYPE=Release -DVCPKG_TARGET_TRIPLET="${{ matrix.triplet }}" -DVCPKG_INSTALLED_DIR="${{ github.workspace }}/vcpkg_installed" -DCMAKE_TOOLCHAIN_FILE="$VCPKG_INSTALLATION_ROOT/scripts/buildsystems/vcpkg.cmake" -DCMAKE_CXX_COMPILER=g++ .. cmake --build . strip tilemaker @@ -457,7 +460,7 @@ jobs: run: | mkdir build-no-clip-cache cd build-no-clip-cache - cmake -DTILEMAKER_BUILD_STATIC=ON -DTILEMAKER_DISABLE_CLIP_CACHE=ON -DCMAKE_BUILD_TYPE=Release -DVCPKG_TARGET_TRIPLET="${{ matrix.triplet }}" -DCMAKE_TOOLCHAIN_FILE="$VCPKG_INSTALLATION_ROOT/scripts/buildsystems/vcpkg.cmake" -DCMAKE_CXX_COMPILER=g++ .. + cmake -DTILEMAKER_BUILD_STATIC=ON -DTILEMAKER_DISABLE_CLIP_CACHE=ON -DCMAKE_BUILD_TYPE=Release -DVCPKG_TARGET_TRIPLET="${{ matrix.triplet }}" -DVCPKG_INSTALLED_DIR="${{ github.workspace }}/vcpkg_installed" -DCMAKE_TOOLCHAIN_FILE="$VCPKG_INSTALLATION_ROOT/scripts/buildsystems/vcpkg.cmake" -DCMAKE_CXX_COMPILER=g++ .. cmake --build . strip tilemaker diff --git a/vcpkg.json b/vcpkg.json new file mode 100644 index 00000000..b67dd5a8 --- /dev/null +++ b/vcpkg.json @@ -0,0 +1,27 @@ +{ + "$schema": "https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg.schema.json", + "name": "tilemaker", + "version-string": "3.1.0", + "builtin-baseline": "cea592f4772491abdb7c483387a59ea89889f4be", + "dependencies": [ + "lua", + "shapelib", + "zlib", + { + "name": "protobuf", + "features": [ + "zlib" + ] + }, + "sqlite3", + "boost-program-options", + "boost-filesystem", + "boost-geometry", + "boost-system", + "boost-asio", + "boost-interprocess", + "boost-iostreams", + "boost-sort", + "rapidjson" + ] +} From 7f2c98babea8aac3bd35e85d07ed883e7dd741e8 Mon Sep 17 00:00:00 2001 From: Symmetricity <184246+Symmetricity@users.noreply.github.com> Date: Thu, 4 Jun 2026 18:48:56 +0200 Subject: [PATCH 3/6] Improve CI cache reuse and tile checks Add compiler caching to the CMake and Makefile jobs so repeated CI runs can reuse object files in addition to the vcpkg binary cache. Move repeated tile generation commands into shared Bash and PowerShell helpers, then require uploaded artifacts to exist and avoid spending time recompressing generated tile outputs. Add a configure-only CMake smoke job and parse checks for PowerShell CI scripts so common workflow or build-setup mistakes fail before the full matrix starts. Co-authored-by: Codex --- .github/dependabot.yml | 5 + .github/scripts/generate-tiles.ps1 | 68 ++++++++ .github/scripts/generate-tiles.sh | 54 +++++++ .github/workflows/ci.yml | 244 +++++++++++++---------------- 4 files changed, 232 insertions(+), 139 deletions(-) create mode 100644 .github/scripts/generate-tiles.ps1 create mode 100755 .github/scripts/generate-tiles.sh diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 654d43b2..3cf52b82 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,5 +1,10 @@ version: 2 updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + - package-ecosystem: "vcpkg" directory: "/" schedule: diff --git a/.github/scripts/generate-tiles.ps1 b/.github/scripts/generate-tiles.ps1 new file mode 100644 index 00000000..9ada6fdd --- /dev/null +++ b/.github/scripts/generate-tiles.ps1 @@ -0,0 +1,68 @@ +param( + [Parameter(Mandatory = $true)] + [string] $Tilemaker, + + [Parameter(Mandatory = $true)] + [string] $Area, + + [Parameter(Mandatory = $true)] + [string] $OutputSuffix, + + [Parameter(Mandatory = $true)] + [string] $StoreRoot +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +function Invoke-Tilemaker { + param( + [Parameter(Mandatory = $true)] + [string] $Output, + + [Parameter(Mandatory = $true)] + [string[]] $Arguments + ) + + Write-Host "::group::Build $Output" + & $Tilemaker @Arguments + $status = $LASTEXITCODE + Write-Host "::endgroup::" + if ($status -ne 0) { + Write-Error "tilemaker failed while writing $Output with exit code $status" + exit $status + } + if (!(Test-Path -LiteralPath $Output) -or (Get-Item -LiteralPath $Output).Length -eq 0) { + Write-Error "tilemaker did not create $Output" + exit 1 + } +} + +$inputPbf = "${Area}.osm.pbf" +$common = @( + $inputPbf, + "--config=resources/config-openmaptiles.json", + "--process=resources/process-openmaptiles.lua" +) + +Invoke-Tilemaker "${Area}${OutputSuffix}.pmtiles" ` + ($common + @("--output=${Area}${OutputSuffix}.pmtiles", "--verbose")) + +Invoke-Tilemaker "${Area}${OutputSuffix}-repeat.pmtiles" ` + ($common + @("--output=${Area}${OutputSuffix}-repeat.pmtiles", "--verbose")) + +Invoke-Tilemaker "${Area}${OutputSuffix}.mbtiles" ` + ($common + @( + "--output=${Area}${OutputSuffix}.mbtiles", + "--store", + (Join-Path $StoreRoot "store${OutputSuffix}"), + "--verbose" + )) + +Invoke-Tilemaker "${Area}${OutputSuffix}-repeat.mbtiles" ` + ($common + @( + "--output=${Area}${OutputSuffix}-repeat.mbtiles", + "--store", + (Join-Path $StoreRoot "store${OutputSuffix}-repeat"), + "--verbose" + )) diff --git a/.github/scripts/generate-tiles.sh b/.github/scripts/generate-tiles.sh new file mode 100755 index 00000000..47062a1e --- /dev/null +++ b/.github/scripts/generate-tiles.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [ "$#" -ne 4 ]; then + echo "usage: $0 " >&2 + exit 2 +fi + +tilemaker="$1" +area="$2" +suffix="$3" +store_root="$4" + +run_tilemaker() { + output="$1" + shift + echo "::group::Build ${output}" + if "$@"; then + echo "::endgroup::" + else + status="$?" + echo "::endgroup::" + echo "::error title=Tile generation failed::tilemaker failed while writing ${output} with exit code ${status}" + exit "${status}" + fi + if [ ! -s "${output}" ]; then + echo "::error title=Tile output missing::tilemaker did not create ${output}" + exit 1 + fi +} + +common=( + "${area}.osm.pbf" + --config=resources/config-openmaptiles.json + --process=resources/process-openmaptiles.lua +) + +run_tilemaker "${area}${suffix}.pmtiles" \ + "${tilemaker}" "${common[@]}" \ + --output="${area}${suffix}.pmtiles" --verbose + +run_tilemaker "${area}${suffix}-repeat.pmtiles" \ + "${tilemaker}" "${common[@]}" \ + --output="${area}${suffix}-repeat.pmtiles" --verbose + +run_tilemaker "${area}${suffix}.mbtiles" \ + "${tilemaker}" "${common[@]}" \ + --output="${area}${suffix}.mbtiles" \ + --verbose --store "${store_root}/store${suffix}" + +run_tilemaker "${area}${suffix}-repeat.mbtiles" \ + "${tilemaker}" "${common[@]}" \ + --output="${area}${suffix}-repeat.mbtiles" \ + --verbose --store "${store_root}/store${suffix}-repeat" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 278c1ae1..e627452c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,6 +59,9 @@ jobs: .github/workflows/ci.yml) runtime=true ;; + .github/scripts/*) + runtime=true + ;; CMakeLists.txt|Makefile|Dockerfile|action.yml) runtime=true ;; @@ -141,6 +144,33 @@ jobs: git ls-files -z -- '.github/scripts/*.py' | xargs -0 -r python3 -m py_compile + - name: Validate PowerShell CI scripts + run: | + while IFS= read -r -d '' file; do + SCRIPT_PATH="$file" pwsh -NoProfile -Command "\$ErrorActionPreference = 'Stop'; [scriptblock]::Create((Get-Content -Raw -LiteralPath \$env:SCRIPT_PATH)) | Out-Null" + done < <(git ls-files -z -- '.github/scripts/*.ps1') + + cmake-configure: + name: CMake configure smoke + runs-on: ubuntu-latest + timeout-minutes: 10 + needs: + - changes + - static-validation + if: ${{ needs.changes.outputs.runtime == 'true' }} + + steps: + - uses: actions/checkout@v6 + + - name: Install CMake configure dependencies + run: | + sudo apt-get update + sudo apt-get install -y build-essential cmake libboost-dev libboost-filesystem-dev libboost-program-options-dev libboost-system-dev lua5.4 liblua5.4-dev libshp-dev libsqlite3-dev rapidjson-dev zlib1g-dev + + - name: Configure CMake + run: | + cmake -S . -B build-check -DCMAKE_BUILD_TYPE=Release + download-pbf: name: Download PBF fixture runs-on: ubuntu-latest @@ -148,6 +178,7 @@ jobs: needs: - changes - static-validation + - cmake-configure if: ${{ needs.changes.outputs.runtime == 'true' }} outputs: pbf-available: ${{ steps.fixture.outputs.pbf-available }} @@ -276,6 +307,8 @@ jobs: with: name: pbf-fixture retention-days: 1 + if-no-files-found: error + compression-level: 0 path: ${{ env.AREA }}.osm.pbf Windows-Build: @@ -286,10 +319,14 @@ jobs: if: ${{ needs.download-pbf.outputs.pbf-available == 'true' }} env: VCPKG_BINARY_SOURCES: clear;files,${{ github.workspace }}\vcpkg-binary-cache,readwrite + SCCACHE_GHA_ENABLED: "true" steps: - uses: actions/checkout@v6 + - name: Install sccache + uses: mozilla-actions/sccache-action@v0.0.9 + - name: Download PBF fixture uses: actions/download-artifact@v8 with: @@ -318,44 +355,27 @@ jobs: - name: Build tilemaker run: | mkdir ${{ github.workspace }}\build - cd ${{ github.workspace }}\build && cmake -DTILEMAKER_BUILD_STATIC=ON -DVCPKG_TARGET_TRIPLET="x64-windows-static-md" -DVCPKG_INSTALLED_DIR="${{ github.workspace }}\vcpkg_installed" -DCMAKE_TOOLCHAIN_FILE="${env:VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake" .. + cd ${{ github.workspace }}\build && cmake -DTILEMAKER_BUILD_STATIC=ON -DVCPKG_TARGET_TRIPLET="x64-windows-static-md" -DVCPKG_INSTALLED_DIR="${{ github.workspace }}\vcpkg_installed" -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_TOOLCHAIN_FILE="${env:VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake" .. cd ${{ github.workspace }}\build && cmake --build . --config RelWithDebInfo - name: Build tilemaker without clip cache run: | mkdir ${{ github.workspace }}\build-no-clip-cache - cd ${{ github.workspace }}\build-no-clip-cache && cmake -DTILEMAKER_BUILD_STATIC=ON -DTILEMAKER_DISABLE_CLIP_CACHE=ON -DVCPKG_TARGET_TRIPLET="x64-windows-static-md" -DVCPKG_INSTALLED_DIR="${{ github.workspace }}\vcpkg_installed" -DCMAKE_TOOLCHAIN_FILE="${env:VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake" .. - cd ${{ github.workspace }}\build-no-clip-cache && cmake --build . --config RelWithDebInfo + cd ${{ github.workspace }}\build-no-clip-cache && cmake -DTILEMAKER_BUILD_STATIC=ON -DTILEMAKER_DISABLE_CLIP_CACHE=ON -DVCPKG_TARGET_TRIPLET="x64-windows-static-md" -DVCPKG_INSTALLED_DIR="${{ github.workspace }}\vcpkg_installed" -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_TOOLCHAIN_FILE="${env:VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake" .. + cd ${{ github.workspace }}\build-no-clip-cache && cmake --build . --config RelWithDebInfo --target tilemaker - name: Build openmaptiles-compatible mbtiles files of Liechtenstein run: | $tilemaker = "${{ github.workspace }}\build\RelWithDebInfo\tilemaker.exe" - function Invoke-Tilemaker { - param([string] $Output, [string[]] $Arguments) - Write-Host "::group::Build $Output" - & $tilemaker @Arguments - $status = $LASTEXITCODE - Write-Host "::endgroup::" - if ($status -ne 0) { - Write-Error "tilemaker failed while writing $Output with exit code $status" - exit $status - } - if (!(Test-Path -LiteralPath $Output) -or (Get-Item -LiteralPath $Output).Length -eq 0) { - Write-Error "tilemaker did not create $Output" - exit 1 - } - } - - Invoke-Tilemaker "${{ env.AREA }}.pmtiles" @("${{ env.AREA }}.osm.pbf", "--config=resources/config-openmaptiles.json", "--process=resources/process-openmaptiles.lua", "--output=${{ env.AREA }}.pmtiles", "--verbose") - Invoke-Tilemaker "${{ env.AREA }}-repeat.pmtiles" @("${{ env.AREA }}.osm.pbf", "--config=resources/config-openmaptiles.json", "--process=resources/process-openmaptiles.lua", "--output=${{ env.AREA }}-repeat.pmtiles", "--verbose") - Invoke-Tilemaker "${{ env.AREA }}.mbtiles" @("${{ env.AREA }}.osm.pbf", "--config=resources/config-openmaptiles.json", "--process=resources/process-openmaptiles.lua", "--output=${{ env.AREA }}.mbtiles", "--store", "${{ runner.temp }}\osm_store", "--verbose") - Invoke-Tilemaker "${{ env.AREA }}-repeat.mbtiles" @("${{ env.AREA }}.osm.pbf", "--config=resources/config-openmaptiles.json", "--process=resources/process-openmaptiles.lua", "--output=${{ env.AREA }}-repeat.mbtiles", "--store", "${{ runner.temp }}\osm_store_repeat", "--verbose") + .\.github\scripts\generate-tiles.ps1 -Tilemaker $tilemaker -Area "${{ env.AREA }}" -OutputSuffix "" -StoreRoot "${{ runner.temp }}" - name: Upload generated tiles uses: actions/upload-artifact@v7 with: name: tile-outputs-windows-cmake retention-days: 14 + if-no-files-found: error + compression-level: 0 path: | ${{ env.AREA }}.mbtiles ${{ env.AREA }}.pmtiles @@ -365,32 +385,15 @@ jobs: - name: Build semantic-verification tiles without clip cache run: | $tilemaker = "${{ github.workspace }}\build-no-clip-cache\RelWithDebInfo\tilemaker.exe" - function Invoke-Tilemaker { - param([string] $Output, [string[]] $Arguments) - Write-Host "::group::Build $Output" - & $tilemaker @Arguments - $status = $LASTEXITCODE - Write-Host "::endgroup::" - if ($status -ne 0) { - Write-Error "tilemaker failed while writing $Output with exit code $status" - exit $status - } - if (!(Test-Path -LiteralPath $Output) -or (Get-Item -LiteralPath $Output).Length -eq 0) { - Write-Error "tilemaker did not create $Output" - exit 1 - } - } - - Invoke-Tilemaker "${{ env.AREA }}-no-clip-cache.pmtiles" @("${{ env.AREA }}.osm.pbf", "--config=resources/config-openmaptiles.json", "--process=resources/process-openmaptiles.lua", "--output=${{ env.AREA }}-no-clip-cache.pmtiles", "--verbose") - Invoke-Tilemaker "${{ env.AREA }}-no-clip-cache-repeat.pmtiles" @("${{ env.AREA }}.osm.pbf", "--config=resources/config-openmaptiles.json", "--process=resources/process-openmaptiles.lua", "--output=${{ env.AREA }}-no-clip-cache-repeat.pmtiles", "--verbose") - Invoke-Tilemaker "${{ env.AREA }}-no-clip-cache.mbtiles" @("${{ env.AREA }}.osm.pbf", "--config=resources/config-openmaptiles.json", "--process=resources/process-openmaptiles.lua", "--output=${{ env.AREA }}-no-clip-cache.mbtiles", "--store", "${{ runner.temp }}\osm_store_no_clip_cache", "--verbose") - Invoke-Tilemaker "${{ env.AREA }}-no-clip-cache-repeat.mbtiles" @("${{ env.AREA }}.osm.pbf", "--config=resources/config-openmaptiles.json", "--process=resources/process-openmaptiles.lua", "--output=${{ env.AREA }}-no-clip-cache-repeat.mbtiles", "--store", "${{ runner.temp }}\osm_store_no_clip_cache_repeat", "--verbose") + .\.github\scripts\generate-tiles.ps1 -Tilemaker $tilemaker -Area "${{ env.AREA }}" -OutputSuffix "-no-clip-cache" -StoreRoot "${{ runner.temp }}" - name: Upload semantic-verification tiles uses: actions/upload-artifact@v7 with: name: tile-outputs-no-clip-cache-windows-cmake retention-days: 14 + if-no-files-found: error + compression-level: 0 path: | ${{ env.AREA }}-no-clip-cache.mbtiles ${{ env.AREA }}-no-clip-cache.pmtiles @@ -402,11 +405,17 @@ jobs: with: name: tilemaker-windows retention-days: 14 + if-no-files-found: error path: | ${{ github.workspace }}\resources ${{ github.workspace }}\build\RelWithDebInfo\tilemaker.exe ${{ github.workspace }}\build\RelWithDebInfo\*.pdb + - name: Show sccache stats + if: always() + run: | + sccache --show-stats + unix-build: strategy: fail-fast: false @@ -422,6 +431,7 @@ jobs: if: ${{ needs.download-pbf.outputs.pbf-available == 'true' }} env: VCPKG_BINARY_SOURCES: clear;files,${{ github.workspace }}/vcpkg-binary-cache,readwrite + CCACHE_DIR: ${{ github.workspace }}/.ccache steps: - uses: actions/checkout@v6 @@ -437,6 +447,18 @@ jobs: echo "image-os=${ImageOS:-unknown}" >> "$GITHUB_OUTPUT" echo "image-version=${ImageVersion:-unknown}" >> "$GITHUB_OUTPUT" + - name: Install ccache + run: | + sudo apt-get update + sudo apt-get install -y ccache + + - name: Enable ccache + uses: actions/cache@v5 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ runner.os }}-${{ steps.runner-image.outputs.image-os }}-${{ steps.runner-image.outputs.image-version }}-${{ matrix.triplet }}-${{ hashFiles('CMakeLists.txt', 'include/**', 'src/**') }} + restore-keys: ccache-${{ runner.os }}-${{ steps.runner-image.outputs.image-os }}-${{ steps.runner-image.outputs.image-version }}-${{ matrix.triplet }}- + - name: Enable vcpkg binary cache uses: actions/cache@v5 with: @@ -450,51 +472,35 @@ jobs: - name: Build tilemaker run: | + ccache --zero-stats mkdir build cd build - cmake -DTILEMAKER_BUILD_STATIC=ON -DCMAKE_BUILD_TYPE=Release -DVCPKG_TARGET_TRIPLET="${{ matrix.triplet }}" -DVCPKG_INSTALLED_DIR="${{ github.workspace }}/vcpkg_installed" -DCMAKE_TOOLCHAIN_FILE="$VCPKG_INSTALLATION_ROOT/scripts/buildsystems/vcpkg.cmake" -DCMAKE_CXX_COMPILER=g++ .. + cmake -DTILEMAKER_BUILD_STATIC=ON -DCMAKE_BUILD_TYPE=Release -DVCPKG_TARGET_TRIPLET="${{ matrix.triplet }}" -DVCPKG_INSTALLED_DIR="${{ github.workspace }}/vcpkg_installed" -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_TOOLCHAIN_FILE="$VCPKG_INSTALLATION_ROOT/scripts/buildsystems/vcpkg.cmake" -DCMAKE_CXX_COMPILER=g++ .. cmake --build . strip tilemaker + ccache --show-stats - name: Build tilemaker without clip cache run: | mkdir build-no-clip-cache cd build-no-clip-cache - cmake -DTILEMAKER_BUILD_STATIC=ON -DTILEMAKER_DISABLE_CLIP_CACHE=ON -DCMAKE_BUILD_TYPE=Release -DVCPKG_TARGET_TRIPLET="${{ matrix.triplet }}" -DVCPKG_INSTALLED_DIR="${{ github.workspace }}/vcpkg_installed" -DCMAKE_TOOLCHAIN_FILE="$VCPKG_INSTALLATION_ROOT/scripts/buildsystems/vcpkg.cmake" -DCMAKE_CXX_COMPILER=g++ .. - cmake --build . + cmake -DTILEMAKER_BUILD_STATIC=ON -DTILEMAKER_DISABLE_CLIP_CACHE=ON -DCMAKE_BUILD_TYPE=Release -DVCPKG_TARGET_TRIPLET="${{ matrix.triplet }}" -DVCPKG_INSTALLED_DIR="${{ github.workspace }}/vcpkg_installed" -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_TOOLCHAIN_FILE="$VCPKG_INSTALLATION_ROOT/scripts/buildsystems/vcpkg.cmake" -DCMAKE_CXX_COMPILER=g++ .. + cmake --build . --target tilemaker strip tilemaker + ccache --show-stats - name: Build openmaptiles-compatible mbtiles files of Liechtenstein run: | - run_tilemaker() { - output="$1" - shift - echo "::group::Build ${output}" - if "$@"; then - echo "::endgroup::" - else - status="$?" - echo "::endgroup::" - echo "::error title=Tile generation failed::tilemaker failed while writing ${output} with exit code ${status}" - exit "${status}" - fi - if [ ! -s "${output}" ]; then - echo "::error title=Tile output missing::tilemaker did not create ${output}" - exit 1 - fi - } - tilemaker="${{ github.workspace }}/build/${{ matrix.executable }}" - run_tilemaker "${{ env.AREA }}.pmtiles" "${tilemaker}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}.pmtiles" --verbose - run_tilemaker "${{ env.AREA }}-repeat.pmtiles" "${tilemaker}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-repeat.pmtiles" --verbose - run_tilemaker "${{ env.AREA }}.mbtiles" "${tilemaker}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}.mbtiles" --verbose --store "${{ runner.temp }}/store" - run_tilemaker "${{ env.AREA }}-repeat.mbtiles" "${tilemaker}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-repeat.mbtiles" --verbose --store "${{ runner.temp }}/store-repeat" + .github/scripts/generate-tiles.sh "${tilemaker}" "${{ env.AREA }}" "" "${{ runner.temp }}" - name: Upload generated tiles uses: actions/upload-artifact@v7 with: name: tile-outputs-${{ matrix.os }}-cmake retention-days: 14 + if-no-files-found: error + compression-level: 0 path: | ${{ env.AREA }}.mbtiles ${{ env.AREA }}.pmtiles @@ -503,35 +509,16 @@ jobs: - name: Build semantic-verification tiles without clip cache run: | - run_tilemaker() { - output="$1" - shift - echo "::group::Build ${output}" - if "$@"; then - echo "::endgroup::" - else - status="$?" - echo "::endgroup::" - echo "::error title=Tile generation failed::tilemaker failed while writing ${output} with exit code ${status}" - exit "${status}" - fi - if [ ! -s "${output}" ]; then - echo "::error title=Tile output missing::tilemaker did not create ${output}" - exit 1 - fi - } - tilemaker="${{ github.workspace }}/build-no-clip-cache/${{ matrix.executable }}" - run_tilemaker "${{ env.AREA }}-no-clip-cache.pmtiles" "${tilemaker}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-no-clip-cache.pmtiles" --verbose - run_tilemaker "${{ env.AREA }}-no-clip-cache-repeat.pmtiles" "${tilemaker}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-no-clip-cache-repeat.pmtiles" --verbose - run_tilemaker "${{ env.AREA }}-no-clip-cache.mbtiles" "${tilemaker}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-no-clip-cache.mbtiles" --verbose --store "${{ runner.temp }}/store-no-clip-cache" - run_tilemaker "${{ env.AREA }}-no-clip-cache-repeat.mbtiles" "${tilemaker}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-no-clip-cache-repeat.mbtiles" --verbose --store "${{ runner.temp }}/store-no-clip-cache-repeat" + .github/scripts/generate-tiles.sh "${tilemaker}" "${{ env.AREA }}" "-no-clip-cache" "${{ runner.temp }}" - name: Upload semantic-verification tiles uses: actions/upload-artifact@v7 with: name: tile-outputs-no-clip-cache-${{ matrix.os }}-cmake retention-days: 14 + if-no-files-found: error + compression-level: 0 path: | ${{ env.AREA }}-no-clip-cache.mbtiles ${{ env.AREA }}-no-clip-cache.pmtiles @@ -543,6 +530,7 @@ jobs: with: name: tilemaker-${{ matrix.os }} retention-days: 14 + if-no-files-found: error path: | ${{ github.workspace }}/resources ${{ github.workspace }}/build/${{ matrix.executable }} @@ -561,6 +549,8 @@ jobs: timeout-minutes: 30 needs: download-pbf if: ${{ needs.download-pbf.outputs.pbf-available == 'true' }} + env: + CCACHE_DIR: ${{ github.workspace }}/.ccache steps: - uses: actions/checkout@v6 @@ -573,49 +563,39 @@ jobs: - name: Install Linux dependencies if: ${{ matrix.os == 'ubuntu-22.04' }} run: | - sudo apt install build-essential libboost-dev libboost-filesystem-dev libboost-iostreams-dev libboost-program-options-dev libboost-system-dev luajit libluajit-5.1-dev liblua5.1-0-dev libshp-dev libsqlite3-dev rapidjson-dev zlib1g-dev + sudo apt install build-essential ccache libboost-dev libboost-filesystem-dev libboost-iostreams-dev libboost-program-options-dev libboost-system-dev luajit libluajit-5.1-dev liblua5.1-0-dev libshp-dev libsqlite3-dev rapidjson-dev zlib1g-dev - name: Install Mac OS X dependencies if: ${{ matrix.os == 'macos-latest' || matrix.os == 'macos-14' }} run: | c++ --version - brew install boost lua shapelib rapidjson + brew install boost ccache lua shapelib rapidjson + + - name: Enable ccache + uses: actions/cache@v5 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ runner.os }}-${{ matrix.os }}-makefile-${{ hashFiles('Makefile', 'include/**', 'src/**', 'test/**') }} + restore-keys: ccache-${{ runner.os }}-${{ matrix.os }}-makefile- - name: Build tilemaker run: | - make -j4 - make test + ccache --zero-stats + make -j4 CXX="ccache c++" CC="ccache cc" + make test CXX="ccache c++" CC="ccache cc" + ccache --show-stats - name: Build openmaptiles-compatible mbtiles files of Liechtenstein run: | - run_tilemaker() { - output="$1" - shift - echo "::group::Build ${output}" - if "$@"; then - echo "::endgroup::" - else - status="$?" - echo "::endgroup::" - echo "::error title=Tile generation failed::tilemaker failed while writing ${output} with exit code ${status}" - exit "${status}" - fi - if [ ! -s "${output}" ]; then - echo "::error title=Tile output missing::tilemaker did not create ${output}" - exit 1 - fi - } - - run_tilemaker "${{ env.AREA }}.pmtiles" ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}.pmtiles" --verbose - run_tilemaker "${{ env.AREA }}-repeat.pmtiles" ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-repeat.pmtiles" --verbose - run_tilemaker "${{ env.AREA }}.mbtiles" ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}.mbtiles" --verbose --store "${{ runner.temp }}/store" - run_tilemaker "${{ env.AREA }}-repeat.mbtiles" ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-repeat.mbtiles" --verbose --store "${{ runner.temp }}/store-repeat" + .github/scripts/generate-tiles.sh ./tilemaker "${{ env.AREA }}" "" "${{ runner.temp }}" - name: Upload generated tiles uses: actions/upload-artifact@v7 with: name: tile-outputs-${{ matrix.os }}-makefile retention-days: 14 + if-no-files-found: error + compression-level: 0 path: | ${{ env.AREA }}.mbtiles ${{ env.AREA }}.pmtiles @@ -625,38 +605,20 @@ jobs: - name: Build tilemaker without clip cache run: | make clean - make -j4 CONFIG=-DTILEMAKER_DISABLE_CLIP_CACHE tilemaker + make -j4 CONFIG=-DTILEMAKER_DISABLE_CLIP_CACHE CXX="ccache c++" CC="ccache cc" tilemaker + ccache --show-stats - name: Build semantic-verification tiles without clip cache run: | - run_tilemaker() { - output="$1" - shift - echo "::group::Build ${output}" - if "$@"; then - echo "::endgroup::" - else - status="$?" - echo "::endgroup::" - echo "::error title=Tile generation failed::tilemaker failed while writing ${output} with exit code ${status}" - exit "${status}" - fi - if [ ! -s "${output}" ]; then - echo "::error title=Tile output missing::tilemaker did not create ${output}" - exit 1 - fi - } - - run_tilemaker "${{ env.AREA }}-no-clip-cache.pmtiles" ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-no-clip-cache.pmtiles" --verbose - run_tilemaker "${{ env.AREA }}-no-clip-cache-repeat.pmtiles" ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-no-clip-cache-repeat.pmtiles" --verbose - run_tilemaker "${{ env.AREA }}-no-clip-cache.mbtiles" ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-no-clip-cache.mbtiles" --verbose --store "${{ runner.temp }}/store-no-clip-cache" - run_tilemaker "${{ env.AREA }}-no-clip-cache-repeat.mbtiles" ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-no-clip-cache-repeat.mbtiles" --verbose --store "${{ runner.temp }}/store-no-clip-cache-repeat" + .github/scripts/generate-tiles.sh ./tilemaker "${{ env.AREA }}" "-no-clip-cache" "${{ runner.temp }}" - name: Upload semantic-verification tiles uses: actions/upload-artifact@v7 with: name: tile-outputs-no-clip-cache-${{ matrix.os }}-makefile retention-days: 14 + if-no-files-found: error + compression-level: 0 path: | ${{ env.AREA }}-no-clip-cache.mbtiles ${{ env.AREA }}-no-clip-cache.pmtiles @@ -724,6 +686,8 @@ jobs: with: name: tile-outputs-github-action retention-days: 14 + if-no-files-found: error + compression-level: 0 path: | ${{ env.AREA }}.mbtiles ${{ env.AREA }}.pmtiles @@ -775,6 +739,7 @@ jobs: needs: - changes - static-validation + - cmake-configure if: ${{ needs.changes.outputs.runtime == 'true' && github.ref != 'refs/heads/master' }} permissions: contents: read @@ -820,6 +785,7 @@ jobs: needs: - changes - static-validation + - cmake-configure if: ${{ needs.changes.outputs.runtime == 'true' && github.ref == 'refs/heads/master' }} permissions: contents: read From d01e8b528f011fc269e8fbfa1b54ca6f39abb94b Mon Sep 17 00:00:00 2001 From: Symmetricity <184246+Symmetricity@users.noreply.github.com> Date: Thu, 4 Jun 2026 19:00:42 +0200 Subject: [PATCH 4/6] Fetch vcpkg manifest baseline in CI Hosted runner vcpkg checkouts can be too shallow to resolve the manifest builtin-baseline, which makes vcpkg install fail before CMake starts. Fetch the baseline named in vcpkg.json before dependency installation so Dependabot baseline updates remain usable across the Windows and Linux CMake jobs. Co-authored-by: Codex --- .github/workflows/ci.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e627452c..7eac80a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -348,6 +348,12 @@ jobs: key: vcpkg-binary-${{ runner.os }}-${{ steps.runner-image.outputs.image-os }}-${{ steps.runner-image.outputs.image-version }}-x64-windows-static-md-${{ hashFiles('vcpkg.json', 'vcpkg-configuration.json') }} restore-keys: vcpkg-binary-${{ runner.os }}-${{ steps.runner-image.outputs.image-os }}-${{ steps.runner-image.outputs.image-version }}-x64-windows-static-md- + - name: Fetch vcpkg manifest baseline + shell: pwsh + run: | + $baseline = (Get-Content vcpkg.json | ConvertFrom-Json).'builtin-baseline' + git -C $env:VCPKG_INSTALLATION_ROOT fetch --depth=1 origin $baseline + - name: Build dependencies run: | vcpkg install --triplet=x64-windows-static-md @@ -466,6 +472,11 @@ jobs: key: vcpkg-binary-${{ runner.os }}-${{ steps.runner-image.outputs.image-os }}-${{ steps.runner-image.outputs.image-version }}-${{ matrix.triplet }}-${{ hashFiles('vcpkg.json', 'vcpkg-configuration.json') }} restore-keys: vcpkg-binary-${{ runner.os }}-${{ steps.runner-image.outputs.image-os }}-${{ steps.runner-image.outputs.image-version }}-${{ matrix.triplet }}- + - name: Fetch vcpkg manifest baseline + run: | + baseline="$(python3 -c 'import json; print(json.load(open("vcpkg.json"))["builtin-baseline"])')" + git -C "$VCPKG_INSTALLATION_ROOT" fetch --depth=1 origin "$baseline" + - name: Build dependencies run: | vcpkg install --triplet="${{ matrix.triplet }}" From 8090534ac4a3323d44d4f9a3f477ae32d92890f1 Mon Sep 17 00:00:00 2001 From: Symmetricity <184246+Symmetricity@users.noreply.github.com> Date: Thu, 4 Jun 2026 19:06:22 +0200 Subject: [PATCH 5/6] Checkout vcpkg manifest baseline in CI Fetching the manifest baseline is not enough for hosted runner vcpkg checkouts because vcpkg also reads the working-tree versions database. Check out the baseline named in vcpkg.json before installing dependencies so the registry files and builtin-baseline agree on both Windows and Linux CMake jobs. Co-authored-by: Codex --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7eac80a3..f50d25db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -348,11 +348,12 @@ jobs: key: vcpkg-binary-${{ runner.os }}-${{ steps.runner-image.outputs.image-os }}-${{ steps.runner-image.outputs.image-version }}-x64-windows-static-md-${{ hashFiles('vcpkg.json', 'vcpkg-configuration.json') }} restore-keys: vcpkg-binary-${{ runner.os }}-${{ steps.runner-image.outputs.image-os }}-${{ steps.runner-image.outputs.image-version }}-x64-windows-static-md- - - name: Fetch vcpkg manifest baseline + - name: Checkout vcpkg manifest baseline shell: pwsh run: | $baseline = (Get-Content vcpkg.json | ConvertFrom-Json).'builtin-baseline' git -C $env:VCPKG_INSTALLATION_ROOT fetch --depth=1 origin $baseline + git -C $env:VCPKG_INSTALLATION_ROOT checkout --force $baseline - name: Build dependencies run: | @@ -472,10 +473,11 @@ jobs: key: vcpkg-binary-${{ runner.os }}-${{ steps.runner-image.outputs.image-os }}-${{ steps.runner-image.outputs.image-version }}-${{ matrix.triplet }}-${{ hashFiles('vcpkg.json', 'vcpkg-configuration.json') }} restore-keys: vcpkg-binary-${{ runner.os }}-${{ steps.runner-image.outputs.image-os }}-${{ steps.runner-image.outputs.image-version }}-${{ matrix.triplet }}- - - name: Fetch vcpkg manifest baseline + - name: Checkout vcpkg manifest baseline run: | baseline="$(python3 -c 'import json; print(json.load(open("vcpkg.json"))["builtin-baseline"])')" git -C "$VCPKG_INSTALLATION_ROOT" fetch --depth=1 origin "$baseline" + git -C "$VCPKG_INSTALLATION_ROOT" checkout --force "$baseline" - name: Build dependencies run: | From ecb80c81a312af2c1a14902bce7dc11c2bfbcc88 Mon Sep 17 00:00:00 2001 From: Symmetricity <184246+Symmetricity@users.noreply.github.com> Date: Thu, 4 Jun 2026 19:23:59 +0200 Subject: [PATCH 6/6] Allow empty tile output suffix in CI The normal tile-generation path passes an empty suffix so that the generated files keep their historical names. PowerShell treats a mandatory string parameter as rejecting empty input unless it is explicitly allowed, which made the Windows CMake job fail before tilemaker ran. Allow only this parameter to accept an empty string. The non-empty no-clip-cache suffix path and the shell script behavior are unchanged. Co-authored-by: Codex --- .github/scripts/generate-tiles.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/scripts/generate-tiles.ps1 b/.github/scripts/generate-tiles.ps1 index 9ada6fdd..79af3158 100644 --- a/.github/scripts/generate-tiles.ps1 +++ b/.github/scripts/generate-tiles.ps1 @@ -6,6 +6,7 @@ param( [string] $Area, [Parameter(Mandatory = $true)] + [AllowEmptyString()] [string] $OutputSuffix, [Parameter(Mandatory = $true)]