diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..3cf52b82 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + + - package-ecosystem: "vcpkg" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/scripts/generate-tiles.ps1 b/.github/scripts/generate-tiles.ps1 new file mode 100644 index 00000000..79af3158 --- /dev/null +++ b/.github/scripts/generate-tiles.ps1 @@ -0,0 +1,69 @@ +param( + [Parameter(Mandatory = $true)] + [string] $Tilemaker, + + [Parameter(Mandatory = $true)] + [string] $Area, + + [Parameter(Mandatory = $true)] + [AllowEmptyString()] + [string] $OutputSuffix, + + [Parameter(Mandatory = $true)] + [string] $StoreRoot +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +function Invoke-Tilemaker { + param( + [Parameter(Mandatory = $true)] + [string] $Output, + + [Parameter(Mandatory = $true)] + [string[]] $Arguments + ) + + Write-Host "::group::Build $Output" + & $Tilemaker @Arguments + $status = $LASTEXITCODE + Write-Host "::endgroup::" + if ($status -ne 0) { + Write-Error "tilemaker failed while writing $Output with exit code $status" + exit $status + } + if (!(Test-Path -LiteralPath $Output) -or (Get-Item -LiteralPath $Output).Length -eq 0) { + Write-Error "tilemaker did not create $Output" + exit 1 + } +} + +$inputPbf = "${Area}.osm.pbf" +$common = @( + $inputPbf, + "--config=resources/config-openmaptiles.json", + "--process=resources/process-openmaptiles.lua" +) + +Invoke-Tilemaker "${Area}${OutputSuffix}.pmtiles" ` + ($common + @("--output=${Area}${OutputSuffix}.pmtiles", "--verbose")) + +Invoke-Tilemaker "${Area}${OutputSuffix}-repeat.pmtiles" ` + ($common + @("--output=${Area}${OutputSuffix}-repeat.pmtiles", "--verbose")) + +Invoke-Tilemaker "${Area}${OutputSuffix}.mbtiles" ` + ($common + @( + "--output=${Area}${OutputSuffix}.mbtiles", + "--store", + (Join-Path $StoreRoot "store${OutputSuffix}"), + "--verbose" + )) + +Invoke-Tilemaker "${Area}${OutputSuffix}-repeat.mbtiles" ` + ($common + @( + "--output=${Area}${OutputSuffix}-repeat.mbtiles", + "--store", + (Join-Path $StoreRoot "store${OutputSuffix}-repeat"), + "--verbose" + )) diff --git a/.github/scripts/generate-tiles.sh b/.github/scripts/generate-tiles.sh new file mode 100755 index 00000000..47062a1e --- /dev/null +++ b/.github/scripts/generate-tiles.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [ "$#" -ne 4 ]; then + echo "usage: $0 " >&2 + exit 2 +fi + +tilemaker="$1" +area="$2" +suffix="$3" +store_root="$4" + +run_tilemaker() { + output="$1" + shift + echo "::group::Build ${output}" + if "$@"; then + echo "::endgroup::" + else + status="$?" + echo "::endgroup::" + echo "::error title=Tile generation failed::tilemaker failed while writing ${output} with exit code ${status}" + exit "${status}" + fi + if [ ! -s "${output}" ]; then + echo "::error title=Tile output missing::tilemaker did not create ${output}" + exit 1 + fi +} + +common=( + "${area}.osm.pbf" + --config=resources/config-openmaptiles.json + --process=resources/process-openmaptiles.lua +) + +run_tilemaker "${area}${suffix}.pmtiles" \ + "${tilemaker}" "${common[@]}" \ + --output="${area}${suffix}.pmtiles" --verbose + +run_tilemaker "${area}${suffix}-repeat.pmtiles" \ + "${tilemaker}" "${common[@]}" \ + --output="${area}${suffix}-repeat.pmtiles" --verbose + +run_tilemaker "${area}${suffix}.mbtiles" \ + "${tilemaker}" "${common[@]}" \ + --output="${area}${suffix}.mbtiles" \ + --verbose --store "${store_root}/store${suffix}" + +run_tilemaker "${area}${suffix}-repeat.mbtiles" \ + "${tilemaker}" "${common[@]}" \ + --output="${area}${suffix}-repeat.mbtiles" \ + --verbose --store "${store_root}/store${suffix}-repeat" diff --git a/.github/scripts/verify-generated-tiles.py b/.github/scripts/verify-generated-tiles.py index 15dde652..4aef81f4 100644 --- a/.github/scripts/verify-generated-tiles.py +++ b/.github/scripts/verify-generated-tiles.py @@ -36,6 +36,12 @@ def error(path, message): status = 1 +def workflow_error(message): + global status + print(f"::error::{message}") + status = 1 + + def notice(path, message): label = archive_label(path) print(f"::notice title={label}::{label}: {message}") @@ -90,10 +96,15 @@ def build_label(path): return str(path.parent) label = artifact.removeprefix("tile-outputs-") + no_clip_cache = label.startswith("no-clip-cache-") + if no_clip_cache: + label = label.removeprefix("no-clip-cache-") + + mode = ", no clip cache" if no_clip_cache else "" if label.endswith("-cmake"): - return f"{runner_label(label.removesuffix('-cmake'))} (CMake)" + return f"{runner_label(label.removesuffix('-cmake'))} (CMake{mode})" if label.endswith("-makefile"): - return f"{runner_label(label.removesuffix('-makefile'))} (Makefile)" + return f"{runner_label(label.removesuffix('-makefile'))} (Makefile{mode})" return artifact @@ -115,6 +126,10 @@ def should_validate_archive(path): return path.parent.name != "tile-outputs-github-action" +def should_semantic_compare_archive(path): + return path.parent.name.startswith("tile-outputs-no-clip-cache-") + + def decompress_payload(data): for wbits in (16 + zlib.MAX_WBITS, zlib.MAX_WBITS): try: @@ -846,10 +861,24 @@ def main(): "PMTiles archive failed verification", ) - check_repeat(mbtiles, "mbtiles", mbtiles_decode_semantic_tiles, "MBTiles archive failed verification") - check_repeat(pmtiles, "pmtiles", pmtiles_decode_semantic_tiles, "PMTiles archive failed verification") - check_cross_runner(mbtiles, "mbtiles", mbtiles_decode_semantic_tiles, "MBTiles archive failed verification") - check_cross_runner(pmtiles, "pmtiles", pmtiles_decode_semantic_tiles, "PMTiles archive failed verification") + semantic_mbtiles = { + path: content for path, content in mbtiles.items() + if should_semantic_compare_archive(path) + } + semantic_pmtiles = { + path: content for path, content in pmtiles.items() + if should_semantic_compare_archive(path) + } + + if not semantic_mbtiles: + workflow_error("no no-clip-cache MBTiles artifacts found for semantic verification") + if not semantic_pmtiles: + workflow_error("no no-clip-cache PMTiles artifacts found for semantic verification") + + check_repeat(semantic_mbtiles, "mbtiles", mbtiles_decode_semantic_tiles, "MBTiles archive failed verification") + check_repeat(semantic_pmtiles, "pmtiles", pmtiles_decode_semantic_tiles, "PMTiles archive failed verification") + check_cross_runner(semantic_mbtiles, "mbtiles", mbtiles_decode_semantic_tiles, "MBTiles archive failed verification") + check_cross_runner(semantic_pmtiles, "pmtiles", pmtiles_decode_semantic_tiles, "PMTiles archive failed verification") return status diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d6fda88..f50d25db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,9 +59,15 @@ jobs: .github/workflows/ci.yml) runtime=true ;; + .github/scripts/*) + runtime=true + ;; CMakeLists.txt|Makefile|Dockerfile|action.yml) runtime=true ;; + vcpkg.json|vcpkg-configuration.json) + runtime=true + ;; get-coastline.sh|get-landcover.sh|get-monaco.sh) runtime=true ;; @@ -138,6 +144,33 @@ jobs: git ls-files -z -- '.github/scripts/*.py' | xargs -0 -r python3 -m py_compile + - name: Validate PowerShell CI scripts + run: | + while IFS= read -r -d '' file; do + SCRIPT_PATH="$file" pwsh -NoProfile -Command "\$ErrorActionPreference = 'Stop'; [scriptblock]::Create((Get-Content -Raw -LiteralPath \$env:SCRIPT_PATH)) | Out-Null" + done < <(git ls-files -z -- '.github/scripts/*.ps1') + + cmake-configure: + name: CMake configure smoke + runs-on: ubuntu-latest + timeout-minutes: 10 + needs: + - changes + - static-validation + if: ${{ needs.changes.outputs.runtime == 'true' }} + + steps: + - uses: actions/checkout@v6 + + - name: Install CMake configure dependencies + run: | + sudo apt-get update + sudo apt-get install -y build-essential cmake libboost-dev libboost-filesystem-dev libboost-program-options-dev libboost-system-dev lua5.4 liblua5.4-dev libshp-dev libsqlite3-dev rapidjson-dev zlib1g-dev + + - name: Configure CMake + run: | + cmake -S . -B build-check -DCMAKE_BUILD_TYPE=Release + download-pbf: name: Download PBF fixture runs-on: ubuntu-latest @@ -145,6 +178,7 @@ jobs: needs: - changes - static-validation + - cmake-configure if: ${{ needs.changes.outputs.runtime == 'true' }} outputs: pbf-available: ${{ steps.fixture.outputs.pbf-available }} @@ -273,6 +307,8 @@ jobs: with: name: pbf-fixture retention-days: 1 + if-no-files-found: error + compression-level: 0 path: ${{ env.AREA }}.osm.pbf Windows-Build: @@ -283,10 +319,14 @@ jobs: if: ${{ needs.download-pbf.outputs.pbf-available == 'true' }} env: VCPKG_BINARY_SOURCES: clear;files,${{ github.workspace }}\vcpkg-binary-cache,readwrite + SCCACHE_GHA_ENABLED: "true" steps: - uses: actions/checkout@v6 + - name: Install sccache + uses: mozilla-actions/sccache-action@v0.0.9 + - name: Download PBF fixture uses: actions/download-artifact@v8 with: @@ -305,64 +345,84 @@ jobs: uses: actions/cache@v5 with: path: ${{ github.workspace }}\vcpkg-binary-cache - key: vcpkg-binary-${{ runner.os }}-${{ steps.runner-image.outputs.image-os }}-${{ steps.runner-image.outputs.image-version }}-x64-windows-static-md-${{ hashFiles('.github/workflows/ci.yml') }} + key: vcpkg-binary-${{ runner.os }}-${{ steps.runner-image.outputs.image-os }}-${{ steps.runner-image.outputs.image-version }}-x64-windows-static-md-${{ hashFiles('vcpkg.json', 'vcpkg-configuration.json') }} restore-keys: vcpkg-binary-${{ runner.os }}-${{ steps.runner-image.outputs.image-os }}-${{ steps.runner-image.outputs.image-version }}-x64-windows-static-md- + - name: Checkout vcpkg manifest baseline + shell: pwsh + run: | + $baseline = (Get-Content vcpkg.json | ConvertFrom-Json).'builtin-baseline' + git -C $env:VCPKG_INSTALLATION_ROOT fetch --depth=1 origin $baseline + git -C $env:VCPKG_INSTALLATION_ROOT checkout --force $baseline + - name: Build dependencies run: | - vcpkg install --triplet=x64-windows-static-md lua shapelib zlib protobuf[zlib] sqlite3 boost-program-options boost-filesystem boost-geometry boost-system boost-asio boost-interprocess boost-iostreams boost-sort rapidjson + vcpkg install --triplet=x64-windows-static-md - name: Build tilemaker run: | mkdir ${{ github.workspace }}\build - cd ${{ github.workspace }}\build && cmake -DTILEMAKER_BUILD_STATIC=ON -DVCPKG_TARGET_TRIPLET="x64-windows-static-md" -DCMAKE_TOOLCHAIN_FILE="${env:VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake" .. + cd ${{ github.workspace }}\build && cmake -DTILEMAKER_BUILD_STATIC=ON -DVCPKG_TARGET_TRIPLET="x64-windows-static-md" -DVCPKG_INSTALLED_DIR="${{ github.workspace }}\vcpkg_installed" -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_TOOLCHAIN_FILE="${env:VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake" .. cd ${{ github.workspace }}\build && cmake --build . --config RelWithDebInfo + - name: Build tilemaker without clip cache + run: | + mkdir ${{ github.workspace }}\build-no-clip-cache + cd ${{ github.workspace }}\build-no-clip-cache && cmake -DTILEMAKER_BUILD_STATIC=ON -DTILEMAKER_DISABLE_CLIP_CACHE=ON -DVCPKG_TARGET_TRIPLET="x64-windows-static-md" -DVCPKG_INSTALLED_DIR="${{ github.workspace }}\vcpkg_installed" -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_TOOLCHAIN_FILE="${env:VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake" .. + cd ${{ github.workspace }}\build-no-clip-cache && cmake --build . --config RelWithDebInfo --target tilemaker + - name: Build openmaptiles-compatible mbtiles files of Liechtenstein run: | $tilemaker = "${{ github.workspace }}\build\RelWithDebInfo\tilemaker.exe" - function Invoke-Tilemaker { - param([string] $Output, [string[]] $Arguments) - Write-Host "::group::Build $Output" - & $tilemaker @Arguments - $status = $LASTEXITCODE - Write-Host "::endgroup::" - if ($status -ne 0) { - Write-Error "tilemaker failed while writing $Output with exit code $status" - exit $status - } - if (!(Test-Path -LiteralPath $Output) -or (Get-Item -LiteralPath $Output).Length -eq 0) { - Write-Error "tilemaker did not create $Output" - exit 1 - } - } - - Invoke-Tilemaker "${{ env.AREA }}.pmtiles" @("${{ env.AREA }}.osm.pbf", "--config=resources/config-openmaptiles.json", "--process=resources/process-openmaptiles.lua", "--output=${{ env.AREA }}.pmtiles", "--verbose") - Invoke-Tilemaker "${{ env.AREA }}-repeat.pmtiles" @("${{ env.AREA }}.osm.pbf", "--config=resources/config-openmaptiles.json", "--process=resources/process-openmaptiles.lua", "--output=${{ env.AREA }}-repeat.pmtiles", "--verbose") - Invoke-Tilemaker "${{ env.AREA }}.mbtiles" @("${{ env.AREA }}.osm.pbf", "--config=resources/config-openmaptiles.json", "--process=resources/process-openmaptiles.lua", "--output=${{ env.AREA }}.mbtiles", "--store", "${{ runner.temp }}\osm_store", "--verbose") - Invoke-Tilemaker "${{ env.AREA }}-repeat.mbtiles" @("${{ env.AREA }}.osm.pbf", "--config=resources/config-openmaptiles.json", "--process=resources/process-openmaptiles.lua", "--output=${{ env.AREA }}-repeat.mbtiles", "--store", "${{ runner.temp }}\osm_store_repeat", "--verbose") + .\.github\scripts\generate-tiles.ps1 -Tilemaker $tilemaker -Area "${{ env.AREA }}" -OutputSuffix "" -StoreRoot "${{ runner.temp }}" - name: Upload generated tiles uses: actions/upload-artifact@v7 with: name: tile-outputs-windows-cmake retention-days: 14 + if-no-files-found: error + compression-level: 0 path: | ${{ env.AREA }}.mbtiles ${{ env.AREA }}.pmtiles ${{ env.AREA }}-repeat.mbtiles ${{ env.AREA }}-repeat.pmtiles + - name: Build semantic-verification tiles without clip cache + run: | + $tilemaker = "${{ github.workspace }}\build-no-clip-cache\RelWithDebInfo\tilemaker.exe" + .\.github\scripts\generate-tiles.ps1 -Tilemaker $tilemaker -Area "${{ env.AREA }}" -OutputSuffix "-no-clip-cache" -StoreRoot "${{ runner.temp }}" + + - name: Upload semantic-verification tiles + uses: actions/upload-artifact@v7 + with: + name: tile-outputs-no-clip-cache-windows-cmake + retention-days: 14 + if-no-files-found: error + compression-level: 0 + path: | + ${{ env.AREA }}-no-clip-cache.mbtiles + ${{ env.AREA }}-no-clip-cache.pmtiles + ${{ env.AREA }}-no-clip-cache-repeat.mbtiles + ${{ env.AREA }}-no-clip-cache-repeat.pmtiles + - name: 'Upload compiled executable' uses: actions/upload-artifact@v7 with: name: tilemaker-windows retention-days: 14 + if-no-files-found: error path: | ${{ github.workspace }}\resources ${{ github.workspace }}\build\RelWithDebInfo\tilemaker.exe ${{ github.workspace }}\build\RelWithDebInfo\*.pdb + - name: Show sccache stats + if: always() + run: | + sccache --show-stats + unix-build: strategy: fail-fast: false @@ -378,6 +438,7 @@ jobs: if: ${{ needs.download-pbf.outputs.pbf-available == 'true' }} env: VCPKG_BINARY_SOURCES: clear;files,${{ github.workspace }}/vcpkg-binary-cache,readwrite + CCACHE_DIR: ${{ github.workspace }}/.ccache steps: - uses: actions/checkout@v6 @@ -393,67 +454,96 @@ jobs: echo "image-os=${ImageOS:-unknown}" >> "$GITHUB_OUTPUT" echo "image-version=${ImageVersion:-unknown}" >> "$GITHUB_OUTPUT" + - name: Install ccache + run: | + sudo apt-get update + sudo apt-get install -y ccache + + - name: Enable ccache + uses: actions/cache@v5 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ runner.os }}-${{ steps.runner-image.outputs.image-os }}-${{ steps.runner-image.outputs.image-version }}-${{ matrix.triplet }}-${{ hashFiles('CMakeLists.txt', 'include/**', 'src/**') }} + restore-keys: ccache-${{ runner.os }}-${{ steps.runner-image.outputs.image-os }}-${{ steps.runner-image.outputs.image-version }}-${{ matrix.triplet }}- + - name: Enable vcpkg binary cache uses: actions/cache@v5 with: path: ${{ github.workspace }}/vcpkg-binary-cache - key: vcpkg-binary-${{ runner.os }}-${{ steps.runner-image.outputs.image-os }}-${{ steps.runner-image.outputs.image-version }}-${{ matrix.triplet }}-${{ hashFiles('.github/workflows/ci.yml') }} + key: vcpkg-binary-${{ runner.os }}-${{ steps.runner-image.outputs.image-os }}-${{ steps.runner-image.outputs.image-version }}-${{ matrix.triplet }}-${{ hashFiles('vcpkg.json', 'vcpkg-configuration.json') }} restore-keys: vcpkg-binary-${{ runner.os }}-${{ steps.runner-image.outputs.image-os }}-${{ steps.runner-image.outputs.image-version }}-${{ matrix.triplet }}- + - name: Checkout vcpkg manifest baseline + run: | + baseline="$(python3 -c 'import json; print(json.load(open("vcpkg.json"))["builtin-baseline"])')" + git -C "$VCPKG_INSTALLATION_ROOT" fetch --depth=1 origin "$baseline" + git -C "$VCPKG_INSTALLATION_ROOT" checkout --force "$baseline" + - name: Build dependencies run: | - vcpkg install --triplet="${{ matrix.triplet }}" lua shapelib zlib protobuf[zlib] sqlite3 boost-program-options boost-filesystem boost-geometry boost-system boost-asio boost-interprocess boost-iostreams boost-sort rapidjson + vcpkg install --triplet="${{ matrix.triplet }}" - name: Build tilemaker run: | + ccache --zero-stats mkdir build cd build - cmake -DTILEMAKER_BUILD_STATIC=ON -DCMAKE_BUILD_TYPE=Release -DVCPKG_TARGET_TRIPLET="${{ matrix.triplet }}" -DCMAKE_TOOLCHAIN_FILE="$VCPKG_INSTALLATION_ROOT/scripts/buildsystems/vcpkg.cmake" -DCMAKE_CXX_COMPILER=g++ .. + cmake -DTILEMAKER_BUILD_STATIC=ON -DCMAKE_BUILD_TYPE=Release -DVCPKG_TARGET_TRIPLET="${{ matrix.triplet }}" -DVCPKG_INSTALLED_DIR="${{ github.workspace }}/vcpkg_installed" -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_TOOLCHAIN_FILE="$VCPKG_INSTALLATION_ROOT/scripts/buildsystems/vcpkg.cmake" -DCMAKE_CXX_COMPILER=g++ .. cmake --build . strip tilemaker + ccache --show-stats - - name: Build openmaptiles-compatible mbtiles files of Liechtenstein + - name: Build tilemaker without clip cache run: | - run_tilemaker() { - output="$1" - shift - echo "::group::Build ${output}" - if "$@"; then - echo "::endgroup::" - else - status="$?" - echo "::endgroup::" - echo "::error title=Tile generation failed::tilemaker failed while writing ${output} with exit code ${status}" - exit "${status}" - fi - if [ ! -s "${output}" ]; then - echo "::error title=Tile output missing::tilemaker did not create ${output}" - exit 1 - fi - } + mkdir build-no-clip-cache + cd build-no-clip-cache + cmake -DTILEMAKER_BUILD_STATIC=ON -DTILEMAKER_DISABLE_CLIP_CACHE=ON -DCMAKE_BUILD_TYPE=Release -DVCPKG_TARGET_TRIPLET="${{ matrix.triplet }}" -DVCPKG_INSTALLED_DIR="${{ github.workspace }}/vcpkg_installed" -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_TOOLCHAIN_FILE="$VCPKG_INSTALLATION_ROOT/scripts/buildsystems/vcpkg.cmake" -DCMAKE_CXX_COMPILER=g++ .. + cmake --build . --target tilemaker + strip tilemaker + ccache --show-stats + - name: Build openmaptiles-compatible mbtiles files of Liechtenstein + run: | tilemaker="${{ github.workspace }}/build/${{ matrix.executable }}" - run_tilemaker "${{ env.AREA }}.pmtiles" "${tilemaker}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}.pmtiles" --verbose - run_tilemaker "${{ env.AREA }}-repeat.pmtiles" "${tilemaker}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-repeat.pmtiles" --verbose - run_tilemaker "${{ env.AREA }}.mbtiles" "${tilemaker}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}.mbtiles" --verbose --store "${{ runner.temp }}/store" - run_tilemaker "${{ env.AREA }}-repeat.mbtiles" "${tilemaker}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-repeat.mbtiles" --verbose --store "${{ runner.temp }}/store-repeat" + .github/scripts/generate-tiles.sh "${tilemaker}" "${{ env.AREA }}" "" "${{ runner.temp }}" - name: Upload generated tiles uses: actions/upload-artifact@v7 with: name: tile-outputs-${{ matrix.os }}-cmake retention-days: 14 + if-no-files-found: error + compression-level: 0 path: | ${{ env.AREA }}.mbtiles ${{ env.AREA }}.pmtiles ${{ env.AREA }}-repeat.mbtiles ${{ env.AREA }}-repeat.pmtiles + - name: Build semantic-verification tiles without clip cache + run: | + tilemaker="${{ github.workspace }}/build-no-clip-cache/${{ matrix.executable }}" + .github/scripts/generate-tiles.sh "${tilemaker}" "${{ env.AREA }}" "-no-clip-cache" "${{ runner.temp }}" + + - name: Upload semantic-verification tiles + uses: actions/upload-artifact@v7 + with: + name: tile-outputs-no-clip-cache-${{ matrix.os }}-cmake + retention-days: 14 + if-no-files-found: error + compression-level: 0 + path: | + ${{ env.AREA }}-no-clip-cache.mbtiles + ${{ env.AREA }}-no-clip-cache.pmtiles + ${{ env.AREA }}-no-clip-cache-repeat.mbtiles + ${{ env.AREA }}-no-clip-cache-repeat.pmtiles + - name: 'Upload compiled executable' uses: actions/upload-artifact@v7 with: name: tilemaker-${{ matrix.os }} retention-days: 14 + if-no-files-found: error path: | ${{ github.workspace }}/resources ${{ github.workspace }}/build/${{ matrix.executable }} @@ -472,6 +562,8 @@ jobs: timeout-minutes: 30 needs: download-pbf if: ${{ needs.download-pbf.outputs.pbf-available == 'true' }} + env: + CCACHE_DIR: ${{ github.workspace }}/.ccache steps: - uses: actions/checkout@v6 @@ -484,55 +576,68 @@ jobs: - name: Install Linux dependencies if: ${{ matrix.os == 'ubuntu-22.04' }} run: | - sudo apt install build-essential libboost-dev libboost-filesystem-dev libboost-iostreams-dev libboost-program-options-dev libboost-system-dev luajit libluajit-5.1-dev liblua5.1-0-dev libshp-dev libsqlite3-dev rapidjson-dev zlib1g-dev + sudo apt install build-essential ccache libboost-dev libboost-filesystem-dev libboost-iostreams-dev libboost-program-options-dev libboost-system-dev luajit libluajit-5.1-dev liblua5.1-0-dev libshp-dev libsqlite3-dev rapidjson-dev zlib1g-dev - name: Install Mac OS X dependencies if: ${{ matrix.os == 'macos-latest' || matrix.os == 'macos-14' }} run: | c++ --version - brew install boost lua shapelib rapidjson + brew install boost ccache lua shapelib rapidjson + + - name: Enable ccache + uses: actions/cache@v5 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ runner.os }}-${{ matrix.os }}-makefile-${{ hashFiles('Makefile', 'include/**', 'src/**', 'test/**') }} + restore-keys: ccache-${{ runner.os }}-${{ matrix.os }}-makefile- - name: Build tilemaker run: | - make -j4 - make test + ccache --zero-stats + make -j4 CXX="ccache c++" CC="ccache cc" + make test CXX="ccache c++" CC="ccache cc" + ccache --show-stats - name: Build openmaptiles-compatible mbtiles files of Liechtenstein run: | - run_tilemaker() { - output="$1" - shift - echo "::group::Build ${output}" - if "$@"; then - echo "::endgroup::" - else - status="$?" - echo "::endgroup::" - echo "::error title=Tile generation failed::tilemaker failed while writing ${output} with exit code ${status}" - exit "${status}" - fi - if [ ! -s "${output}" ]; then - echo "::error title=Tile output missing::tilemaker did not create ${output}" - exit 1 - fi - } - - run_tilemaker "${{ env.AREA }}.pmtiles" ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}.pmtiles" --verbose - run_tilemaker "${{ env.AREA }}-repeat.pmtiles" ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-repeat.pmtiles" --verbose - run_tilemaker "${{ env.AREA }}.mbtiles" ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}.mbtiles" --verbose --store "${{ runner.temp }}/store" - run_tilemaker "${{ env.AREA }}-repeat.mbtiles" ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-repeat.mbtiles" --verbose --store "${{ runner.temp }}/store-repeat" + .github/scripts/generate-tiles.sh ./tilemaker "${{ env.AREA }}" "" "${{ runner.temp }}" - name: Upload generated tiles uses: actions/upload-artifact@v7 with: name: tile-outputs-${{ matrix.os }}-makefile retention-days: 14 + if-no-files-found: error + compression-level: 0 path: | ${{ env.AREA }}.mbtiles ${{ env.AREA }}.pmtiles ${{ env.AREA }}-repeat.mbtiles ${{ env.AREA }}-repeat.pmtiles + - name: Build tilemaker without clip cache + run: | + make clean + make -j4 CONFIG=-DTILEMAKER_DISABLE_CLIP_CACHE CXX="ccache c++" CC="ccache cc" tilemaker + ccache --show-stats + + - name: Build semantic-verification tiles without clip cache + run: | + .github/scripts/generate-tiles.sh ./tilemaker "${{ env.AREA }}" "-no-clip-cache" "${{ runner.temp }}" + + - name: Upload semantic-verification tiles + uses: actions/upload-artifact@v7 + with: + name: tile-outputs-no-clip-cache-${{ matrix.os }}-makefile + retention-days: 14 + if-no-files-found: error + compression-level: 0 + path: | + ${{ env.AREA }}-no-clip-cache.mbtiles + ${{ env.AREA }}-no-clip-cache.pmtiles + ${{ env.AREA }}-no-clip-cache-repeat.mbtiles + ${{ env.AREA }}-no-clip-cache-repeat.pmtiles + Github-Action: name: Generate mbtiles with Github Action @@ -594,6 +699,8 @@ jobs: with: name: tile-outputs-github-action retention-days: 14 + if-no-files-found: error + compression-level: 0 path: | ${{ env.AREA }}.mbtiles ${{ env.AREA }}.pmtiles @@ -645,6 +752,7 @@ jobs: needs: - changes - static-validation + - cmake-configure if: ${{ needs.changes.outputs.runtime == 'true' && github.ref != 'refs/heads/master' }} permissions: contents: read @@ -690,6 +798,7 @@ jobs: needs: - changes - static-validation + - cmake-configure if: ${{ needs.changes.outputs.runtime == 'true' && github.ref == 'refs/heads/master' }} permissions: contents: read diff --git a/CMakeLists.txt b/CMakeLists.txt index c67b73ad..ffadcb96 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,6 +7,7 @@ endif() project(tilemaker) OPTION(TILEMAKER_BUILD_STATIC "Attempt to link dependencies static" OFF) +OPTION(TILEMAKER_DISABLE_CLIP_CACHE "Disable geometry clip cache" OFF) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") @@ -144,6 +145,9 @@ file(GLOB tilemaker_src_files src/way_stores.cpp ) add_executable(tilemaker ${tilemaker_src_files}) +IF (TILEMAKER_DISABLE_CLIP_CACHE) + target_compile_definitions(tilemaker PRIVATE TILEMAKER_DISABLE_CLIP_CACHE) +ENDIF () target_include_directories(tilemaker PRIVATE include) target_include_directories(tilemaker PRIVATE ${CMAKE_BINARY_DIR}) # for generated files target_link_libraries(tilemaker diff --git a/include/clip_cache.h b/include/clip_cache.h index 353356dd..bad7c246 100644 --- a/include/clip_cache.h +++ b/include/clip_cache.h @@ -11,6 +11,17 @@ class TileBbox; template class ClipCache { public: +#ifdef TILEMAKER_DISABLE_CLIP_CACHE + ClipCache(size_t, unsigned int) { + } + + const std::shared_ptr get(uint, TileCoordinate, TileCoordinate, NodeID) const{ + return nullptr; + } + + void add(const TileBbox&, const NodeID, const T&) { + } +#else ClipCache(size_t threadNum, unsigned int baseZoom): baseZoom(baseZoom), clipCache(threadNum * 16), @@ -74,6 +85,7 @@ class ClipCache { std::vector, std::shared_ptr>> clipCache; mutable std::vector clipCacheMutex; std::vector clipCacheSize; +#endif }; #endif diff --git a/vcpkg.json b/vcpkg.json new file mode 100644 index 00000000..b67dd5a8 --- /dev/null +++ b/vcpkg.json @@ -0,0 +1,27 @@ +{ + "$schema": "https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg.schema.json", + "name": "tilemaker", + "version-string": "3.1.0", + "builtin-baseline": "cea592f4772491abdb7c483387a59ea89889f4be", + "dependencies": [ + "lua", + "shapelib", + "zlib", + { + "name": "protobuf", + "features": [ + "zlib" + ] + }, + "sqlite3", + "boost-program-options", + "boost-filesystem", + "boost-geometry", + "boost-system", + "boost-asio", + "boost-interprocess", + "boost-iostreams", + "boost-sort", + "rapidjson" + ] +}