From 36177972d88ce1bd8af1033b7509ef252ee9a226 Mon Sep 17 00:00:00 2001 From: Symmetricity <184246+Symmetricity@users.noreply.github.com> Date: Wed, 13 May 2026 06:09:37 +0200 Subject: [PATCH 01/10] Improve CI workflow setup Update the workflow defaults before adding deeper output checks. This adds explicit workflow permissions, run cancellation, shared fixture download and verification, newer GitHub Actions versions, runner-scoped vcpkg binary caches, and Docker build caching. This reduces duplicated work across jobs, avoids using an unchecked PBF download independently on each runner, makes cache reuse less brittle across runner image changes, and keeps PR Docker builds separate from master publishes. --- .github/workflows/ci.yml | 218 ++++++++++++++++++++++++++++++++------- 1 file changed, 180 insertions(+), 38 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bbd50fbe..7a710247 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,24 +5,90 @@ on: branches: [ master ] pull_request: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + env: AREA: liechtenstein jobs: + download-pbf: + name: Download PBF fixture + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + pbf-available: ${{ steps.download.outputs.pbf-available }} + + steps: + - name: Download and verify PBF file + id: download + run: | + if curl -fsSL "https://download.geofabrik.de/europe/${AREA}-latest.osm.pbf" -o "${AREA}-latest.osm.pbf" && + curl -fsSL "https://download.geofabrik.de/europe/${AREA}-latest.osm.pbf.md5" -o "${AREA}-latest.osm.pbf.md5" && + md5sum -c "${AREA}-latest.osm.pbf.md5" && + mv "${AREA}-latest.osm.pbf" "${AREA}.osm.pbf"; then + echo "pbf-available=true" >> "$GITHUB_OUTPUT" + else + echo "::warning::Geofabrik PBF fixture could not be downloaded or verified; fixture-dependent jobs will be skipped." + echo "pbf-available=false" >> "$GITHUB_OUTPUT" + fi + + - name: Report unavailable PBF fixture + if: ${{ steps.download.outputs.pbf-available != 'true' }} + run: | + echo "::warning title=Tile generation not verified::Geofabrik PBF fixture could not be downloaded or verified. Fixture-dependent jobs were skipped." + { + echo "### Tile generation not verified" + echo "" + echo "The Geofabrik PBF fixture could not be downloaded or verified, so jobs that generate MBTiles/PMTiles were skipped." + echo "This does not indicate a code failure, but this run did not verify tile generation." + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload PBF fixture + if: ${{ steps.download.outputs.pbf-available == 'true' }} + uses: actions/upload-artifact@v7 + with: + name: pbf-fixture + retention-days: 1 + path: ${{ env.AREA }}.osm.pbf + Windows-Build: name: Windows (CMake) runs-on: windows-latest + timeout-minutes: 120 + needs: download-pbf + if: ${{ needs.download-pbf.outputs.pbf-available == 'true' }} + env: + VCPKG_BINARY_SOURCES: clear;files,${{ github.workspace }}\vcpkg-binary-cache,readwrite steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - - name: Enable vcpkg cache - uses: actions/cache@v4 + - name: Download PBF fixture + uses: actions/download-artifact@v8 with: - path: c:\vcpkg\installed - key: windows-vcpkg-x64-0 # Increase the number whenever dependencies are modified - restore-keys: windows-vcpkg-x64 + name: pbf-fixture + + - name: Detect runner image + id: runner-image + shell: pwsh + run: | + $imageOS = if ($env:ImageOS) { $env:ImageOS } else { "unknown" } + $imageVersion = if ($env:ImageVersion) { $env:ImageVersion } else { "unknown" } + "image-os=$imageOS" >> $env:GITHUB_OUTPUT + "image-version=$imageVersion" >> $env:GITHUB_OUTPUT + + - name: Enable vcpkg binary cache + uses: actions/cache@v5 + with: + path: ${{ github.workspace }}\vcpkg-binary-cache + key: vcpkg-binary-${{ runner.os }}-${{ steps.runner-image.outputs.image-os }}-${{ steps.runner-image.outputs.image-version }}-x64-windows-static-md-${{ hashFiles('.github/workflows/ci.yml') }} + restore-keys: vcpkg-binary-${{ runner.os }}-${{ steps.runner-image.outputs.image-os }}-${{ steps.runner-image.outputs.image-version }}-x64-windows-static-md- - name: Build dependencies run: | @@ -30,20 +96,29 @@ jobs: - name: Build tilemaker run: | - mkdir ${{ github.workspace }}\build - cd ${{ github.workspace }}\build && cmake -DTILEMAKER_BUILD_STATIC=ON -DVCPKG_TARGET_TRIPLET="x64-windows-static-md" -DCMAKE_TOOLCHAIN_FILE="c:\vcpkg\scripts\buildsystems\vcpkg.cmake" .. + mkdir ${{ github.workspace }}\build + cd ${{ github.workspace }}\build && cmake -DTILEMAKER_BUILD_STATIC=ON -DVCPKG_TARGET_TRIPLET="x64-windows-static-md" -DCMAKE_TOOLCHAIN_FILE="${env:VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake" .. cd ${{ github.workspace }}\build && cmake --build . --config RelWithDebInfo - name: Build openmaptiles-compatible mbtiles files of Liechtenstein run: | - Invoke-WebRequest -Uri https://download.geofabrik.de/europe/${{ env.AREA }}-latest.osm.pbf -OutFile ${{ env.AREA }}.osm.pbf -MaximumRedirection 5 ${{ github.workspace }}\build\RelWithDebInfo\tilemaker.exe ${{ env.AREA }}.osm.pbf --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output=${{ env.AREA }}.pmtiles --verbose - ${{ github.workspace }}\build\RelWithDebInfo\tilemaker.exe ${{ env.AREA }}.osm.pbf --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output=${{ env.AREA }}.mbtiles --store osm_store --verbose + ${{ github.workspace }}\build\RelWithDebInfo\tilemaker.exe ${{ env.AREA }}.osm.pbf --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output=${{ env.AREA }}.mbtiles --store "${{ runner.temp }}\osm_store" --verbose + + - name: Upload generated tiles + uses: actions/upload-artifact@v7 + with: + name: tile-outputs-windows-cmake + retention-days: 14 + path: | + ${{ env.AREA }}.mbtiles + ${{ env.AREA }}.pmtiles - name: 'Upload compiled executable' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: tilemaker-windows + retention-days: 14 path: | ${{ github.workspace }}\resources ${{ github.workspace }}\build\RelWithDebInfo\tilemaker.exe @@ -56,43 +131,66 @@ jobs: - os: ubuntu-22.04 triplet: x64-linux executable: tilemaker - path: /usr/local/share/vcpkg/installed - toolchain: /usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake name: ${{ matrix.os }} (CMake) runs-on: ${{ matrix.os }} + timeout-minutes: 90 + needs: download-pbf + if: ${{ needs.download-pbf.outputs.pbf-available == 'true' }} + env: + VCPKG_BINARY_SOURCES: clear;files,${{ github.workspace }}/vcpkg-binary-cache,readwrite steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 + + - name: Download PBF fixture + uses: actions/download-artifact@v8 + with: + name: pbf-fixture + + - name: Detect runner image + id: runner-image + run: | + echo "image-os=${ImageOS:-unknown}" >> "$GITHUB_OUTPUT" + echo "image-version=${ImageVersion:-unknown}" >> "$GITHUB_OUTPUT" - - name: Enable vcpkg cache - uses: actions/cache@v4 + - name: Enable vcpkg binary cache + uses: actions/cache@v5 with: - path: ${{ matrix.path }} - key: vcpkg-${{ matrix.triplet }}-0 # Increase the number whenever dependencies are modified - restore-keys: vcpkg-${{ matrix.triplet }} + path: ${{ github.workspace }}/vcpkg-binary-cache + key: vcpkg-binary-${{ runner.os }}-${{ steps.runner-image.outputs.image-os }}-${{ steps.runner-image.outputs.image-version }}-${{ matrix.triplet }}-${{ hashFiles('.github/workflows/ci.yml') }} + restore-keys: vcpkg-binary-${{ runner.os }}-${{ steps.runner-image.outputs.image-os }}-${{ steps.runner-image.outputs.image-version }}-${{ matrix.triplet }}- - name: Build dependencies run: | - vcpkg install --triplet=${{ matrix.triplet }} lua shapelib zlib protobuf[zlib] sqlite3 boost-program-options boost-filesystem boost-geometry boost-system boost-asio boost-interprocess boost-iostreams boost-sort rapidjson + vcpkg install --triplet="${{ matrix.triplet }}" lua shapelib zlib protobuf[zlib] sqlite3 boost-program-options boost-filesystem boost-geometry boost-system boost-asio boost-interprocess boost-iostreams boost-sort rapidjson - name: Build tilemaker run: | mkdir build cd build - cmake -DTILEMAKER_BUILD_STATIC=ON -DCMAKE_BUILD_TYPE=Release -DVCPKG_TARGET_TRIPLET=${{ matrix.triplet }} -DCMAKE_TOOLCHAIN_FILE=${{ matrix.toolchain }} -DCMAKE_CXX_COMPILER=g++ .. + cmake -DTILEMAKER_BUILD_STATIC=ON -DCMAKE_BUILD_TYPE=Release -DVCPKG_TARGET_TRIPLET="${{ matrix.triplet }}" -DCMAKE_TOOLCHAIN_FILE="$VCPKG_INSTALLATION_ROOT/scripts/buildsystems/vcpkg.cmake" -DCMAKE_CXX_COMPILER=g++ .. cmake --build . strip tilemaker - name: Build openmaptiles-compatible mbtiles files of Liechtenstein run: | - curl -L https://download.geofabrik.de/europe/${{ env.AREA }}-latest.osm.pbf -o ${{ env.AREA }}.osm.pbf - ${{ github.workspace }}/build/${{ matrix.executable }} ${{ env.AREA }}.osm.pbf --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output=${{ env.AREA }}.pmtiles --verbose - ${{ github.workspace }}/build/${{ matrix.executable }} ${{ env.AREA }}.osm.pbf --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output=${{ env.AREA }}.mbtiles --verbose --store /tmp/store + "${{ github.workspace }}/build/${{ matrix.executable }}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}.pmtiles" --verbose + "${{ github.workspace }}/build/${{ matrix.executable }}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}.mbtiles" --verbose --store "${{ runner.temp }}/store" + + - name: Upload generated tiles + uses: actions/upload-artifact@v7 + with: + name: tile-outputs-${{ matrix.os }}-cmake + retention-days: 14 + path: | + ${{ env.AREA }}.mbtiles + ${{ env.AREA }}.pmtiles - name: 'Upload compiled executable' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: tilemaker-${{ matrix.os }} + retention-days: 14 path: | ${{ github.workspace }}/resources ${{ github.workspace }}/build/${{ matrix.executable }} @@ -107,9 +205,17 @@ jobs: name: ${{ matrix.os }} (Makefile) runs-on: ${{ matrix.os }} + timeout-minutes: 30 + needs: download-pbf + if: ${{ needs.download-pbf.outputs.pbf-available == 'true' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 + + - name: Download PBF fixture + uses: actions/download-artifact@v8 + with: + name: pbf-fixture - name: Install Linux dependencies if: ${{ matrix.os == 'ubuntu-22.04' }} @@ -129,44 +235,76 @@ jobs: - name: Build openmaptiles-compatible mbtiles files of Liechtenstein run: | - curl -L https://download.geofabrik.de/europe/${{ env.AREA }}-latest.osm.pbf -o ${{ env.AREA }}.osm.pbf - ./tilemaker ${{ env.AREA }}.osm.pbf --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output=${{ env.AREA }}.pmtiles --verbose - ./tilemaker ${{ env.AREA }}.osm.pbf --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output=${{ env.AREA }}.mbtiles --verbose --store /tmp/store + ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}.pmtiles" --verbose + ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}.mbtiles" --verbose --store "${{ runner.temp }}/store" + + - name: Upload generated tiles + uses: actions/upload-artifact@v7 + with: + name: tile-outputs-${{ matrix.os }}-makefile + retention-days: 14 + path: | + ${{ env.AREA }}.mbtiles + ${{ env.AREA }}.pmtiles Github-Action: name: Generate mbtiles with Github Action runs-on: ubuntu-latest + timeout-minutes: 20 + needs: download-pbf + if: ${{ needs.download-pbf.outputs.pbf-available == 'true' }} steps: - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Download PBF file - run: curl -L https://download.geofabrik.de/europe/${AREA}-latest.osm.pbf -o ${AREA}.osm.pbf + uses: actions/download-artifact@v8 + with: + name: pbf-fixture + + - name: Build openmaptiles-compatible pmtiles file + uses: ./ + with: + input: ${{ env.AREA }}.osm.pbf + output: ${{ env.AREA }}.pmtiles - - name: Build openmaptiles-compatible mbtiles files of given area + - name: Build openmaptiles-compatible mbtiles file uses: ./ with: - input: ${{ env.AREA }}.osm.pbf + input: ${{ env.AREA }}.osm.pbf output: ${{ env.AREA }}.mbtiles + - name: Upload generated tiles + uses: actions/upload-artifact@v7 + with: + name: tile-outputs-github-action + retention-days: 14 + path: | + ${{ env.AREA }}.mbtiles + ${{ env.AREA }}.pmtiles + docker-build: env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} runs-on: ubuntu-latest + timeout-minutes: 45 permissions: contents: read packages: write steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 - name: Log in to the Container registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -174,23 +312,27 @@ jobs: - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@v6 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - name: Build Docker image - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 if: ${{ github.ref != 'refs/heads/master'}} with: context: . push: false tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max - name: Build and push Docker image - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 if: ${{ github.ref == 'refs/heads/master'}} with: context: . push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max From 292cf7ea0f25a1eafd844204e3c0115901a66cf4 Mon Sep 17 00:00:00 2001 From: Symmetricity <184246+Symmetricity@users.noreply.github.com> Date: Wed, 13 May 2026 06:10:26 +0200 Subject: [PATCH 02/10] Verify generated tile outputs in CI Generate MBTiles and PMTiles artifacts from each CI build path, run the generation twice, and compare the results after the build jobs complete. The verification step checks PMTiles archive structure, records raw decompressed tile-byte hashes, and also canonicalizes MVT layer, feature, and tag ordering so ordering-only changes are reported separately from semantic tile-content differences. This turns CI into a correctness check for generated output, not just a compiler check. Current runs show that the new check is finding real issues: Windows PMTiles archives fail structural verification, and repeat runs can produce semantic differences such as different feature counts in the same layer/tile. Because this adds a Python CI helper, ignore Python bytecode and cache directories produced by local validation. --- .github/scripts/verify-generated-tiles.py | 599 ++++++++++++++++++++++ .github/workflows/ci.yml | 170 +++++- .gitignore | 4 + 3 files changed, 770 insertions(+), 3 deletions(-) create mode 100644 .github/scripts/verify-generated-tiles.py diff --git a/.github/scripts/verify-generated-tiles.py b/.github/scripts/verify-generated-tiles.py new file mode 100644 index 00000000..5367ecab --- /dev/null +++ b/.github/scripts/verify-generated-tiles.py @@ -0,0 +1,599 @@ +#!/usr/bin/env python3 + +import gzip +import hashlib +import json +import pathlib +import sqlite3 +import struct +import subprocess +import sys +import zlib + + +COMPRESSION_NONE = 1 +COMPRESSION_GZIP = 2 + + +status = 0 + + +class ArchiveContent: + def __init__(self, raw_hash, semantic_hash, raw_tiles, semantic_tiles, layer_counts): + self.raw_hash = raw_hash + self.semantic_hash = semantic_hash + self.raw_tiles = raw_tiles + self.semantic_tiles = semantic_tiles + self.layer_counts = layer_counts + + +def error(path, message): + global status + label = archive_label(path) + print(f"::error title={label}::{label}: {message}") + status = 1 + + +def notice(path, message): + label = archive_label(path) + print(f"::notice title={label}::{label}: {message}") + + +def archive_sha256(path): + digest = hashlib.sha256() + with path.open("rb") as handle: + while True: + chunk = handle.read(1024 * 1024) + if not chunk: + break + digest.update(chunk) + return digest.hexdigest() + + +def build_label(path): + artifact = path.parent.name + if artifact == "tile-outputs-github-action": + return "GitHub Action" + if not artifact.startswith("tile-outputs-"): + return str(path.parent) + + label = artifact.removeprefix("tile-outputs-") + if label.endswith("-cmake"): + return f"{runner_label(label.removesuffix('-cmake'))} (CMake)" + if label.endswith("-makefile"): + return f"{runner_label(label.removesuffix('-makefile'))} (Makefile)" + return artifact + + +def runner_label(label): + if label == "windows": + return "Windows" + if label.startswith("ubuntu-"): + return "Ubuntu " + label.removeprefix("ubuntu-") + if label.startswith("macos-"): + return "macOS " + label.removeprefix("macos-") + return label.replace("-", " ") + + +def archive_label(path): + return f"{build_label(path)} {path.name}" + + +def decompress_payload(data): + for wbits in (16 + zlib.MAX_WBITS, zlib.MAX_WBITS): + try: + return zlib.decompress(data, wbits) + except zlib.error: + pass + return data + + +def decompress_pmtiles(data, compression): + if compression == COMPRESSION_NONE: + return data + if compression == COMPRESSION_GZIP: + return gzip.decompress(data) + raise RuntimeError(f"unsupported PMTiles compression {compression}") + + +def tileid_to_zxy(tileid): + acc = 0 + for z in range(32): + num_tiles = (1 << z) * (1 << z) + if acc + num_tiles > tileid: + return t_on_level(z, tileid - acc) + acc += num_tiles + raise RuntimeError("tile zoom exceeds 64-bit limit") + + +def t_on_level(z, pos): + n = 1 << z + t = pos + x = 0 + y = 0 + s = 1 + while s < n: + rx = 1 & (t // 2) + ry = 1 & (t ^ rx) + if ry == 0: + if rx == 1: + x = s - 1 - x + y = s - 1 - y + x, y = y, x + x += s * rx + y += s * ry + t //= 4 + s *= 2 + return z, x, y + + +def decode_varint(data, pos): + value = 0 + shift = 0 + for _ in range(10): + if pos >= len(data): + raise RuntimeError("end of buffer while reading varint") + byte = data[pos] + pos += 1 + value |= (byte & 0x7f) << shift + if byte < 0x80: + return value, pos + shift += 7 + raise RuntimeError("varint too long") + + +def deserialize_directory(data): + pos = 0 + count, pos = decode_varint(data, pos) + entries = [{"tile_id": 0, "run_length": 0, "length": 0, "offset": 0} for _ in range(count)] + last_id = 0 + for entry in entries: + value, pos = decode_varint(data, pos) + entry["tile_id"] = last_id + value + last_id = entry["tile_id"] + for entry in entries: + entry["run_length"], pos = decode_varint(data, pos) + for entry in entries: + entry["length"], pos = decode_varint(data, pos) + for index, entry in enumerate(entries): + value, pos = decode_varint(data, pos) + if index > 0 and value == 0: + prev = entries[index - 1] + entry["offset"] = prev["offset"] + prev["length"] + else: + entry["offset"] = value - 1 + if pos != len(data): + raise RuntimeError("trailing bytes in PMTiles directory") + return entries + + +def zigzag_decode(value): + return (value >> 1) ^ -(value & 1) + + +def protobuf_fields(data): + pos = 0 + while pos < len(data): + tag, pos = decode_varint(data, pos) + field = tag >> 3 + wire_type = tag & 7 + if wire_type == 0: + value, pos = decode_varint(data, pos) + yield field, wire_type, value + elif wire_type == 1: + if pos + 8 > len(data): + raise RuntimeError("end of buffer while reading fixed64") + yield field, wire_type, data[pos:pos + 8] + pos += 8 + elif wire_type == 2: + length, pos = decode_varint(data, pos) + if pos + length > len(data): + raise RuntimeError("end of buffer while reading length-delimited field") + yield field, wire_type, data[pos:pos + length] + pos += length + elif wire_type == 5: + if pos + 4 > len(data): + raise RuntimeError("end of buffer while reading fixed32") + yield field, wire_type, data[pos:pos + 4] + pos += 4 + else: + raise RuntimeError(f"unsupported protobuf wire type {wire_type}") + + +def packed_varints(data): + pos = 0 + values = [] + while pos < len(data): + value, pos = decode_varint(data, pos) + values.append(value) + return values + + +def decode_mvt_value(data): + values = [] + for field, wire_type, value in protobuf_fields(data): + if field == 1: + values.append(["string", value.decode("utf-8", "replace")]) + elif field == 2: + values.append(["float", struct.unpack("= len(keys) or value_index >= len(values): + raise RuntimeError("MVT feature tag index is out of range") + tags.append([keys[key_index], values[value_index]]) + elif field == 3: + geometry_type = value + elif field == 4: + geometry = packed_varints(value) + tags.sort(key=canonical_json) + return [feature_id, geometry_type, tags, geometry] + + +def decode_mvt_layer(data): + name = None + features = [] + keys = [] + values = [] + extent = None + version = None + for field, wire_type, value in protobuf_fields(data): + if field == 1: + name = value.decode("utf-8", "replace") + elif field == 2: + features.append(value) + elif field == 3: + keys.append(value.decode("utf-8", "replace")) + elif field == 4: + values.append(decode_mvt_value(value)) + elif field == 5: + extent = value + elif field == 15: + version = value + + decoded_features = [decode_mvt_feature(feature, keys, values) for feature in features] + decoded_features.sort(key=canonical_json) + return [name, version, extent, decoded_features] + + +def canonical_mvt(data): + layers = [] + for field, wire_type, value in protobuf_fields(data): + if field == 3: + layers.append(decode_mvt_layer(value)) + layers.sort(key=canonical_json) + return layers + + +def canonical_json(value): + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def semantic_tile_content(data): + tile = canonical_mvt(data) + digest = hashlib.sha256(canonical_json(tile).encode("utf-8", "surrogateescape")).hexdigest() + layer_counts = {layer[0]: len(layer[3]) for layer in tile} + return digest, layer_counts + + +def add_tile_to_digests(raw_digest, semantic_digest, raw_tiles, semantic_tiles, layer_counts, z, x, y, payload): + tile = (z, x, y) + raw_hash = hashlib.sha256(payload).hexdigest() + semantic_hash, counts = semantic_tile_content(payload) + raw_tiles[tile] = raw_hash + semantic_tiles[tile] = semantic_hash + layer_counts[tile] = counts + + raw_digest.update(f"T\t{z}\t{x}\t{y}\t{len(payload)}\t{raw_hash}\n".encode()) + semantic_digest.update(f"T\t{z}\t{x}\t{y}\t{semantic_hash}\n".encode()) + + +def mbtiles_fingerprint(path): + con = sqlite3.connect(f"file:{path}?mode=ro", uri=True) + cur = con.cursor() + tables = {row[0] for row in cur.execute("select name from sqlite_master where type = 'table'")} + missing = {"metadata", "tiles"} - tables + if missing: + raise RuntimeError("missing tables: " + ", ".join(sorted(missing))) + tile_count = cur.execute("select count(*) from tiles").fetchone()[0] + if tile_count == 0: + raise RuntimeError("tiles table is empty") + empty_tiles = cur.execute("select count(*) from tiles where tile_data is null or length(tile_data) = 0").fetchone()[0] + if empty_tiles: + raise RuntimeError(f"{empty_tiles} empty tile blobs") + minzoom, maxzoom = cur.execute("select min(zoom_level), max(zoom_level) from tiles").fetchone() + metadata_rows = list(cur.execute("select name, value from metadata order by name, value")) + if not metadata_rows: + raise RuntimeError("metadata table is empty") + + raw_digest = hashlib.sha256() + semantic_digest = hashlib.sha256() + for name, value in metadata_rows: + metadata = f"M\t{name}\t{canonical_metadata_value(value)}\n".encode("utf-8", "surrogateescape") + raw_digest.update(metadata) + semantic_digest.update(metadata) + + raw_tiles = {} + semantic_tiles = {} + layer_counts = {} + for z, x, y, tile_data in cur.execute("select zoom_level, tile_column, tile_row, tile_data from tiles order by zoom_level, tile_column, tile_row"): + payload = decompress_payload(bytes(tile_data)) + add_tile_to_digests(raw_digest, semantic_digest, raw_tiles, semantic_tiles, layer_counts, z, x, y, payload) + con.close() + + content = ArchiveContent(raw_digest.hexdigest(), semantic_digest.hexdigest(), raw_tiles, semantic_tiles, layer_counts) + print( + f"{archive_label(path)}: {tile_count} tiles, zoom {minzoom}-{maxzoom}, " + f"{len(metadata_rows)} metadata rows, raw {content.raw_hash}, semantic {content.semantic_hash}" + ) + return content + + +def canonical_metadata_value(value): + if value is None: + return "" + try: + return json.dumps(json.loads(value), sort_keys=True, separators=(",", ":")) + except (TypeError, json.JSONDecodeError): + return str(value) + + +def verify_pmtiles(path): + print(f"{archive_label(path)}: verifying PMTiles archive") + result = subprocess.run( + ["pmtiles", "verify", str(path)], + capture_output=True, + text=True, + check=False, + ) + if result.stdout: + print(result.stdout, end="") + if result.stderr: + print(result.stderr, end="") + if result.returncode != 0: + raise RuntimeError(pmtiles_failure_message(result)) + + +def pmtiles_failure_message(result): + lines = [ + line.strip() + for line in (result.stdout + "\n" + result.stderr).splitlines() + if line.strip() + ] + for line in reversed(lines): + pos = line.find("Failed to verify archive") + if pos >= 0: + return line[pos:] + if lines: + return lines[-1] + return "pmtiles verify failed" + + +def pmtiles_header(data): + if len(data) < 127: + raise RuntimeError("PMTiles header is truncated") + if data[:7] != b"PMTiles" or data[7] != 3: + raise RuntimeError("not a PMTiles v3 archive") + values = struct.unpack_from("> "$GITHUB_OUTPUT" + { + echo "### Changed file check" + echo "" + echo "Unable to determine the changed files, so the full CI matrix will run." + } >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + runtime=false + changed_count=0 + while IFS= read -r -d '' path; do + changed_count=$((changed_count + 1)) + case "$path" in + .github/workflows/ci.yml) + runtime=true + ;; + CMakeLists.txt|Makefile|Dockerfile|action.yml) + runtime=true + ;; + get-coastline.sh|get-landcover.sh|get-monaco.sh) + runtime=true + ;; + cmake/*|include/*|resources/*|server/*|src/*|test/*) + runtime=true + ;; + esac + done < <(git diff --name-only -z "${BASE_SHA}" "${HEAD_SHA}") + + echo "runtime=${runtime}" >> "$GITHUB_OUTPUT" + { + echo "### Changed file check" + echo "" + echo "Changed files: ${changed_count}" + echo "Runtime-impacting changes: ${runtime}" + if [[ "${runtime}" != "true" ]]; then + echo "" + echo "Only non-runtime files changed, so build and tile-generation jobs were skipped." + fi + } >> "$GITHUB_STEP_SUMMARY" + download-pbf: name: Download PBF fixture runs-on: ubuntu-latest timeout-minutes: 10 + needs: changes + if: ${{ needs.changes.outputs.runtime == 'true' }} outputs: pbf-available: ${{ steps.download.outputs.pbf-available }} @@ -103,7 +170,9 @@ jobs: - name: Build openmaptiles-compatible mbtiles files of Liechtenstein run: | ${{ github.workspace }}\build\RelWithDebInfo\tilemaker.exe ${{ env.AREA }}.osm.pbf --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output=${{ env.AREA }}.pmtiles --verbose + ${{ github.workspace }}\build\RelWithDebInfo\tilemaker.exe ${{ env.AREA }}.osm.pbf --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output=${{ env.AREA }}-repeat.pmtiles --verbose ${{ github.workspace }}\build\RelWithDebInfo\tilemaker.exe ${{ env.AREA }}.osm.pbf --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output=${{ env.AREA }}.mbtiles --store "${{ runner.temp }}\osm_store" --verbose + ${{ github.workspace }}\build\RelWithDebInfo\tilemaker.exe ${{ env.AREA }}.osm.pbf --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output=${{ env.AREA }}-repeat.mbtiles --store "${{ runner.temp }}\osm_store_repeat" --verbose - name: Upload generated tiles uses: actions/upload-artifact@v7 @@ -113,6 +182,8 @@ jobs: path: | ${{ env.AREA }}.mbtiles ${{ env.AREA }}.pmtiles + ${{ env.AREA }}-repeat.mbtiles + ${{ env.AREA }}-repeat.pmtiles - name: 'Upload compiled executable' uses: actions/upload-artifact@v7 @@ -126,6 +197,7 @@ jobs: unix-build: strategy: + fail-fast: false matrix: include: - os: ubuntu-22.04 @@ -175,7 +247,9 @@ jobs: - name: Build openmaptiles-compatible mbtiles files of Liechtenstein run: | "${{ github.workspace }}/build/${{ matrix.executable }}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}.pmtiles" --verbose + "${{ github.workspace }}/build/${{ matrix.executable }}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-repeat.pmtiles" --verbose "${{ github.workspace }}/build/${{ matrix.executable }}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}.mbtiles" --verbose --store "${{ runner.temp }}/store" + "${{ github.workspace }}/build/${{ matrix.executable }}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-repeat.mbtiles" --verbose --store "${{ runner.temp }}/store-repeat" - name: Upload generated tiles uses: actions/upload-artifact@v7 @@ -185,6 +259,8 @@ jobs: path: | ${{ env.AREA }}.mbtiles ${{ env.AREA }}.pmtiles + ${{ env.AREA }}-repeat.mbtiles + ${{ env.AREA }}-repeat.pmtiles - name: 'Upload compiled executable' uses: actions/upload-artifact@v7 @@ -197,6 +273,7 @@ jobs: unix-makefile-build: strategy: + fail-fast: false matrix: include: - os: ubuntu-22.04 @@ -236,7 +313,9 @@ jobs: - name: Build openmaptiles-compatible mbtiles files of Liechtenstein run: | ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}.pmtiles" --verbose + ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-repeat.pmtiles" --verbose ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}.mbtiles" --verbose --store "${{ runner.temp }}/store" + ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-repeat.mbtiles" --verbose --store "${{ runner.temp }}/store-repeat" - name: Upload generated tiles uses: actions/upload-artifact@v7 @@ -246,6 +325,8 @@ jobs: path: | ${{ env.AREA }}.mbtiles ${{ env.AREA }}.pmtiles + ${{ env.AREA }}-repeat.mbtiles + ${{ env.AREA }}-repeat.pmtiles Github-Action: @@ -270,12 +351,24 @@ jobs: input: ${{ env.AREA }}.osm.pbf output: ${{ env.AREA }}.pmtiles + - name: Build repeat openmaptiles-compatible pmtiles file + uses: ./ + with: + input: ${{ env.AREA }}.osm.pbf + output: ${{ env.AREA }}-repeat.pmtiles + - name: Build openmaptiles-compatible mbtiles file uses: ./ with: input: ${{ env.AREA }}.osm.pbf output: ${{ env.AREA }}.mbtiles + - name: Build repeat openmaptiles-compatible mbtiles file + uses: ./ + with: + input: ${{ env.AREA }}.osm.pbf + output: ${{ env.AREA }}-repeat.mbtiles + - name: Upload generated tiles uses: actions/upload-artifact@v7 with: @@ -284,6 +377,44 @@ jobs: path: | ${{ env.AREA }}.mbtiles ${{ env.AREA }}.pmtiles + ${{ env.AREA }}-repeat.mbtiles + ${{ env.AREA }}-repeat.pmtiles + + verify-generated-tiles: + name: Verify generated tiles + runs-on: ubuntu-latest + timeout-minutes: 20 + needs: + - Windows-Build + - unix-build + - unix-makefile-build + - Github-Action + env: + PMTILES_VERSION: 1.30.2 + + steps: + - uses: actions/checkout@v6 + with: + sparse-checkout: .github/scripts/verify-generated-tiles.py + sparse-checkout-cone-mode: false + + - name: Download generated tiles + uses: actions/download-artifact@v8 + with: + pattern: tile-outputs-* + path: generated-tiles + + - name: Install pmtiles + run: | + curl -fsSL -o pmtiles.tar.gz "https://github.com/protomaps/go-pmtiles/releases/download/v${PMTILES_VERSION}/go-pmtiles_${PMTILES_VERSION}_Linux_x86_64.tar.gz" + tar -xzf pmtiles.tar.gz + chmod +x pmtiles + sudo mv pmtiles /usr/local/bin/pmtiles + pmtiles version + + - name: Verify generated tile archives + run: | + python3 .github/scripts/verify-generated-tiles.py generated-tiles docker-build: env: @@ -292,9 +423,10 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 45 + needs: changes + if: ${{ needs.changes.outputs.runtime == 'true' && github.ref != 'refs/heads/master' }} permissions: contents: read - packages: write steps: - name: Checkout repository @@ -318,7 +450,6 @@ jobs: - name: Build Docker image uses: docker/build-push-action@v7 - if: ${{ github.ref != 'refs/heads/master'}} with: context: . push: false @@ -326,9 +457,42 @@ jobs: labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max + + docker-publish: + env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + + runs-on: ubuntu-latest + timeout-minutes: 45 + needs: changes + if: ${{ needs.changes.outputs.runtime == 'true' && github.ref == 'refs/heads/master' }} + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Log in to the Container registry + uses: docker/login-action@v4 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v6 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + - name: Build and push Docker image uses: docker/build-push-action@v7 - if: ${{ github.ref == 'refs/heads/master'}} with: context: . push: true diff --git a/.gitignore b/.gitignore index 18805e51..b80da080 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,10 @@ tilemaker.dSYM/ # editor temporaries *~ +# python cache files +__pycache__/ +*.pyc + # vagrant local stuff .vagrant/ From 57b1c3193cddcd674203a504b54f00469933b76d Mon Sep 17 00:00:00 2001 From: Symmetricity <184246+Symmetricity@users.noreply.github.com> Date: Wed, 13 May 2026 06:59:46 +0200 Subject: [PATCH 03/10] Refine generated tile verification Decode MVT geometry before calculating semantic hashes so equivalent point ordering and polygon ring rotation do not fail CI as content changes. Keep the published Docker action output in repeat-output checks, but exclude it from cross-runner comparisons because that action uses the ghcr.io master image rather than the PR-built binaries. --- .github/scripts/verify-generated-tiles.py | 109 ++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/.github/scripts/verify-generated-tiles.py b/.github/scripts/verify-generated-tiles.py index 5367ecab..830c6227 100644 --- a/.github/scripts/verify-generated-tiles.py +++ b/.github/scripts/verify-generated-tiles.py @@ -209,6 +209,112 @@ def packed_varints(data): return values +def decode_geometry(geometry): + pos = 0 + x = 0 + y = 0 + paths = [] + current = [] + + while pos < len(geometry): + command = geometry[pos] & 7 + count = geometry[pos] >> 3 + pos += 1 + + if command == 1: + for _ in range(count): + if current: + paths.append(current) + current = [] + x += zigzag_decode(geometry[pos]) + y += zigzag_decode(geometry[pos + 1]) + pos += 2 + current.append([x, y]) + elif command == 2: + for _ in range(count): + if not current: + raise RuntimeError("MVT LineTo command without an active path") + x += zigzag_decode(geometry[pos]) + y += zigzag_decode(geometry[pos + 1]) + pos += 2 + current.append([x, y]) + elif command == 7: + for _ in range(count): + if current: + paths.append(current) + current = [] + else: + raise RuntimeError(f"unsupported MVT geometry command {command}") + + if current: + paths.append(current) + return paths + + +def canonical_geometry(geometry_type, geometry): + paths = decode_geometry(geometry) + if geometry_type == 1: + points = [point for path in paths for point in path] + points.sort(key=canonical_json) + return ["points", points] + if geometry_type == 3: + return ["polygons", canonical_polygons(paths)] + return ["geometry", paths] + + +def canonical_polygons(paths): + polygons = [] + current = None + outer_area = 0 + + for path in paths: + area = ring_area(path) + ring = canonical_ring(path, area) + + if area == 0: + polygons.append(["degenerate", ring]) + elif outer_area == 0 or area == outer_area: + if current: + current[1].sort(key=canonical_json) + polygons.append(current) + outer_area = area + current = [ring, []] + elif current: + current[1].append(ring) + else: + polygons.append(["orphan", ring]) + + if current: + current[1].sort(key=canonical_json) + polygons.append(current) + + polygons.sort(key=canonical_json) + return polygons + + +def canonical_ring(path, area): + ring = path + if len(ring) > 1 and ring[0] == ring[-1]: + ring = ring[:-1] + if not ring: + return [0, []] + + rotations = [ring[index:] + ring[:index] for index in range(len(ring))] + return [area, min(rotations, key=canonical_json)] + + +def ring_area(ring): + area = 0 + for index, point in enumerate(ring): + next_point = ring[(index + 1) % len(ring)] + area += point[0] * next_point[1] - next_point[0] * point[1] + if area < 0: + return -1 + if area > 0: + return 1 + return 0 + + def decode_mvt_value(data): values = [] for field, wire_type, value in protobuf_fields(data): @@ -254,6 +360,7 @@ def decode_mvt_feature(data, keys, values): elif field == 4: geometry = packed_varints(value) tags.sort(key=canonical_json) + geometry = canonical_geometry(geometry_type, geometry) return [feature_id, geometry_type, tags, geometry] @@ -563,6 +670,8 @@ def check_cross_runner(contents, suffix): for path, content in contents.items(): if path.name.endswith(f"-repeat.{suffix}"): continue + if path.parent.name == "tile-outputs-github-action": + continue groups.setdefault(path.name, []).append((path, content)) for name, values in sorted(groups.items()): reference_path, reference = values[0] From 8332f8fdbad9fb661ca87d0f847b92316ec04e4c Mon Sep 17 00:00:00 2001 From: Symmetricity <184246+Symmetricity@users.noreply.github.com> Date: Wed, 13 May 2026 22:56:22 +0200 Subject: [PATCH 04/10] Fail fast on tile generation errors CI generated tile artifacts can otherwise be incomplete when one tilemaker invocation fails but a later command in the same step succeeds. This made the Windows CMake job upload only three of the four expected tile files while the job itself still passed. Wrap direct tilemaker calls so each output is checked immediately, and verify generated files before artifact upload. This makes the failing output visible at the generation step instead of deferring the problem to the verifier or silently uploading partial artifacts. --- .github/workflows/ci.yml | 91 ++++++++++++++++++++++++++++++++++------ 1 file changed, 79 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c4412c08..621fe131 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -169,10 +169,27 @@ jobs: - name: Build openmaptiles-compatible mbtiles files of Liechtenstein run: | - ${{ github.workspace }}\build\RelWithDebInfo\tilemaker.exe ${{ env.AREA }}.osm.pbf --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output=${{ env.AREA }}.pmtiles --verbose - ${{ github.workspace }}\build\RelWithDebInfo\tilemaker.exe ${{ env.AREA }}.osm.pbf --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output=${{ env.AREA }}-repeat.pmtiles --verbose - ${{ github.workspace }}\build\RelWithDebInfo\tilemaker.exe ${{ env.AREA }}.osm.pbf --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output=${{ env.AREA }}.mbtiles --store "${{ runner.temp }}\osm_store" --verbose - ${{ github.workspace }}\build\RelWithDebInfo\tilemaker.exe ${{ env.AREA }}.osm.pbf --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output=${{ env.AREA }}-repeat.mbtiles --store "${{ runner.temp }}\osm_store_repeat" --verbose + $tilemaker = "${{ github.workspace }}\build\RelWithDebInfo\tilemaker.exe" + function Invoke-Tilemaker { + param([string] $Output, [string[]] $Arguments) + Write-Host "::group::Build $Output" + & $tilemaker @Arguments + $status = $LASTEXITCODE + Write-Host "::endgroup::" + if ($status -ne 0) { + Write-Error "tilemaker failed while writing $Output with exit code $status" + exit $status + } + if (!(Test-Path -LiteralPath $Output) -or (Get-Item -LiteralPath $Output).Length -eq 0) { + Write-Error "tilemaker did not create $Output" + exit 1 + } + } + + Invoke-Tilemaker "${{ env.AREA }}.pmtiles" @("${{ env.AREA }}.osm.pbf", "--config=resources/config-openmaptiles.json", "--process=resources/process-openmaptiles.lua", "--output=${{ env.AREA }}.pmtiles", "--verbose") + Invoke-Tilemaker "${{ env.AREA }}-repeat.pmtiles" @("${{ env.AREA }}.osm.pbf", "--config=resources/config-openmaptiles.json", "--process=resources/process-openmaptiles.lua", "--output=${{ env.AREA }}-repeat.pmtiles", "--verbose") + Invoke-Tilemaker "${{ env.AREA }}.mbtiles" @("${{ env.AREA }}.osm.pbf", "--config=resources/config-openmaptiles.json", "--process=resources/process-openmaptiles.lua", "--output=${{ env.AREA }}.mbtiles", "--store", "${{ runner.temp }}\osm_store", "--verbose") + Invoke-Tilemaker "${{ env.AREA }}-repeat.mbtiles" @("${{ env.AREA }}.osm.pbf", "--config=resources/config-openmaptiles.json", "--process=resources/process-openmaptiles.lua", "--output=${{ env.AREA }}-repeat.mbtiles", "--store", "${{ runner.temp }}\osm_store_repeat", "--verbose") - name: Upload generated tiles uses: actions/upload-artifact@v7 @@ -246,10 +263,29 @@ jobs: - name: Build openmaptiles-compatible mbtiles files of Liechtenstein run: | - "${{ github.workspace }}/build/${{ matrix.executable }}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}.pmtiles" --verbose - "${{ github.workspace }}/build/${{ matrix.executable }}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-repeat.pmtiles" --verbose - "${{ github.workspace }}/build/${{ matrix.executable }}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}.mbtiles" --verbose --store "${{ runner.temp }}/store" - "${{ github.workspace }}/build/${{ matrix.executable }}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-repeat.mbtiles" --verbose --store "${{ runner.temp }}/store-repeat" + run_tilemaker() { + output="$1" + shift + echo "::group::Build ${output}" + if "$@"; then + echo "::endgroup::" + else + status="$?" + echo "::endgroup::" + echo "::error title=Tile generation failed::tilemaker failed while writing ${output} with exit code ${status}" + exit "${status}" + fi + if [ ! -s "${output}" ]; then + echo "::error title=Tile output missing::tilemaker did not create ${output}" + exit 1 + fi + } + + tilemaker="${{ github.workspace }}/build/${{ matrix.executable }}" + run_tilemaker "${{ env.AREA }}.pmtiles" "${tilemaker}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}.pmtiles" --verbose + run_tilemaker "${{ env.AREA }}-repeat.pmtiles" "${tilemaker}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-repeat.pmtiles" --verbose + run_tilemaker "${{ env.AREA }}.mbtiles" "${tilemaker}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}.mbtiles" --verbose --store "${{ runner.temp }}/store" + run_tilemaker "${{ env.AREA }}-repeat.mbtiles" "${tilemaker}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-repeat.mbtiles" --verbose --store "${{ runner.temp }}/store-repeat" - name: Upload generated tiles uses: actions/upload-artifact@v7 @@ -312,10 +348,28 @@ jobs: - name: Build openmaptiles-compatible mbtiles files of Liechtenstein run: | - ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}.pmtiles" --verbose - ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-repeat.pmtiles" --verbose - ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}.mbtiles" --verbose --store "${{ runner.temp }}/store" - ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-repeat.mbtiles" --verbose --store "${{ runner.temp }}/store-repeat" + run_tilemaker() { + output="$1" + shift + echo "::group::Build ${output}" + if "$@"; then + echo "::endgroup::" + else + status="$?" + echo "::endgroup::" + echo "::error title=Tile generation failed::tilemaker failed while writing ${output} with exit code ${status}" + exit "${status}" + fi + if [ ! -s "${output}" ]; then + echo "::error title=Tile output missing::tilemaker did not create ${output}" + exit 1 + fi + } + + run_tilemaker "${{ env.AREA }}.pmtiles" ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}.pmtiles" --verbose + run_tilemaker "${{ env.AREA }}-repeat.pmtiles" ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-repeat.pmtiles" --verbose + run_tilemaker "${{ env.AREA }}.mbtiles" ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}.mbtiles" --verbose --store "${{ runner.temp }}/store" + run_tilemaker "${{ env.AREA }}-repeat.mbtiles" ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-repeat.mbtiles" --verbose --store "${{ runner.temp }}/store-repeat" - name: Upload generated tiles uses: actions/upload-artifact@v7 @@ -369,6 +423,19 @@ jobs: input: ${{ env.AREA }}.osm.pbf output: ${{ env.AREA }}-repeat.mbtiles + - name: Check generated tile files + run: | + for output in \ + "${AREA}.mbtiles" \ + "${AREA}.pmtiles" \ + "${AREA}-repeat.mbtiles" \ + "${AREA}-repeat.pmtiles"; do + if [ ! -s "${output}" ]; then + echo "::error title=Tile output missing::tilemaker did not create ${output}" + exit 1 + fi + done + - name: Upload generated tiles uses: actions/upload-artifact@v7 with: From d6478bcc2c45b79a0ca8f0fd8d11cd9a0e18c83e Mon Sep 17 00:00:00 2001 From: Symmetricity <184246+Symmetricity@users.noreply.github.com> Date: Sat, 16 May 2026 08:24:48 +0200 Subject: [PATCH 05/10] Speed up generated tile verification Hash ordered tile payloads before falling back to semantic decoding so repeat outputs that differ only in archive layout can pass without expensive MVT canonicalization. When raw tile bytes differ, decode only the differing tiles and stop at the first semantic mismatch. This keeps the existing verification behaviour while reducing work for deterministic runs and making failures report quickly. --- .github/scripts/verify-generated-tiles.py | 158 ++++++++++++++++------ 1 file changed, 113 insertions(+), 45 deletions(-) diff --git a/.github/scripts/verify-generated-tiles.py b/.github/scripts/verify-generated-tiles.py index 830c6227..47bf201f 100644 --- a/.github/scripts/verify-generated-tiles.py +++ b/.github/scripts/verify-generated-tiles.py @@ -19,12 +19,12 @@ class ArchiveContent: - def __init__(self, raw_hash, semantic_hash, raw_tiles, semantic_tiles, layer_counts): + def __init__(self, raw_hash, raw_tiles, metadata_hash): self.raw_hash = raw_hash - self.semantic_hash = semantic_hash self.raw_tiles = raw_tiles - self.semantic_tiles = semantic_tiles - self.layer_counts = layer_counts + self.metadata_hash = metadata_hash + self.semantic_tiles = {} + self.layer_counts = {} def error(path, message): @@ -410,16 +410,23 @@ def semantic_tile_content(data): return digest, layer_counts -def add_tile_to_digests(raw_digest, semantic_digest, raw_tiles, semantic_tiles, layer_counts, z, x, y, payload): +def content_summary(content): + return f"raw {content.raw_hash}" + + +def add_tile_to_raw_digest(raw_digest, raw_tiles, z, x, y, payload): tile = (z, x, y) raw_hash = hashlib.sha256(payload).hexdigest() - semantic_hash, counts = semantic_tile_content(payload) raw_tiles[tile] = raw_hash - semantic_tiles[tile] = semantic_hash - layer_counts[tile] = counts raw_digest.update(f"T\t{z}\t{x}\t{y}\t{len(payload)}\t{raw_hash}\n".encode()) - semantic_digest.update(f"T\t{z}\t{x}\t{y}\t{semantic_hash}\n".encode()) + + +def add_tile_to_semantic_maps(content, z, x, y, payload): + tile = (z, x, y) + semantic_hash, counts = semantic_tile_content(payload) + content.semantic_tiles[tile] = semantic_hash + content.layer_counts[tile] = counts def mbtiles_fingerprint(path): @@ -441,28 +448,42 @@ def mbtiles_fingerprint(path): raise RuntimeError("metadata table is empty") raw_digest = hashlib.sha256() - semantic_digest = hashlib.sha256() + metadata_digest = hashlib.sha256() for name, value in metadata_rows: metadata = f"M\t{name}\t{canonical_metadata_value(value)}\n".encode("utf-8", "surrogateescape") raw_digest.update(metadata) - semantic_digest.update(metadata) + metadata_digest.update(metadata) raw_tiles = {} - semantic_tiles = {} - layer_counts = {} for z, x, y, tile_data in cur.execute("select zoom_level, tile_column, tile_row, tile_data from tiles order by zoom_level, tile_column, tile_row"): payload = decompress_payload(bytes(tile_data)) - add_tile_to_digests(raw_digest, semantic_digest, raw_tiles, semantic_tiles, layer_counts, z, x, y, payload) + add_tile_to_raw_digest(raw_digest, raw_tiles, z, x, y, payload) con.close() - content = ArchiveContent(raw_digest.hexdigest(), semantic_digest.hexdigest(), raw_tiles, semantic_tiles, layer_counts) + content = ArchiveContent(raw_digest.hexdigest(), raw_tiles, metadata_digest.hexdigest()) print( f"{archive_label(path)}: {tile_count} tiles, zoom {minzoom}-{maxzoom}, " - f"{len(metadata_rows)} metadata rows, raw {content.raw_hash}, semantic {content.semantic_hash}" + f"{len(metadata_rows)} metadata rows, raw {content.raw_hash}" ) return content +def mbtiles_decode_semantic_tiles(path, content, tiles): + con = sqlite3.connect(f"file:{path}?mode=ro", uri=True) + cur = con.cursor() + for z, x, y in sorted(tiles): + row = cur.execute( + "select tile_data from tiles where zoom_level=? and tile_column=? and tile_row=?", + (z, x, y), + ).fetchone() + if row is None: + continue + tile_data = row[0] + payload = decompress_payload(bytes(tile_data)) + add_tile_to_semantic_maps(content, z, x, y, payload) + con.close() + + def canonical_metadata_value(value): if value is None: return "" @@ -553,26 +574,35 @@ def pmtiles_fingerprint(path): entries = [] collect_pmtiles_entries(data, header, header["root_dir_offset"], header["root_dir_bytes"], entries) raw_digest = hashlib.sha256() - semantic_digest = hashlib.sha256() metadata_hash = hashlib.sha256(metadata).hexdigest() raw_digest.update(f"M\t{metadata_hash}\n".encode()) - semantic_digest.update(f"M\t{metadata_hash}\n".encode()) raw_tiles = {} - semantic_tiles = {} - layer_counts = {} for z, x, y, offset, length in sorted(entries): payload = decompress_pmtiles(data[offset:offset + length], header["tile_compression"]) - add_tile_to_digests(raw_digest, semantic_digest, raw_tiles, semantic_tiles, layer_counts, z, x, y, payload) + add_tile_to_raw_digest(raw_digest, raw_tiles, z, x, y, payload) - content = ArchiveContent(raw_digest.hexdigest(), semantic_digest.hexdigest(), raw_tiles, semantic_tiles, layer_counts) + content = ArchiveContent(raw_digest.hexdigest(), raw_tiles, metadata_hash) print( f"{archive_label(path)}: {len(entries)} addressed tiles, " - f"raw {content.raw_hash}, semantic {content.semantic_hash}" + f"raw {content.raw_hash}" ) return content +def pmtiles_decode_semantic_tiles(path, content, tiles): + data = path.read_bytes() + header = pmtiles_header(data) + entries = [] + collect_pmtiles_entries(data, header, header["root_dir_offset"], header["root_dir_bytes"], entries) + wanted = set(tiles) + for z, x, y, offset, length in sorted(entries): + if (z, x, y) not in wanted: + continue + payload = decompress_pmtiles(data[offset:offset + length], header["tile_compression"]) + add_tile_to_semantic_maps(content, z, x, y, payload) + + def fingerprint_archives(paths, fingerprint, failure_message): cache = {} contents = {} @@ -599,7 +629,7 @@ def fingerprint_archives(paths, fingerprint, failure_message): if source_path != path: print( f"{archive_label(path)}: same archive as {archive_label(source_path)}; " - f"raw {content_hash.raw_hash}, semantic {content_hash.semantic_hash}" + f"{content_summary(content_hash)}" ) return contents @@ -639,22 +669,60 @@ def semantic_tile_difference(left, right, tile): return f"first semantic difference at {tile_label(tile)}" -def compare_content(path, content, other_path, other_content, relation): - if content.semantic_hash != other_content.semantic_hash: - error( - path, - f"semantic content differs from {relation} {archive_label(other_path)}; " - f"{tile_difference(content, other_content, True)}", - ) - elif content.raw_hash != other_content.raw_hash: - notice( - path, - f"raw tile bytes differ from {relation} {archive_label(other_path)}, " - f"but semantic content matches; {tile_difference(content, other_content, False)}", - ) +def raw_differing_tiles(left, right): + differing = [] + for tile in sorted(set(left.raw_tiles) | set(right.raw_tiles)): + if left.raw_tiles.get(tile) != right.raw_tiles.get(tile): + differing.append(tile) + return differing + + +def ensure_semantic_tiles(path, content, tiles, semantic_decoder, failure_message): + needed = [tile for tile in tiles if tile in content.raw_tiles and tile not in content.semantic_tiles] + if not needed: + return True + try: + semantic_decoder(path, content, needed) + except Exception as err: + error(path, f"{failure_message}: {err}") + return False + + return True + + +def compare_content(path, content, other_path, other_content, relation, semantic_decoder, failure_message): + if content.raw_hash == other_content.raw_hash: + return + + if content.metadata_hash != other_content.metadata_hash: + error(path, f"metadata differs from {relation} {archive_label(other_path)}") + return + + differing_tiles = raw_differing_tiles(content, other_content) + + for tile in differing_tiles: + if not ensure_semantic_tiles(path, content, [tile], semantic_decoder, failure_message): + return + + if not ensure_semantic_tiles(other_path, other_content, [tile], semantic_decoder, failure_message): + return + + if content.semantic_tiles.get(tile) != other_content.semantic_tiles.get(tile): + error( + path, + f"semantic content differs from {relation} {archive_label(other_path)}; " + f"{tile_difference(content, other_content, True)}", + ) + return + + notice( + path, + f"raw tile bytes differ from {relation} {archive_label(other_path)}, " + f"but semantic content matches; {tile_difference(content, other_content, False)}", + ) -def check_repeat(contents, suffix): +def check_repeat(contents, suffix, semantic_decoder, failure_message): for path, content in sorted(contents.items()): if path.name.endswith(f"-repeat.{suffix}"): continue @@ -662,10 +730,10 @@ def check_repeat(contents, suffix): if repeat not in contents: error(path, f"missing repeat output {archive_label(repeat)}") else: - compare_content(path, content, repeat, contents[repeat], "repeat output") + compare_content(path, content, repeat, contents[repeat], "repeat output", semantic_decoder, failure_message) -def check_cross_runner(contents, suffix): +def check_cross_runner(contents, suffix, semantic_decoder, failure_message): groups = {} for path, content in contents.items(): if path.name.endswith(f"-repeat.{suffix}"): @@ -676,7 +744,7 @@ def check_cross_runner(contents, suffix): for name, values in sorted(groups.items()): reference_path, reference = values[0] for path, content in values[1:]: - compare_content(path, content, reference_path, reference, "output") + compare_content(path, content, reference_path, reference, "output", semantic_decoder, failure_message) def main(): @@ -697,10 +765,10 @@ def main(): "PMTiles archive failed verification", ) - check_repeat(mbtiles, "mbtiles") - check_repeat(pmtiles, "pmtiles") - check_cross_runner(mbtiles, "mbtiles") - check_cross_runner(pmtiles, "pmtiles") + check_repeat(mbtiles, "mbtiles", mbtiles_decode_semantic_tiles, "MBTiles archive failed verification") + check_repeat(pmtiles, "pmtiles", pmtiles_decode_semantic_tiles, "PMTiles archive failed verification") + check_cross_runner(mbtiles, "mbtiles", mbtiles_decode_semantic_tiles, "MBTiles archive failed verification") + check_cross_runner(pmtiles, "pmtiles", pmtiles_decode_semantic_tiles, "PMTiles archive failed verification") return status From 960c8b0ee170ffbed6a9b99196d587e719ed5876 Mon Sep 17 00:00:00 2001 From: Symmetricity <184246+Symmetricity@users.noreply.github.com> Date: Sat, 16 May 2026 13:15:27 +0200 Subject: [PATCH 06/10] Parallelize generated tile verification Generated tile verification can spend a long time canonicalizing MVT payloads when PMTiles archives differ at the raw tile-byte level. Large fixtures can have tens of thousands of raw-different tiles even when the semantic content matches. Compare PMTiles tile pairs in worker processes and report progress with an ETA. This keeps the existing semantic comparison while making larger local and CI artifacts practical to diagnose. --- .github/scripts/verify-generated-tiles.py | 256 ++++++++++++++++++++-- 1 file changed, 240 insertions(+), 16 deletions(-) diff --git a/.github/scripts/verify-generated-tiles.py b/.github/scripts/verify-generated-tiles.py index 47bf201f..654bee4a 100644 --- a/.github/scripts/verify-generated-tiles.py +++ b/.github/scripts/verify-generated-tiles.py @@ -1,18 +1,29 @@ #!/usr/bin/env python3 +import concurrent.futures import gzip import hashlib import json +import os import pathlib import sqlite3 import struct import subprocess import sys +import time import zlib COMPRESSION_NONE = 1 COMPRESSION_GZIP = 2 +PROGRESS_INTERVAL = 1024 +SEMANTIC_COMPARE_CHUNK_SIZE = 128 + + +worker_left_archive = None +worker_right_archive = None +worker_left_compression = None +worker_right_compression = None status = 0 @@ -39,6 +50,43 @@ def notice(path, message): print(f"::notice title={label}::{label}: {message}") +def progress(path, message): + print(f"{archive_label(path)}: {message}", flush=True) + + +def format_duration(seconds): + seconds = int(seconds) + minutes, seconds = divmod(seconds, 60) + hours, minutes = divmod(minutes, 60) + if hours: + return f"{hours}h{minutes:02d}m{seconds:02d}s" + if minutes: + return f"{minutes}m{seconds:02d}s" + return f"{seconds}s" + + +def progress_eta(path, action, current, total, started): + elapsed = time.monotonic() - started + percent = (current / total) * 100 if total else 100.0 + if current: + remaining = (elapsed / current) * (total - current) + eta = format_duration(remaining) + else: + eta = "unknown" + progress( + path, + f"{action}: {current}/{total} ({percent:.1f}%), " + f"elapsed {format_duration(elapsed)}, ETA {eta}", + ) + + +def semantic_worker_count(): + value = os.environ.get("VERIFY_TILE_WORKERS") + if value: + return max(1, int(value)) + return max(1, min(4, os.cpu_count() or 1)) + + def archive_sha256(path): digest = hashlib.sha256() with path.open("rb") as handle: @@ -429,6 +477,12 @@ def add_tile_to_semantic_maps(content, z, x, y, payload): content.layer_counts[tile] = counts +def set_tile_semantic_maps(content, z, x, y, semantic_hash, counts): + tile = (z, x, y) + content.semantic_tiles[tile] = semantic_hash + content.layer_counts[tile] = counts + + def mbtiles_fingerprint(path): con = sqlite3.connect(f"file:{path}?mode=ro", uri=True) cur = con.cursor() @@ -590,17 +644,153 @@ def pmtiles_fingerprint(path): return content +def pmtiles_entry_index(path): + data = path.read_bytes() + header = pmtiles_header(data) + entries = [] + collect_pmtiles_entries(data, header, header["root_dir_offset"], header["root_dir_bytes"], entries) + return header["tile_compression"], { + (z, x, y): (offset, length) + for z, x, y, offset, length in entries + } + + +def initialize_pmtiles_compare_worker(left_path, left_compression, right_path, right_compression): + global worker_left_archive + global worker_right_archive + global worker_left_compression + global worker_right_compression + + worker_left_archive = open(left_path, "rb") + worker_right_archive = open(right_path, "rb") + worker_left_compression = left_compression + worker_right_compression = right_compression + + +def read_worker_payload(archive, offset, length, compression): + archive.seek(offset) + return decompress_pmtiles(archive.read(length), compression) + + +def compare_pmtiles_semantic_batch(batch): + for tile, left_offset, left_length, right_offset, right_length in batch: + left_payload = read_worker_payload( + worker_left_archive, + left_offset, + left_length, + worker_left_compression, + ) + right_payload = read_worker_payload( + worker_right_archive, + right_offset, + right_length, + worker_right_compression, + ) + left_hash, left_counts = semantic_tile_content(left_payload) + right_hash, right_counts = semantic_tile_content(right_payload) + if left_hash != right_hash: + return { + "same": False, + "tile": tile, + "message": semantic_counts_difference(tile, left_counts, right_counts), + } + return {"same": True, "count": len(batch)} + + +def compare_pmtiles_semantic_content(path, content, other_path, other_content, differing_tiles, relation, failure_message): + left_compression, left_index = pmtiles_entry_index(path) + right_compression, right_index = pmtiles_entry_index(other_path) + batches = [] + batch = [] + for tile in differing_tiles: + if tile not in left_index: + error(path, f"{tile_label(tile)} is missing from output") + return False + if tile not in right_index: + error(path, f"{tile_label(tile)} is missing from comparison output") + return False + left_offset, left_length = left_index[tile] + right_offset, right_length = right_index[tile] + batch.append((tile, left_offset, left_length, right_offset, right_length)) + if len(batch) == SEMANTIC_COMPARE_CHUNK_SIZE: + batches.append(batch) + batch = [] + if batch: + batches.append(batch) + + workers = semantic_worker_count() + progress( + path, + f"checking {len(differing_tiles)} semantic tiles with {workers} worker processes " + f"against {relation} {archive_label(other_path)}", + ) + started = time.monotonic() + checked = 0 + progress_eta(path, "checked semantic tiles", checked, len(differing_tiles), started) + + executor = concurrent.futures.ProcessPoolExecutor( + max_workers=workers, + initializer=initialize_pmtiles_compare_worker, + initargs=(str(path), left_compression, str(other_path), right_compression), + ) + futures = { + executor.submit(compare_pmtiles_semantic_batch, tile_batch): len(tile_batch) + for tile_batch in batches + } + try: + for future in concurrent.futures.as_completed(futures): + result = future.result() + checked += futures[future] + if not result["same"]: + tile = result["tile"] + error( + path, + f"semantic content differs from {relation} {archive_label(other_path)}; " + f"{result['message']}", + ) + for pending in futures: + pending.cancel() + executor.shutdown(wait=False, cancel_futures=True) + return False + if checked % PROGRESS_INTERVAL == 0 or checked == len(differing_tiles): + progress_eta(path, "checked semantic tiles", checked, len(differing_tiles), started) + except Exception as err: + error(path, f"{failure_message}: {err}") + executor.shutdown(wait=False, cancel_futures=True) + return False + + executor.shutdown() + progress( + path, + f"checked {len(differing_tiles)}/{len(differing_tiles)} semantic tiles " + f"against {relation} {archive_label(other_path)}", + ) + return True + + def pmtiles_decode_semantic_tiles(path, content, tiles): data = path.read_bytes() header = pmtiles_header(data) entries = [] collect_pmtiles_entries(data, header, header["root_dir_offset"], header["root_dir_bytes"], entries) wanted = set(tiles) + semantic_cache = {} + decoded = 0 + started = time.monotonic() + progress_eta(path, "decoded semantic tiles", decoded, len(wanted), started) for z, x, y, offset, length in sorted(entries): if (z, x, y) not in wanted: continue - payload = decompress_pmtiles(data[offset:offset + length], header["tile_compression"]) - add_tile_to_semantic_maps(content, z, x, y, payload) + payload_key = (offset, length) + if payload_key not in semantic_cache: + payload = decompress_pmtiles(data[offset:offset + length], header["tile_compression"]) + semantic_cache[payload_key] = semantic_tile_content(payload) + semantic_hash, counts = semantic_cache[payload_key] + set_tile_semantic_maps(content, z, x, y, semantic_hash, counts) + decoded += 1 + if decoded % PROGRESS_INTERVAL == 0 or decoded == len(wanted): + progress_eta(path, "decoded semantic tiles", decoded, len(wanted), started) + progress(path, f"decoded {len(semantic_cache)} unique semantic tile payloads for {decoded} addressed tiles") def fingerprint_archives(paths, fingerprint, failure_message): @@ -658,15 +848,19 @@ def tile_difference(left, right, semantic): def semantic_tile_difference(left, right, tile): left_counts = left.layer_counts.get(tile, {}) right_counts = right.layer_counts.get(tile, {}) + return semantic_counts_difference(tile, left_counts, right_counts) + + +def semantic_counts_difference(tile, left_counts, right_counts): for layer in sorted(set(left_counts) | set(right_counts)): left_count = left_counts.get(layer, 0) right_count = right_counts.get(layer, 0) if left_count != right_count: return ( - f"first semantic difference at {tile_label(tile)}: " + f"semantic difference at {tile_label(tile)}: " f"layer {layer} has {left_count} vs {right_count} features" ) - return f"first semantic difference at {tile_label(tile)}" + return f"semantic difference at {tile_label(tile)}" def raw_differing_tiles(left, right): @@ -690,7 +884,7 @@ def ensure_semantic_tiles(path, content, tiles, semantic_decoder, failure_messag return True -def compare_content(path, content, other_path, other_content, relation, semantic_decoder, failure_message): +def compare_content(path, content, other_path, other_content, relation, semantic_decoder, failure_message, semantic_comparator=None): if content.raw_hash == other_content.raw_hash: return @@ -699,14 +893,36 @@ def compare_content(path, content, other_path, other_content, relation, semantic return differing_tiles = raw_differing_tiles(content, other_content) + progress( + path, + f"semantic comparison needed for {len(differing_tiles)} raw-different tiles " + f"against {relation} {archive_label(other_path)}", + ) - for tile in differing_tiles: - if not ensure_semantic_tiles(path, content, [tile], semantic_decoder, failure_message): - return + if semantic_comparator: + if semantic_comparator(path, content, other_path, other_content, differing_tiles, relation, failure_message): + notice( + path, + f"raw tile bytes differ from {relation} {archive_label(other_path)}, " + f"but semantic content matches; {tile_difference(content, other_content, False)}", + ) + return - if not ensure_semantic_tiles(other_path, other_content, [tile], semantic_decoder, failure_message): - return + progress(path, f"decoding semantic tiles from {archive_label(path)}") + if not ensure_semantic_tiles(path, content, differing_tiles, semantic_decoder, failure_message): + return + progress(path, f"decoding semantic tiles from {archive_label(other_path)}") + if not ensure_semantic_tiles(other_path, other_content, differing_tiles, semantic_decoder, failure_message): + return + + progress( + path, + f"checking {len(differing_tiles)} semantic tiles against {relation} {archive_label(other_path)}", + ) + started = time.monotonic() + progress_eta(path, "checked semantic tiles", 0, len(differing_tiles), started) + for index, tile in enumerate(differing_tiles, 1): if content.semantic_tiles.get(tile) != other_content.semantic_tiles.get(tile): error( path, @@ -714,6 +930,14 @@ def compare_content(path, content, other_path, other_content, relation, semantic f"{tile_difference(content, other_content, True)}", ) return + if index % PROGRESS_INTERVAL == 0 or index == len(differing_tiles): + progress_eta(path, "checked semantic tiles", index, len(differing_tiles), started) + + progress( + path, + f"checked {len(differing_tiles)}/{len(differing_tiles)} semantic tiles " + f"against {relation} {archive_label(other_path)}", + ) notice( path, @@ -722,7 +946,7 @@ def compare_content(path, content, other_path, other_content, relation, semantic ) -def check_repeat(contents, suffix, semantic_decoder, failure_message): +def check_repeat(contents, suffix, semantic_decoder, failure_message, semantic_comparator=None): for path, content in sorted(contents.items()): if path.name.endswith(f"-repeat.{suffix}"): continue @@ -730,10 +954,10 @@ def check_repeat(contents, suffix, semantic_decoder, failure_message): if repeat not in contents: error(path, f"missing repeat output {archive_label(repeat)}") else: - compare_content(path, content, repeat, contents[repeat], "repeat output", semantic_decoder, failure_message) + compare_content(path, content, repeat, contents[repeat], "repeat output", semantic_decoder, failure_message, semantic_comparator) -def check_cross_runner(contents, suffix, semantic_decoder, failure_message): +def check_cross_runner(contents, suffix, semantic_decoder, failure_message, semantic_comparator=None): groups = {} for path, content in contents.items(): if path.name.endswith(f"-repeat.{suffix}"): @@ -744,7 +968,7 @@ def check_cross_runner(contents, suffix, semantic_decoder, failure_message): for name, values in sorted(groups.items()): reference_path, reference = values[0] for path, content in values[1:]: - compare_content(path, content, reference_path, reference, "output", semantic_decoder, failure_message) + compare_content(path, content, reference_path, reference, "output", semantic_decoder, failure_message, semantic_comparator) def main(): @@ -766,9 +990,9 @@ def main(): ) check_repeat(mbtiles, "mbtiles", mbtiles_decode_semantic_tiles, "MBTiles archive failed verification") - check_repeat(pmtiles, "pmtiles", pmtiles_decode_semantic_tiles, "PMTiles archive failed verification") + check_repeat(pmtiles, "pmtiles", pmtiles_decode_semantic_tiles, "PMTiles archive failed verification", compare_pmtiles_semantic_content) check_cross_runner(mbtiles, "mbtiles", mbtiles_decode_semantic_tiles, "MBTiles archive failed verification") - check_cross_runner(pmtiles, "pmtiles", pmtiles_decode_semantic_tiles, "PMTiles archive failed verification") + check_cross_runner(pmtiles, "pmtiles", pmtiles_decode_semantic_tiles, "PMTiles archive failed verification", compare_pmtiles_semantic_content) return status From 15fbad86d80467f767a42190f2b111203296f826 Mon Sep 17 00:00:00 2001 From: Symmetricity <184246+Symmetricity@users.noreply.github.com> Date: Sat, 16 May 2026 20:39:39 +0200 Subject: [PATCH 07/10] Cache Geofabrik fixture by metadata The CI fixture job downloaded the Liechtenstein PBF on every runtime-impacting run, even when Geofabrik had not republished the file. Resolve the current Geofabrik freshness metadata with a HEAD request and use a normalized hash of URL, effective URL, ETag, Last-Modified, and Content-Length as the Actions cache key. Restore the cached PBF when that metadata matches, otherwise download the current file. Validate both cached and newly downloaded fixtures against Geofabrik's published MD5 before uploading the artifact. This keeps repeat runs faster without pinning CI to a stale checked-in fixture. --- .github/workflows/ci.yml | 113 +++++++++++++++++++++++++++++++++++---- 1 file changed, 103 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 621fe131..8bde3b52 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,35 +89,128 @@ jobs: needs: changes if: ${{ needs.changes.outputs.runtime == 'true' }} outputs: - pbf-available: ${{ steps.download.outputs.pbf-available }} + pbf-available: ${{ steps.fixture.outputs.pbf-available }} steps: + - name: Resolve Geofabrik fixture metadata + id: metadata + env: + PBF_URL: https://download.geofabrik.de/europe/${{ env.AREA }}-latest.osm.pbf + run: | + python3 <<'PY' + import hashlib + import os + import sys + import urllib.request + + url = os.environ["PBF_URL"] + output_path = os.environ["GITHUB_OUTPUT"] + + try: + request = urllib.request.Request(url, method="HEAD") + with urllib.request.urlopen(request, timeout=30) as response: + headers = response.headers + effective_url = response.geturl() + etag = headers.get("ETag", "") + last_modified = headers.get("Last-Modified", "") + content_length = headers.get("Content-Length", "") + + if not etag and not last_modified: + raise RuntimeError("Geofabrik response did not include ETag or Last-Modified") + + metadata = "\n".join([ + f"url={url}", + f"effective-url={effective_url}", + f"etag={etag}", + f"last-modified={last_modified}", + f"content-length={content_length}", + ]) + cache_key = hashlib.sha256(metadata.encode("utf-8")).hexdigest() + + with open("geofabrik-fixture-metadata.txt", "w", encoding="utf-8") as metadata_file: + metadata_file.write(metadata) + metadata_file.write("\n") + + with open(output_path, "a", encoding="utf-8") as output: + output.write("metadata-available=true\n") + output.write(f"cache-key={cache_key}\n") + output.write(f"effective-url={effective_url}\n") + output.write(f"etag={etag}\n") + output.write(f"last-modified={last_modified}\n") + output.write(f"content-length={content_length}\n") + + print(metadata) + except Exception as error: + print(f"::warning::Geofabrik PBF fixture metadata could not be resolved: {error}") + with open(output_path, "a", encoding="utf-8") as output: + output.write("metadata-available=false\n") + sys.exit(0) + PY + + - name: Restore PBF fixture cache + id: pbf-cache + if: ${{ steps.metadata.outputs.metadata-available == 'true' }} + uses: actions/cache@v5 + with: + path: ${{ env.AREA }}.osm.pbf + key: geofabrik-${{ env.AREA }}-${{ steps.metadata.outputs.cache-key }} + - name: Download and verify PBF file - id: download + id: fixture + env: + PBF_URL: https://download.geofabrik.de/europe/${{ env.AREA }}-latest.osm.pbf + MD5_URL: https://download.geofabrik.de/europe/${{ env.AREA }}-latest.osm.pbf.md5 run: | - if curl -fsSL "https://download.geofabrik.de/europe/${AREA}-latest.osm.pbf" -o "${AREA}-latest.osm.pbf" && - curl -fsSL "https://download.geofabrik.de/europe/${AREA}-latest.osm.pbf.md5" -o "${AREA}-latest.osm.pbf.md5" && - md5sum -c "${AREA}-latest.osm.pbf.md5" && - mv "${AREA}-latest.osm.pbf" "${AREA}.osm.pbf"; then + set -u + + if [ "${{ steps.metadata.outputs.metadata-available }}" != "true" ]; then + echo "::warning::Geofabrik PBF fixture metadata could not be resolved; fixture-dependent jobs will be skipped." + echo "pbf-available=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if [ -s "${AREA}.osm.pbf" ]; then + echo "Using cached ${AREA}.osm.pbf" + else + echo "Downloading ${PBF_URL}" + if ! curl -fsSL "${PBF_URL}" -o "${AREA}.osm.pbf"; then + echo "::warning::Geofabrik PBF fixture could not be downloaded; fixture-dependent jobs will be skipped." + rm -f "${AREA}.osm.pbf" + echo "pbf-available=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + fi + + if curl -fsSL "${MD5_URL}" -o "${AREA}-latest.osm.pbf.md5"; then + expected_md5="$(awk '{ print $1 }' "${AREA}-latest.osm.pbf.md5")" + else + echo "::warning::Geofabrik PBF fixture MD5 file could not be downloaded; fixture-dependent jobs will be skipped." + rm -f "${AREA}.osm.pbf" + echo "pbf-available=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if printf '%s %s\n' "${expected_md5}" "${AREA}.osm.pbf" | md5sum -c -; then echo "pbf-available=true" >> "$GITHUB_OUTPUT" else - echo "::warning::Geofabrik PBF fixture could not be downloaded or verified; fixture-dependent jobs will be skipped." + echo "::warning::Geofabrik PBF fixture did not match the published MD5; fixture-dependent jobs will be skipped." + rm -f "${AREA}.osm.pbf" echo "pbf-available=false" >> "$GITHUB_OUTPUT" fi - name: Report unavailable PBF fixture - if: ${{ steps.download.outputs.pbf-available != 'true' }} + if: ${{ steps.fixture.outputs.pbf-available != 'true' }} run: | echo "::warning title=Tile generation not verified::Geofabrik PBF fixture could not be downloaded or verified. Fixture-dependent jobs were skipped." { echo "### Tile generation not verified" echo "" - echo "The Geofabrik PBF fixture could not be downloaded or verified, so jobs that generate MBTiles/PMTiles were skipped." + echo "The Geofabrik PBF fixture could not be downloaded or did not match the published MD5, so jobs that generate MBTiles/PMTiles were skipped." echo "This does not indicate a code failure, but this run did not verify tile generation." } >> "$GITHUB_STEP_SUMMARY" - name: Upload PBF fixture - if: ${{ steps.download.outputs.pbf-available == 'true' }} + if: ${{ steps.fixture.outputs.pbf-available == 'true' }} uses: actions/upload-artifact@v7 with: name: pbf-fixture From 1b23c7881e09438d9750257381cecde3e12e98f8 Mon Sep 17 00:00:00 2001 From: Symmetricity <184246+Symmetricity@users.noreply.github.com> Date: Sat, 16 May 2026 20:49:29 +0200 Subject: [PATCH 08/10] Add static CI validation The workflow can spend runner time downloading fixtures, restoring dependencies, and building C++ before catching simple profile, workflow, or script syntax errors. Add a cheap static-validation job after the changed-file gate and before fixture download, Docker, and build-heavy jobs. It validates GitHub workflows with actionlint, shell scripts with shellcheck, tracked JSON files with Python, Lua profiles with luac, and Python CI helpers with py_compile. This gives CI a faster failure path for the profile and workflow files we have been changing, without adding compiler or vcpkg work to the early validation layer. --- .github/workflows/ci.yml | 68 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 65 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8bde3b52..546e4620 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,6 +14,7 @@ permissions: env: AREA: liechtenstein + ACTIONLINT_VERSION: 1.7.12 jobs: @@ -82,11 +83,68 @@ jobs: fi } >> "$GITHUB_STEP_SUMMARY" + static-validation: + name: Static validation + runs-on: ubuntu-latest + timeout-minutes: 10 + needs: changes + if: ${{ needs.changes.outputs.runtime == 'true' }} + + steps: + - uses: actions/checkout@v6 + + - name: Install static validation tools + run: | + sudo apt-get update + sudo apt-get install -y lua5.4 shellcheck + mkdir -p "${RUNNER_TEMP}/bin" + GOBIN="${RUNNER_TEMP}/bin" go install "github.com/rhysd/actionlint/cmd/actionlint@v${ACTIONLINT_VERSION}" + + - name: Validate GitHub workflows + run: | + git ls-files -z -- '.github/workflows/*.yml' '.github/workflows/*.yaml' | + xargs -0 -r "${RUNNER_TEMP}/bin/actionlint" + + - name: Validate shell scripts + run: | + git ls-files -z -- '*.sh' | + xargs -0 -r shellcheck + + - name: Validate JSON files + run: | + python3 <<'PY' + import json + import pathlib + import subprocess + + files = subprocess.check_output([ + "git", "ls-files", "-z", "--", "*.json" + ]).decode().split("\0") + + for filename in sorted(filter(None, files)): + path = pathlib.Path(filename) + with path.open(encoding="utf-8") as handle: + json.load(handle) + print(path) + PY + + - name: Validate Lua profiles + run: | + git ls-files -z -- 'resources/*.lua' | + xargs -0 -r luac5.4 -p + + - name: Validate Python CI scripts + run: | + git ls-files -z -- '.github/scripts/*.py' | + xargs -0 -r python3 -m py_compile + download-pbf: name: Download PBF fixture runs-on: ubuntu-latest timeout-minutes: 10 - needs: changes + needs: + - changes + - static-validation if: ${{ needs.changes.outputs.runtime == 'true' }} outputs: pbf-available: ${{ steps.fixture.outputs.pbf-available }} @@ -583,7 +641,9 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 45 - needs: changes + needs: + - changes + - static-validation if: ${{ needs.changes.outputs.runtime == 'true' && github.ref != 'refs/heads/master' }} permissions: contents: read @@ -625,7 +685,9 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 45 - needs: changes + needs: + - changes + - static-validation if: ${{ needs.changes.outputs.runtime == 'true' && github.ref == 'refs/heads/master' }} permissions: contents: read From 8f6e1161f2038302083daad387b23b7f6e641dee Mon Sep 17 00:00:00 2001 From: Symmetricity <184246+Symmetricity@users.noreply.github.com> Date: Sat, 16 May 2026 20:54:32 +0200 Subject: [PATCH 09/10] Skip published Action tiles in validation The generated-tile verifier compares outputs from binaries built by this workflow. The local GitHub Action uses action.yml, which points at the published ghcr.io/systemed/tilemaker:master image instead of the PR-built binary. Exclude the Action artifact from tile validation and run that job only after docker-publish on master, where it is a smoke test for the published Action image rather than a cross-build equivalence input. --- .github/scripts/verify-generated-tiles.py | 8 ++++++-- .github/workflows/ci.yml | 7 ++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/scripts/verify-generated-tiles.py b/.github/scripts/verify-generated-tiles.py index 654bee4a..674e0999 100644 --- a/.github/scripts/verify-generated-tiles.py +++ b/.github/scripts/verify-generated-tiles.py @@ -127,6 +127,10 @@ def archive_label(path): return f"{build_label(path)} {path.name}" +def should_validate_archive(path): + return path.parent.name != "tile-outputs-github-action" + + def decompress_payload(data): for wbits in (16 + zlib.MAX_WBITS, zlib.MAX_WBITS): try: @@ -973,8 +977,8 @@ def check_cross_runner(contents, suffix, semantic_decoder, failure_message, sema def main(): root = pathlib.Path(sys.argv[1]) - mbtiles_paths = sorted(root.glob("**/*.mbtiles")) - pmtiles_paths = sorted(root.glob("**/*.pmtiles")) + mbtiles_paths = sorted(filter(should_validate_archive, root.glob("**/*.mbtiles"))) + pmtiles_paths = sorted(filter(should_validate_archive, root.glob("**/*.pmtiles"))) print("Archive SHA-256") mbtiles = fingerprint_archives( diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 546e4620..a8812931 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -538,8 +538,10 @@ jobs: name: Generate mbtiles with Github Action runs-on: ubuntu-latest timeout-minutes: 20 - needs: download-pbf - if: ${{ needs.download-pbf.outputs.pbf-available == 'true' }} + needs: + - download-pbf + - docker-publish + if: ${{ github.ref == 'refs/heads/master' && needs.download-pbf.outputs.pbf-available == 'true' }} steps: - name: Check out repository @@ -606,7 +608,6 @@ jobs: - Windows-Build - unix-build - unix-makefile-build - - Github-Action env: PMTILES_VERSION: 1.30.2 From 606e83954f18ef317196dcbd417726e8c6f65d8f Mon Sep 17 00:00:00 2001 From: Symmetricity <184246+Symmetricity@users.noreply.github.com> Date: Sun, 17 May 2026 19:59:03 +0200 Subject: [PATCH 10/10] Reuse semantic tile cache in verifier Remove the PMTiles-only pairwise worker comparator and route PMTiles comparisons through the same archive-level semantic tile cache used by MBTiles. This avoids repeatedly decoding the same archive when a verifier run compares three or more generated outputs, while keeping the same semantic comparison rules and error reporting path. --- .github/scripts/verify-generated-tiles.py | 163 +--------------------- 1 file changed, 7 insertions(+), 156 deletions(-) diff --git a/.github/scripts/verify-generated-tiles.py b/.github/scripts/verify-generated-tiles.py index 674e0999..c6b9929d 100644 --- a/.github/scripts/verify-generated-tiles.py +++ b/.github/scripts/verify-generated-tiles.py @@ -1,10 +1,8 @@ #!/usr/bin/env python3 -import concurrent.futures import gzip import hashlib import json -import os import pathlib import sqlite3 import struct @@ -17,13 +15,6 @@ COMPRESSION_NONE = 1 COMPRESSION_GZIP = 2 PROGRESS_INTERVAL = 1024 -SEMANTIC_COMPARE_CHUNK_SIZE = 128 - - -worker_left_archive = None -worker_right_archive = None -worker_left_compression = None -worker_right_compression = None status = 0 @@ -80,13 +71,6 @@ def progress_eta(path, action, current, total, started): ) -def semantic_worker_count(): - value = os.environ.get("VERIFY_TILE_WORKERS") - if value: - return max(1, int(value)) - return max(1, min(4, os.cpu_count() or 1)) - - def archive_sha256(path): digest = hashlib.sha256() with path.open("rb") as handle: @@ -648,130 +632,6 @@ def pmtiles_fingerprint(path): return content -def pmtiles_entry_index(path): - data = path.read_bytes() - header = pmtiles_header(data) - entries = [] - collect_pmtiles_entries(data, header, header["root_dir_offset"], header["root_dir_bytes"], entries) - return header["tile_compression"], { - (z, x, y): (offset, length) - for z, x, y, offset, length in entries - } - - -def initialize_pmtiles_compare_worker(left_path, left_compression, right_path, right_compression): - global worker_left_archive - global worker_right_archive - global worker_left_compression - global worker_right_compression - - worker_left_archive = open(left_path, "rb") - worker_right_archive = open(right_path, "rb") - worker_left_compression = left_compression - worker_right_compression = right_compression - - -def read_worker_payload(archive, offset, length, compression): - archive.seek(offset) - return decompress_pmtiles(archive.read(length), compression) - - -def compare_pmtiles_semantic_batch(batch): - for tile, left_offset, left_length, right_offset, right_length in batch: - left_payload = read_worker_payload( - worker_left_archive, - left_offset, - left_length, - worker_left_compression, - ) - right_payload = read_worker_payload( - worker_right_archive, - right_offset, - right_length, - worker_right_compression, - ) - left_hash, left_counts = semantic_tile_content(left_payload) - right_hash, right_counts = semantic_tile_content(right_payload) - if left_hash != right_hash: - return { - "same": False, - "tile": tile, - "message": semantic_counts_difference(tile, left_counts, right_counts), - } - return {"same": True, "count": len(batch)} - - -def compare_pmtiles_semantic_content(path, content, other_path, other_content, differing_tiles, relation, failure_message): - left_compression, left_index = pmtiles_entry_index(path) - right_compression, right_index = pmtiles_entry_index(other_path) - batches = [] - batch = [] - for tile in differing_tiles: - if tile not in left_index: - error(path, f"{tile_label(tile)} is missing from output") - return False - if tile not in right_index: - error(path, f"{tile_label(tile)} is missing from comparison output") - return False - left_offset, left_length = left_index[tile] - right_offset, right_length = right_index[tile] - batch.append((tile, left_offset, left_length, right_offset, right_length)) - if len(batch) == SEMANTIC_COMPARE_CHUNK_SIZE: - batches.append(batch) - batch = [] - if batch: - batches.append(batch) - - workers = semantic_worker_count() - progress( - path, - f"checking {len(differing_tiles)} semantic tiles with {workers} worker processes " - f"against {relation} {archive_label(other_path)}", - ) - started = time.monotonic() - checked = 0 - progress_eta(path, "checked semantic tiles", checked, len(differing_tiles), started) - - executor = concurrent.futures.ProcessPoolExecutor( - max_workers=workers, - initializer=initialize_pmtiles_compare_worker, - initargs=(str(path), left_compression, str(other_path), right_compression), - ) - futures = { - executor.submit(compare_pmtiles_semantic_batch, tile_batch): len(tile_batch) - for tile_batch in batches - } - try: - for future in concurrent.futures.as_completed(futures): - result = future.result() - checked += futures[future] - if not result["same"]: - tile = result["tile"] - error( - path, - f"semantic content differs from {relation} {archive_label(other_path)}; " - f"{result['message']}", - ) - for pending in futures: - pending.cancel() - executor.shutdown(wait=False, cancel_futures=True) - return False - if checked % PROGRESS_INTERVAL == 0 or checked == len(differing_tiles): - progress_eta(path, "checked semantic tiles", checked, len(differing_tiles), started) - except Exception as err: - error(path, f"{failure_message}: {err}") - executor.shutdown(wait=False, cancel_futures=True) - return False - - executor.shutdown() - progress( - path, - f"checked {len(differing_tiles)}/{len(differing_tiles)} semantic tiles " - f"against {relation} {archive_label(other_path)}", - ) - return True - - def pmtiles_decode_semantic_tiles(path, content, tiles): data = path.read_bytes() header = pmtiles_header(data) @@ -888,7 +748,7 @@ def ensure_semantic_tiles(path, content, tiles, semantic_decoder, failure_messag return True -def compare_content(path, content, other_path, other_content, relation, semantic_decoder, failure_message, semantic_comparator=None): +def compare_content(path, content, other_path, other_content, relation, semantic_decoder, failure_message): if content.raw_hash == other_content.raw_hash: return @@ -903,15 +763,6 @@ def compare_content(path, content, other_path, other_content, relation, semantic f"against {relation} {archive_label(other_path)}", ) - if semantic_comparator: - if semantic_comparator(path, content, other_path, other_content, differing_tiles, relation, failure_message): - notice( - path, - f"raw tile bytes differ from {relation} {archive_label(other_path)}, " - f"but semantic content matches; {tile_difference(content, other_content, False)}", - ) - return - progress(path, f"decoding semantic tiles from {archive_label(path)}") if not ensure_semantic_tiles(path, content, differing_tiles, semantic_decoder, failure_message): return @@ -950,7 +801,7 @@ def compare_content(path, content, other_path, other_content, relation, semantic ) -def check_repeat(contents, suffix, semantic_decoder, failure_message, semantic_comparator=None): +def check_repeat(contents, suffix, semantic_decoder, failure_message): for path, content in sorted(contents.items()): if path.name.endswith(f"-repeat.{suffix}"): continue @@ -958,10 +809,10 @@ def check_repeat(contents, suffix, semantic_decoder, failure_message, semantic_c if repeat not in contents: error(path, f"missing repeat output {archive_label(repeat)}") else: - compare_content(path, content, repeat, contents[repeat], "repeat output", semantic_decoder, failure_message, semantic_comparator) + compare_content(path, content, repeat, contents[repeat], "repeat output", semantic_decoder, failure_message) -def check_cross_runner(contents, suffix, semantic_decoder, failure_message, semantic_comparator=None): +def check_cross_runner(contents, suffix, semantic_decoder, failure_message): groups = {} for path, content in contents.items(): if path.name.endswith(f"-repeat.{suffix}"): @@ -972,7 +823,7 @@ def check_cross_runner(contents, suffix, semantic_decoder, failure_message, sema for name, values in sorted(groups.items()): reference_path, reference = values[0] for path, content in values[1:]: - compare_content(path, content, reference_path, reference, "output", semantic_decoder, failure_message, semantic_comparator) + compare_content(path, content, reference_path, reference, "output", semantic_decoder, failure_message) def main(): @@ -994,9 +845,9 @@ def main(): ) check_repeat(mbtiles, "mbtiles", mbtiles_decode_semantic_tiles, "MBTiles archive failed verification") - check_repeat(pmtiles, "pmtiles", pmtiles_decode_semantic_tiles, "PMTiles archive failed verification", compare_pmtiles_semantic_content) + check_repeat(pmtiles, "pmtiles", pmtiles_decode_semantic_tiles, "PMTiles archive failed verification") check_cross_runner(mbtiles, "mbtiles", mbtiles_decode_semantic_tiles, "MBTiles archive failed verification") - check_cross_runner(pmtiles, "pmtiles", pmtiles_decode_semantic_tiles, "PMTiles archive failed verification", compare_pmtiles_semantic_content) + check_cross_runner(pmtiles, "pmtiles", pmtiles_decode_semantic_tiles, "PMTiles archive failed verification") return status