diff --git a/.github/scripts/verify-generated-tiles.py b/.github/scripts/verify-generated-tiles.py new file mode 100644 index 00000000..c6b9929d --- /dev/null +++ b/.github/scripts/verify-generated-tiles.py @@ -0,0 +1,855 @@ +#!/usr/bin/env python3 + +import gzip +import hashlib +import json +import pathlib +import sqlite3 +import struct +import subprocess +import sys +import time +import zlib + + +COMPRESSION_NONE = 1 +COMPRESSION_GZIP = 2 +PROGRESS_INTERVAL = 1024 + + +status = 0 + + +class ArchiveContent: + def __init__(self, raw_hash, raw_tiles, metadata_hash): + self.raw_hash = raw_hash + self.raw_tiles = raw_tiles + self.metadata_hash = metadata_hash + self.semantic_tiles = {} + self.layer_counts = {} + + +def error(path, message): + global status + label = archive_label(path) + print(f"::error title={label}::{label}: {message}") + status = 1 + + +def notice(path, message): + label = archive_label(path) + print(f"::notice title={label}::{label}: {message}") + + +def progress(path, message): + print(f"{archive_label(path)}: {message}", flush=True) + + +def format_duration(seconds): + seconds = int(seconds) + minutes, seconds = divmod(seconds, 60) + hours, minutes = divmod(minutes, 60) + if hours: + return f"{hours}h{minutes:02d}m{seconds:02d}s" + if minutes: + return f"{minutes}m{seconds:02d}s" + return f"{seconds}s" + + +def progress_eta(path, action, current, total, started): + elapsed = time.monotonic() - started + percent = (current / total) * 100 if total else 100.0 + if current: + remaining = (elapsed / current) * (total - current) + eta = format_duration(remaining) + else: + eta = "unknown" + progress( + path, + f"{action}: {current}/{total} ({percent:.1f}%), " + f"elapsed {format_duration(elapsed)}, ETA {eta}", + ) + + +def archive_sha256(path): + digest = hashlib.sha256() + with path.open("rb") as handle: + while True: + chunk = handle.read(1024 * 1024) + if not chunk: + break + digest.update(chunk) + return digest.hexdigest() + + +def build_label(path): + artifact = path.parent.name + if artifact == "tile-outputs-github-action": + return "GitHub Action" + if not artifact.startswith("tile-outputs-"): + return str(path.parent) + + label = artifact.removeprefix("tile-outputs-") + if label.endswith("-cmake"): + return f"{runner_label(label.removesuffix('-cmake'))} (CMake)" + if label.endswith("-makefile"): + return f"{runner_label(label.removesuffix('-makefile'))} (Makefile)" + return artifact + + +def runner_label(label): + if label == "windows": + return "Windows" + if label.startswith("ubuntu-"): + return "Ubuntu " + label.removeprefix("ubuntu-") + if label.startswith("macos-"): + return "macOS " + label.removeprefix("macos-") + return label.replace("-", " ") + + +def archive_label(path): + return f"{build_label(path)} {path.name}" + + +def should_validate_archive(path): + return path.parent.name != "tile-outputs-github-action" + + +def decompress_payload(data): + for wbits in (16 + zlib.MAX_WBITS, zlib.MAX_WBITS): + try: + return zlib.decompress(data, wbits) + except zlib.error: + pass + return data + + +def decompress_pmtiles(data, compression): + if compression == COMPRESSION_NONE: + return data + if compression == COMPRESSION_GZIP: + return gzip.decompress(data) + raise RuntimeError(f"unsupported PMTiles compression {compression}") + + +def tileid_to_zxy(tileid): + acc = 0 + for z in range(32): + num_tiles = (1 << z) * (1 << z) + if acc + num_tiles > tileid: + return t_on_level(z, tileid - acc) + acc += num_tiles + raise RuntimeError("tile zoom exceeds 64-bit limit") + + +def t_on_level(z, pos): + n = 1 << z + t = pos + x = 0 + y = 0 + s = 1 + while s < n: + rx = 1 & (t // 2) + ry = 1 & (t ^ rx) + if ry == 0: + if rx == 1: + x = s - 1 - x + y = s - 1 - y + x, y = y, x + x += s * rx + y += s * ry + t //= 4 + s *= 2 + return z, x, y + + +def decode_varint(data, pos): + value = 0 + shift = 0 + for _ in range(10): + if pos >= len(data): + raise RuntimeError("end of buffer while reading varint") + byte = data[pos] + pos += 1 + value |= (byte & 0x7f) << shift + if byte < 0x80: + return value, pos + shift += 7 + raise RuntimeError("varint too long") + + +def deserialize_directory(data): + pos = 0 + count, pos = decode_varint(data, pos) + entries = [{"tile_id": 0, "run_length": 0, "length": 0, "offset": 0} for _ in range(count)] + last_id = 0 + for entry in entries: + value, pos = decode_varint(data, pos) + entry["tile_id"] = last_id + value + last_id = entry["tile_id"] + for entry in entries: + entry["run_length"], pos = decode_varint(data, pos) + for entry in entries: + entry["length"], pos = decode_varint(data, pos) + for index, entry in enumerate(entries): + value, pos = decode_varint(data, pos) + if index > 0 and value == 0: + prev = entries[index - 1] + entry["offset"] = prev["offset"] + prev["length"] + else: + entry["offset"] = value - 1 + if pos != len(data): + raise RuntimeError("trailing bytes in PMTiles directory") + return entries + + +def zigzag_decode(value): + return (value >> 1) ^ -(value & 1) + + +def protobuf_fields(data): + pos = 0 + while pos < len(data): + tag, pos = decode_varint(data, pos) + field = tag >> 3 + wire_type = tag & 7 + if wire_type == 0: + value, pos = decode_varint(data, pos) + yield field, wire_type, value + elif wire_type == 1: + if pos + 8 > len(data): + raise RuntimeError("end of buffer while reading fixed64") + yield field, wire_type, data[pos:pos + 8] + pos += 8 + elif wire_type == 2: + length, pos = decode_varint(data, pos) + if pos + length > len(data): + raise RuntimeError("end of buffer while reading length-delimited field") + yield field, wire_type, data[pos:pos + length] + pos += length + elif wire_type == 5: + if pos + 4 > len(data): + raise RuntimeError("end of buffer while reading fixed32") + yield field, wire_type, data[pos:pos + 4] + pos += 4 + else: + raise RuntimeError(f"unsupported protobuf wire type {wire_type}") + + +def packed_varints(data): + pos = 0 + values = [] + while pos < len(data): + value, pos = decode_varint(data, pos) + values.append(value) + return values + + +def decode_geometry(geometry): + pos = 0 + x = 0 + y = 0 + paths = [] + current = [] + + while pos < len(geometry): + command = geometry[pos] & 7 + count = geometry[pos] >> 3 + pos += 1 + + if command == 1: + for _ in range(count): + if current: + paths.append(current) + current = [] + x += zigzag_decode(geometry[pos]) + y += zigzag_decode(geometry[pos + 1]) + pos += 2 + current.append([x, y]) + elif command == 2: + for _ in range(count): + if not current: + raise RuntimeError("MVT LineTo command without an active path") + x += zigzag_decode(geometry[pos]) + y += zigzag_decode(geometry[pos + 1]) + pos += 2 + current.append([x, y]) + elif command == 7: + for _ in range(count): + if current: + paths.append(current) + current = [] + else: + raise RuntimeError(f"unsupported MVT geometry command {command}") + + if current: + paths.append(current) + return paths + + +def canonical_geometry(geometry_type, geometry): + paths = decode_geometry(geometry) + if geometry_type == 1: + points = [point for path in paths for point in path] + points.sort(key=canonical_json) + return ["points", points] + if geometry_type == 3: + return ["polygons", canonical_polygons(paths)] + return ["geometry", paths] + + +def canonical_polygons(paths): + polygons = [] + current = None + outer_area = 0 + + for path in paths: + area = ring_area(path) + ring = canonical_ring(path, area) + + if area == 0: + polygons.append(["degenerate", ring]) + elif outer_area == 0 or area == outer_area: + if current: + current[1].sort(key=canonical_json) + polygons.append(current) + outer_area = area + current = [ring, []] + elif current: + current[1].append(ring) + else: + polygons.append(["orphan", ring]) + + if current: + current[1].sort(key=canonical_json) + polygons.append(current) + + polygons.sort(key=canonical_json) + return polygons + + +def canonical_ring(path, area): + ring = path + if len(ring) > 1 and ring[0] == ring[-1]: + ring = ring[:-1] + if not ring: + return [0, []] + + rotations = [ring[index:] + ring[:index] for index in range(len(ring))] + return [area, min(rotations, key=canonical_json)] + + +def ring_area(ring): + area = 0 + for index, point in enumerate(ring): + next_point = ring[(index + 1) % len(ring)] + area += point[0] * next_point[1] - next_point[0] * point[1] + if area < 0: + return -1 + if area > 0: + return 1 + return 0 + + +def decode_mvt_value(data): + values = [] + for field, wire_type, value in protobuf_fields(data): + if field == 1: + values.append(["string", value.decode("utf-8", "replace")]) + elif field == 2: + values.append(["float", struct.unpack("= len(keys) or value_index >= len(values): + raise RuntimeError("MVT feature tag index is out of range") + tags.append([keys[key_index], values[value_index]]) + elif field == 3: + geometry_type = value + elif field == 4: + geometry = packed_varints(value) + tags.sort(key=canonical_json) + geometry = canonical_geometry(geometry_type, geometry) + return [feature_id, geometry_type, tags, geometry] + + +def decode_mvt_layer(data): + name = None + features = [] + keys = [] + values = [] + extent = None + version = None + for field, wire_type, value in protobuf_fields(data): + if field == 1: + name = value.decode("utf-8", "replace") + elif field == 2: + features.append(value) + elif field == 3: + keys.append(value.decode("utf-8", "replace")) + elif field == 4: + values.append(decode_mvt_value(value)) + elif field == 5: + extent = value + elif field == 15: + version = value + + decoded_features = [decode_mvt_feature(feature, keys, values) for feature in features] + decoded_features.sort(key=canonical_json) + return [name, version, extent, decoded_features] + + +def canonical_mvt(data): + layers = [] + for field, wire_type, value in protobuf_fields(data): + if field == 3: + layers.append(decode_mvt_layer(value)) + layers.sort(key=canonical_json) + return layers + + +def canonical_json(value): + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def semantic_tile_content(data): + tile = canonical_mvt(data) + digest = hashlib.sha256(canonical_json(tile).encode("utf-8", "surrogateescape")).hexdigest() + layer_counts = {layer[0]: len(layer[3]) for layer in tile} + return digest, layer_counts + + +def content_summary(content): + return f"raw {content.raw_hash}" + + +def add_tile_to_raw_digest(raw_digest, raw_tiles, z, x, y, payload): + tile = (z, x, y) + raw_hash = hashlib.sha256(payload).hexdigest() + raw_tiles[tile] = raw_hash + + raw_digest.update(f"T\t{z}\t{x}\t{y}\t{len(payload)}\t{raw_hash}\n".encode()) + + +def add_tile_to_semantic_maps(content, z, x, y, payload): + tile = (z, x, y) + semantic_hash, counts = semantic_tile_content(payload) + content.semantic_tiles[tile] = semantic_hash + content.layer_counts[tile] = counts + + +def set_tile_semantic_maps(content, z, x, y, semantic_hash, counts): + tile = (z, x, y) + content.semantic_tiles[tile] = semantic_hash + content.layer_counts[tile] = counts + + +def mbtiles_fingerprint(path): + con = sqlite3.connect(f"file:{path}?mode=ro", uri=True) + cur = con.cursor() + tables = {row[0] for row in cur.execute("select name from sqlite_master where type = 'table'")} + missing = {"metadata", "tiles"} - tables + if missing: + raise RuntimeError("missing tables: " + ", ".join(sorted(missing))) + tile_count = cur.execute("select count(*) from tiles").fetchone()[0] + if tile_count == 0: + raise RuntimeError("tiles table is empty") + empty_tiles = cur.execute("select count(*) from tiles where tile_data is null or length(tile_data) = 0").fetchone()[0] + if empty_tiles: + raise RuntimeError(f"{empty_tiles} empty tile blobs") + minzoom, maxzoom = cur.execute("select min(zoom_level), max(zoom_level) from tiles").fetchone() + metadata_rows = list(cur.execute("select name, value from metadata order by name, value")) + if not metadata_rows: + raise RuntimeError("metadata table is empty") + + raw_digest = hashlib.sha256() + metadata_digest = hashlib.sha256() + for name, value in metadata_rows: + metadata = f"M\t{name}\t{canonical_metadata_value(value)}\n".encode("utf-8", "surrogateescape") + raw_digest.update(metadata) + metadata_digest.update(metadata) + + raw_tiles = {} + for z, x, y, tile_data in cur.execute("select zoom_level, tile_column, tile_row, tile_data from tiles order by zoom_level, tile_column, tile_row"): + payload = decompress_payload(bytes(tile_data)) + add_tile_to_raw_digest(raw_digest, raw_tiles, z, x, y, payload) + con.close() + + content = ArchiveContent(raw_digest.hexdigest(), raw_tiles, metadata_digest.hexdigest()) + print( + f"{archive_label(path)}: {tile_count} tiles, zoom {minzoom}-{maxzoom}, " + f"{len(metadata_rows)} metadata rows, raw {content.raw_hash}" + ) + return content + + +def mbtiles_decode_semantic_tiles(path, content, tiles): + con = sqlite3.connect(f"file:{path}?mode=ro", uri=True) + cur = con.cursor() + for z, x, y in sorted(tiles): + row = cur.execute( + "select tile_data from tiles where zoom_level=? and tile_column=? and tile_row=?", + (z, x, y), + ).fetchone() + if row is None: + continue + tile_data = row[0] + payload = decompress_payload(bytes(tile_data)) + add_tile_to_semantic_maps(content, z, x, y, payload) + con.close() + + +def canonical_metadata_value(value): + if value is None: + return "" + try: + return json.dumps(json.loads(value), sort_keys=True, separators=(",", ":")) + except (TypeError, json.JSONDecodeError): + return str(value) + + +def verify_pmtiles(path): + print(f"{archive_label(path)}: verifying PMTiles archive") + result = subprocess.run( + ["pmtiles", "verify", str(path)], + capture_output=True, + text=True, + check=False, + ) + if result.stdout: + print(result.stdout, end="") + if result.stderr: + print(result.stderr, end="") + if result.returncode != 0: + raise RuntimeError(pmtiles_failure_message(result)) + + +def pmtiles_failure_message(result): + lines = [ + line.strip() + for line in (result.stdout + "\n" + result.stderr).splitlines() + if line.strip() + ] + for line in reversed(lines): + pos = line.find("Failed to verify archive") + if pos >= 0: + return line[pos:] + if lines: + return lines[-1] + return "pmtiles verify failed" + + +def pmtiles_header(data): + if len(data) < 127: + raise RuntimeError("PMTiles header is truncated") + if data[:7] != b"PMTiles" or data[7] != 3: + raise RuntimeError("not a PMTiles v3 archive") + values = struct.unpack_from("> "$GITHUB_OUTPUT" + { + echo "### Changed file check" + echo "" + echo "Unable to determine the changed files, so the full CI matrix will run." + } >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + runtime=false + changed_count=0 + while IFS= read -r -d '' path; do + changed_count=$((changed_count + 1)) + case "$path" in + .github/workflows/ci.yml) + runtime=true + ;; + CMakeLists.txt|Makefile|Dockerfile|action.yml) + runtime=true + ;; + get-coastline.sh|get-landcover.sh|get-monaco.sh) + runtime=true + ;; + cmake/*|include/*|resources/*|server/*|src/*|test/*) + runtime=true + ;; + esac + done < <(git diff --name-only -z "${BASE_SHA}" "${HEAD_SHA}") + + echo "runtime=${runtime}" >> "$GITHUB_OUTPUT" + { + echo "### Changed file check" + echo "" + echo "Changed files: ${changed_count}" + echo "Runtime-impacting changes: ${runtime}" + if [[ "${runtime}" != "true" ]]; then + echo "" + echo "Only non-runtime files changed, so build and tile-generation jobs were skipped." + fi + } >> "$GITHUB_STEP_SUMMARY" + + static-validation: + name: Static validation + runs-on: ubuntu-latest + timeout-minutes: 10 + needs: changes + if: ${{ needs.changes.outputs.runtime == 'true' }} + + steps: + - uses: actions/checkout@v6 + + - name: Install static validation tools + run: | + sudo apt-get update + sudo apt-get install -y lua5.4 shellcheck + mkdir -p "${RUNNER_TEMP}/bin" + GOBIN="${RUNNER_TEMP}/bin" go install "github.com/rhysd/actionlint/cmd/actionlint@v${ACTIONLINT_VERSION}" + + - name: Validate GitHub workflows + run: | + git ls-files -z -- '.github/workflows/*.yml' '.github/workflows/*.yaml' | + xargs -0 -r "${RUNNER_TEMP}/bin/actionlint" + + - name: Validate shell scripts + run: | + git ls-files -z -- '*.sh' | + xargs -0 -r shellcheck + + - name: Validate JSON files + run: | + python3 <<'PY' + import json + import pathlib + import subprocess + + files = subprocess.check_output([ + "git", "ls-files", "-z", "--", "*.json" + ]).decode().split("\0") + + for filename in sorted(filter(None, files)): + path = pathlib.Path(filename) + with path.open(encoding="utf-8") as handle: + json.load(handle) + print(path) + PY + + - name: Validate Lua profiles + run: | + git ls-files -z -- 'resources/*.lua' | + xargs -0 -r luac5.4 -p + + - name: Validate Python CI scripts + run: | + git ls-files -z -- '.github/scripts/*.py' | + xargs -0 -r python3 -m py_compile + + download-pbf: + name: Download PBF fixture + runs-on: ubuntu-latest + timeout-minutes: 10 + needs: + - changes + - static-validation + if: ${{ needs.changes.outputs.runtime == 'true' }} + outputs: + pbf-available: ${{ steps.fixture.outputs.pbf-available }} + + steps: + - name: Resolve Geofabrik fixture metadata + id: metadata + env: + PBF_URL: https://download.geofabrik.de/europe/${{ env.AREA }}-latest.osm.pbf + run: | + python3 <<'PY' + import hashlib + import os + import sys + import urllib.request + + url = os.environ["PBF_URL"] + output_path = os.environ["GITHUB_OUTPUT"] + + try: + request = urllib.request.Request(url, method="HEAD") + with urllib.request.urlopen(request, timeout=30) as response: + headers = response.headers + effective_url = response.geturl() + etag = headers.get("ETag", "") + last_modified = headers.get("Last-Modified", "") + content_length = headers.get("Content-Length", "") + + if not etag and not last_modified: + raise RuntimeError("Geofabrik response did not include ETag or Last-Modified") + + metadata = "\n".join([ + f"url={url}", + f"effective-url={effective_url}", + f"etag={etag}", + f"last-modified={last_modified}", + f"content-length={content_length}", + ]) + cache_key = hashlib.sha256(metadata.encode("utf-8")).hexdigest() + + with open("geofabrik-fixture-metadata.txt", "w", encoding="utf-8") as metadata_file: + metadata_file.write(metadata) + metadata_file.write("\n") + + with open(output_path, "a", encoding="utf-8") as output: + output.write("metadata-available=true\n") + output.write(f"cache-key={cache_key}\n") + output.write(f"effective-url={effective_url}\n") + output.write(f"etag={etag}\n") + output.write(f"last-modified={last_modified}\n") + output.write(f"content-length={content_length}\n") + + print(metadata) + except Exception as error: + print(f"::warning::Geofabrik PBF fixture metadata could not be resolved: {error}") + with open(output_path, "a", encoding="utf-8") as output: + output.write("metadata-available=false\n") + sys.exit(0) + PY + + - name: Restore PBF fixture cache + id: pbf-cache + if: ${{ steps.metadata.outputs.metadata-available == 'true' }} + uses: actions/cache@v5 + with: + path: ${{ env.AREA }}.osm.pbf + key: geofabrik-${{ env.AREA }}-${{ steps.metadata.outputs.cache-key }} + + - name: Download and verify PBF file + id: fixture + env: + PBF_URL: https://download.geofabrik.de/europe/${{ env.AREA }}-latest.osm.pbf + MD5_URL: https://download.geofabrik.de/europe/${{ env.AREA }}-latest.osm.pbf.md5 + run: | + set -u + + if [ "${{ steps.metadata.outputs.metadata-available }}" != "true" ]; then + echo "::warning::Geofabrik PBF fixture metadata could not be resolved; fixture-dependent jobs will be skipped." + echo "pbf-available=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if [ -s "${AREA}.osm.pbf" ]; then + echo "Using cached ${AREA}.osm.pbf" + else + echo "Downloading ${PBF_URL}" + if ! curl -fsSL "${PBF_URL}" -o "${AREA}.osm.pbf"; then + echo "::warning::Geofabrik PBF fixture could not be downloaded; fixture-dependent jobs will be skipped." + rm -f "${AREA}.osm.pbf" + echo "pbf-available=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + fi + + if curl -fsSL "${MD5_URL}" -o "${AREA}-latest.osm.pbf.md5"; then + expected_md5="$(awk '{ print $1 }' "${AREA}-latest.osm.pbf.md5")" + else + echo "::warning::Geofabrik PBF fixture MD5 file could not be downloaded; fixture-dependent jobs will be skipped." + rm -f "${AREA}.osm.pbf" + echo "pbf-available=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if printf '%s %s\n' "${expected_md5}" "${AREA}.osm.pbf" | md5sum -c -; then + echo "pbf-available=true" >> "$GITHUB_OUTPUT" + else + echo "::warning::Geofabrik PBF fixture did not match the published MD5; fixture-dependent jobs will be skipped." + rm -f "${AREA}.osm.pbf" + echo "pbf-available=false" >> "$GITHUB_OUTPUT" + fi + + - name: Report unavailable PBF fixture + if: ${{ steps.fixture.outputs.pbf-available != 'true' }} + run: | + echo "::warning title=Tile generation not verified::Geofabrik PBF fixture could not be downloaded or verified. Fixture-dependent jobs were skipped." + { + echo "### Tile generation not verified" + echo "" + echo "The Geofabrik PBF fixture could not be downloaded or did not match the published MD5, so jobs that generate MBTiles/PMTiles were skipped." + echo "This does not indicate a code failure, but this run did not verify tile generation." + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload PBF fixture + if: ${{ steps.fixture.outputs.pbf-available == 'true' }} + uses: actions/upload-artifact@v7 + with: + name: pbf-fixture + retention-days: 1 + path: ${{ env.AREA }}.osm.pbf + Windows-Build: name: Windows (CMake) runs-on: windows-latest + timeout-minutes: 120 + needs: download-pbf + if: ${{ needs.download-pbf.outputs.pbf-available == 'true' }} + env: + VCPKG_BINARY_SOURCES: clear;files,${{ github.workspace }}\vcpkg-binary-cache,readwrite steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - - name: Enable vcpkg cache - uses: actions/cache@v4 + - name: Download PBF fixture + uses: actions/download-artifact@v8 with: - path: c:\vcpkg\installed - key: windows-vcpkg-x64-0 # Increase the number whenever dependencies are modified - restore-keys: windows-vcpkg-x64 + name: pbf-fixture + + - name: Detect runner image + id: runner-image + shell: pwsh + run: | + $imageOS = if ($env:ImageOS) { $env:ImageOS } else { "unknown" } + $imageVersion = if ($env:ImageVersion) { $env:ImageVersion } else { "unknown" } + "image-os=$imageOS" >> $env:GITHUB_OUTPUT + "image-version=$imageVersion" >> $env:GITHUB_OUTPUT + + - name: Enable vcpkg binary cache + uses: actions/cache@v5 + with: + path: ${{ github.workspace }}\vcpkg-binary-cache + key: vcpkg-binary-${{ runner.os }}-${{ steps.runner-image.outputs.image-os }}-${{ steps.runner-image.outputs.image-version }}-x64-windows-static-md-${{ hashFiles('.github/workflows/ci.yml') }} + restore-keys: vcpkg-binary-${{ runner.os }}-${{ steps.runner-image.outputs.image-os }}-${{ steps.runner-image.outputs.image-version }}-x64-windows-static-md- - name: Build dependencies run: | @@ -30,20 +314,50 @@ jobs: - name: Build tilemaker run: | - mkdir ${{ github.workspace }}\build - cd ${{ github.workspace }}\build && cmake -DTILEMAKER_BUILD_STATIC=ON -DVCPKG_TARGET_TRIPLET="x64-windows-static-md" -DCMAKE_TOOLCHAIN_FILE="c:\vcpkg\scripts\buildsystems\vcpkg.cmake" .. + mkdir ${{ github.workspace }}\build + cd ${{ github.workspace }}\build && cmake -DTILEMAKER_BUILD_STATIC=ON -DVCPKG_TARGET_TRIPLET="x64-windows-static-md" -DCMAKE_TOOLCHAIN_FILE="${env:VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake" .. cd ${{ github.workspace }}\build && cmake --build . --config RelWithDebInfo - name: Build openmaptiles-compatible mbtiles files of Liechtenstein run: | - Invoke-WebRequest -Uri https://download.geofabrik.de/europe/${{ env.AREA }}-latest.osm.pbf -OutFile ${{ env.AREA }}.osm.pbf -MaximumRedirection 5 - ${{ github.workspace }}\build\RelWithDebInfo\tilemaker.exe ${{ env.AREA }}.osm.pbf --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output=${{ env.AREA }}.pmtiles --verbose - ${{ github.workspace }}\build\RelWithDebInfo\tilemaker.exe ${{ env.AREA }}.osm.pbf --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output=${{ env.AREA }}.mbtiles --store osm_store --verbose + $tilemaker = "${{ github.workspace }}\build\RelWithDebInfo\tilemaker.exe" + function Invoke-Tilemaker { + param([string] $Output, [string[]] $Arguments) + Write-Host "::group::Build $Output" + & $tilemaker @Arguments + $status = $LASTEXITCODE + Write-Host "::endgroup::" + if ($status -ne 0) { + Write-Error "tilemaker failed while writing $Output with exit code $status" + exit $status + } + if (!(Test-Path -LiteralPath $Output) -or (Get-Item -LiteralPath $Output).Length -eq 0) { + Write-Error "tilemaker did not create $Output" + exit 1 + } + } + + Invoke-Tilemaker "${{ env.AREA }}.pmtiles" @("${{ env.AREA }}.osm.pbf", "--config=resources/config-openmaptiles.json", "--process=resources/process-openmaptiles.lua", "--output=${{ env.AREA }}.pmtiles", "--verbose") + Invoke-Tilemaker "${{ env.AREA }}-repeat.pmtiles" @("${{ env.AREA }}.osm.pbf", "--config=resources/config-openmaptiles.json", "--process=resources/process-openmaptiles.lua", "--output=${{ env.AREA }}-repeat.pmtiles", "--verbose") + Invoke-Tilemaker "${{ env.AREA }}.mbtiles" @("${{ env.AREA }}.osm.pbf", "--config=resources/config-openmaptiles.json", "--process=resources/process-openmaptiles.lua", "--output=${{ env.AREA }}.mbtiles", "--store", "${{ runner.temp }}\osm_store", "--verbose") + Invoke-Tilemaker "${{ env.AREA }}-repeat.mbtiles" @("${{ env.AREA }}.osm.pbf", "--config=resources/config-openmaptiles.json", "--process=resources/process-openmaptiles.lua", "--output=${{ env.AREA }}-repeat.mbtiles", "--store", "${{ runner.temp }}\osm_store_repeat", "--verbose") + + - name: Upload generated tiles + uses: actions/upload-artifact@v7 + with: + name: tile-outputs-windows-cmake + retention-days: 14 + path: | + ${{ env.AREA }}.mbtiles + ${{ env.AREA }}.pmtiles + ${{ env.AREA }}-repeat.mbtiles + ${{ env.AREA }}-repeat.pmtiles - name: 'Upload compiled executable' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: tilemaker-windows + retention-days: 14 path: | ${{ github.workspace }}\resources ${{ github.workspace }}\build\RelWithDebInfo\tilemaker.exe @@ -51,54 +365,102 @@ jobs: unix-build: strategy: + fail-fast: false matrix: include: - os: ubuntu-22.04 triplet: x64-linux executable: tilemaker - path: /usr/local/share/vcpkg/installed - toolchain: /usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake name: ${{ matrix.os }} (CMake) runs-on: ${{ matrix.os }} + timeout-minutes: 90 + needs: download-pbf + if: ${{ needs.download-pbf.outputs.pbf-available == 'true' }} + env: + VCPKG_BINARY_SOURCES: clear;files,${{ github.workspace }}/vcpkg-binary-cache,readwrite steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - - name: Enable vcpkg cache - uses: actions/cache@v4 + - name: Download PBF fixture + uses: actions/download-artifact@v8 with: - path: ${{ matrix.path }} - key: vcpkg-${{ matrix.triplet }}-0 # Increase the number whenever dependencies are modified - restore-keys: vcpkg-${{ matrix.triplet }} + name: pbf-fixture + + - name: Detect runner image + id: runner-image + run: | + echo "image-os=${ImageOS:-unknown}" >> "$GITHUB_OUTPUT" + echo "image-version=${ImageVersion:-unknown}" >> "$GITHUB_OUTPUT" + + - name: Enable vcpkg binary cache + uses: actions/cache@v5 + with: + path: ${{ github.workspace }}/vcpkg-binary-cache + key: vcpkg-binary-${{ runner.os }}-${{ steps.runner-image.outputs.image-os }}-${{ steps.runner-image.outputs.image-version }}-${{ matrix.triplet }}-${{ hashFiles('.github/workflows/ci.yml') }} + restore-keys: vcpkg-binary-${{ runner.os }}-${{ steps.runner-image.outputs.image-os }}-${{ steps.runner-image.outputs.image-version }}-${{ matrix.triplet }}- - name: Build dependencies run: | - vcpkg install --triplet=${{ matrix.triplet }} lua shapelib zlib protobuf[zlib] sqlite3 boost-program-options boost-filesystem boost-geometry boost-system boost-asio boost-interprocess boost-iostreams boost-sort rapidjson + vcpkg install --triplet="${{ matrix.triplet }}" lua shapelib zlib protobuf[zlib] sqlite3 boost-program-options boost-filesystem boost-geometry boost-system boost-asio boost-interprocess boost-iostreams boost-sort rapidjson - name: Build tilemaker run: | mkdir build cd build - cmake -DTILEMAKER_BUILD_STATIC=ON -DCMAKE_BUILD_TYPE=Release -DVCPKG_TARGET_TRIPLET=${{ matrix.triplet }} -DCMAKE_TOOLCHAIN_FILE=${{ matrix.toolchain }} -DCMAKE_CXX_COMPILER=g++ .. + cmake -DTILEMAKER_BUILD_STATIC=ON -DCMAKE_BUILD_TYPE=Release -DVCPKG_TARGET_TRIPLET="${{ matrix.triplet }}" -DCMAKE_TOOLCHAIN_FILE="$VCPKG_INSTALLATION_ROOT/scripts/buildsystems/vcpkg.cmake" -DCMAKE_CXX_COMPILER=g++ .. cmake --build . strip tilemaker - name: Build openmaptiles-compatible mbtiles files of Liechtenstein run: | - curl -L https://download.geofabrik.de/europe/${{ env.AREA }}-latest.osm.pbf -o ${{ env.AREA }}.osm.pbf - ${{ github.workspace }}/build/${{ matrix.executable }} ${{ env.AREA }}.osm.pbf --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output=${{ env.AREA }}.pmtiles --verbose - ${{ github.workspace }}/build/${{ matrix.executable }} ${{ env.AREA }}.osm.pbf --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output=${{ env.AREA }}.mbtiles --verbose --store /tmp/store + run_tilemaker() { + output="$1" + shift + echo "::group::Build ${output}" + if "$@"; then + echo "::endgroup::" + else + status="$?" + echo "::endgroup::" + echo "::error title=Tile generation failed::tilemaker failed while writing ${output} with exit code ${status}" + exit "${status}" + fi + if [ ! -s "${output}" ]; then + echo "::error title=Tile output missing::tilemaker did not create ${output}" + exit 1 + fi + } + + tilemaker="${{ github.workspace }}/build/${{ matrix.executable }}" + run_tilemaker "${{ env.AREA }}.pmtiles" "${tilemaker}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}.pmtiles" --verbose + run_tilemaker "${{ env.AREA }}-repeat.pmtiles" "${tilemaker}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-repeat.pmtiles" --verbose + run_tilemaker "${{ env.AREA }}.mbtiles" "${tilemaker}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}.mbtiles" --verbose --store "${{ runner.temp }}/store" + run_tilemaker "${{ env.AREA }}-repeat.mbtiles" "${tilemaker}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-repeat.mbtiles" --verbose --store "${{ runner.temp }}/store-repeat" + + - name: Upload generated tiles + uses: actions/upload-artifact@v7 + with: + name: tile-outputs-${{ matrix.os }}-cmake + retention-days: 14 + path: | + ${{ env.AREA }}.mbtiles + ${{ env.AREA }}.pmtiles + ${{ env.AREA }}-repeat.mbtiles + ${{ env.AREA }}-repeat.pmtiles - name: 'Upload compiled executable' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: tilemaker-${{ matrix.os }} + retention-days: 14 path: | ${{ github.workspace }}/resources ${{ github.workspace }}/build/${{ matrix.executable }} unix-makefile-build: strategy: + fail-fast: false matrix: include: - os: ubuntu-22.04 @@ -107,9 +469,17 @@ jobs: name: ${{ matrix.os }} (Makefile) runs-on: ${{ matrix.os }} + timeout-minutes: 30 + needs: download-pbf + if: ${{ needs.download-pbf.outputs.pbf-available == 'true' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 + + - name: Download PBF fixture + uses: actions/download-artifact@v8 + with: + name: pbf-fixture - name: Install Linux dependencies if: ${{ matrix.os == 'ubuntu-22.04' }} @@ -129,44 +499,165 @@ jobs: - name: Build openmaptiles-compatible mbtiles files of Liechtenstein run: | - curl -L https://download.geofabrik.de/europe/${{ env.AREA }}-latest.osm.pbf -o ${{ env.AREA }}.osm.pbf - ./tilemaker ${{ env.AREA }}.osm.pbf --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output=${{ env.AREA }}.pmtiles --verbose - ./tilemaker ${{ env.AREA }}.osm.pbf --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output=${{ env.AREA }}.mbtiles --verbose --store /tmp/store + run_tilemaker() { + output="$1" + shift + echo "::group::Build ${output}" + if "$@"; then + echo "::endgroup::" + else + status="$?" + echo "::endgroup::" + echo "::error title=Tile generation failed::tilemaker failed while writing ${output} with exit code ${status}" + exit "${status}" + fi + if [ ! -s "${output}" ]; then + echo "::error title=Tile output missing::tilemaker did not create ${output}" + exit 1 + fi + } + + run_tilemaker "${{ env.AREA }}.pmtiles" ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}.pmtiles" --verbose + run_tilemaker "${{ env.AREA }}-repeat.pmtiles" ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-repeat.pmtiles" --verbose + run_tilemaker "${{ env.AREA }}.mbtiles" ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}.mbtiles" --verbose --store "${{ runner.temp }}/store" + run_tilemaker "${{ env.AREA }}-repeat.mbtiles" ./tilemaker "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-repeat.mbtiles" --verbose --store "${{ runner.temp }}/store-repeat" + + - name: Upload generated tiles + uses: actions/upload-artifact@v7 + with: + name: tile-outputs-${{ matrix.os }}-makefile + retention-days: 14 + path: | + ${{ env.AREA }}.mbtiles + ${{ env.AREA }}.pmtiles + ${{ env.AREA }}-repeat.mbtiles + ${{ env.AREA }}-repeat.pmtiles Github-Action: name: Generate mbtiles with Github Action runs-on: ubuntu-latest + timeout-minutes: 20 + needs: + - download-pbf + - docker-publish + if: ${{ github.ref == 'refs/heads/master' && needs.download-pbf.outputs.pbf-available == 'true' }} steps: - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Download PBF file - run: curl -L https://download.geofabrik.de/europe/${AREA}-latest.osm.pbf -o ${AREA}.osm.pbf + uses: actions/download-artifact@v8 + with: + name: pbf-fixture + + - name: Build openmaptiles-compatible pmtiles file + uses: ./ + with: + input: ${{ env.AREA }}.osm.pbf + output: ${{ env.AREA }}.pmtiles - - name: Build openmaptiles-compatible mbtiles files of given area + - name: Build repeat openmaptiles-compatible pmtiles file uses: ./ with: - input: ${{ env.AREA }}.osm.pbf + input: ${{ env.AREA }}.osm.pbf + output: ${{ env.AREA }}-repeat.pmtiles + + - name: Build openmaptiles-compatible mbtiles file + uses: ./ + with: + input: ${{ env.AREA }}.osm.pbf output: ${{ env.AREA }}.mbtiles + - name: Build repeat openmaptiles-compatible mbtiles file + uses: ./ + with: + input: ${{ env.AREA }}.osm.pbf + output: ${{ env.AREA }}-repeat.mbtiles + + - name: Check generated tile files + run: | + for output in \ + "${AREA}.mbtiles" \ + "${AREA}.pmtiles" \ + "${AREA}-repeat.mbtiles" \ + "${AREA}-repeat.pmtiles"; do + if [ ! -s "${output}" ]; then + echo "::error title=Tile output missing::tilemaker did not create ${output}" + exit 1 + fi + done + + - name: Upload generated tiles + uses: actions/upload-artifact@v7 + with: + name: tile-outputs-github-action + retention-days: 14 + path: | + ${{ env.AREA }}.mbtiles + ${{ env.AREA }}.pmtiles + ${{ env.AREA }}-repeat.mbtiles + ${{ env.AREA }}-repeat.pmtiles + + verify-generated-tiles: + name: Verify generated tiles + runs-on: ubuntu-latest + timeout-minutes: 20 + needs: + - Windows-Build + - unix-build + - unix-makefile-build + env: + PMTILES_VERSION: 1.30.2 + + steps: + - uses: actions/checkout@v6 + with: + sparse-checkout: .github/scripts/verify-generated-tiles.py + sparse-checkout-cone-mode: false + + - name: Download generated tiles + uses: actions/download-artifact@v8 + with: + pattern: tile-outputs-* + path: generated-tiles + + - name: Install pmtiles + run: | + curl -fsSL -o pmtiles.tar.gz "https://github.com/protomaps/go-pmtiles/releases/download/v${PMTILES_VERSION}/go-pmtiles_${PMTILES_VERSION}_Linux_x86_64.tar.gz" + tar -xzf pmtiles.tar.gz + chmod +x pmtiles + sudo mv pmtiles /usr/local/bin/pmtiles + pmtiles version + + - name: Verify generated tile archives + run: | + python3 .github/scripts/verify-generated-tiles.py generated-tiles + docker-build: env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} runs-on: ubuntu-latest + timeout-minutes: 45 + needs: + - changes + - static-validation + if: ${{ needs.changes.outputs.runtime == 'true' && github.ref != 'refs/heads/master' }} permissions: contents: read - packages: write steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 - name: Log in to the Container registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -174,23 +665,61 @@ jobs: - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@v6 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - name: Build Docker image - uses: docker/build-push-action@v6 - if: ${{ github.ref != 'refs/heads/master'}} + uses: docker/build-push-action@v7 with: context: . push: false tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + docker-publish: + env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + + runs-on: ubuntu-latest + timeout-minutes: 45 + needs: + - changes + - static-validation + if: ${{ needs.changes.outputs.runtime == 'true' && github.ref == 'refs/heads/master' }} + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Log in to the Container registry + uses: docker/login-action@v4 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v6 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + - name: Build and push Docker image - uses: docker/build-push-action@v6 - if: ${{ github.ref == 'refs/heads/master'}} + uses: docker/build-push-action@v7 with: context: . push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.gitignore b/.gitignore index 18805e51..b80da080 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,10 @@ tilemaker.dSYM/ # editor temporaries *~ +# python cache files +__pycache__/ +*.pyc + # vagrant local stuff .vagrant/