From 30d5447354349e66138c503086eca72a9ec32954 Mon Sep 17 00:00:00 2001 From: dktue <2878623+dktue@users.noreply.github.com> Date: Sun, 8 Nov 2026 10:09:04 +0100 Subject: [PATCH] perf(pmtiles): compress the leaf directories on all cores PMTiles::close serialized and gzipped the leaf directories (up to 10M entries each) one after the other while building them: ~9 s for the planet on one thread. Now it first collects the entries and cuts them into leaves exactly as before, then serializes and compresses the leaves in parallel, and writes them in order. The file is byte-identical. Planet (in memory, 96 threads): "Closing pmtiles file" 9 s -> 3 s. Co-Authored-By: Claude Opus 5.5 (1M context) --- include/pmtiles.h | 2 -- src/pmtiles.cpp | 79 +++++++++++++++++++++++------------------------ 2 files changed, 38 insertions(+), 43 deletions(-) diff --git a/include/pmtiles.h b/include/pmtiles.h index 7e8040c8..73a64384 100644 --- a/include/pmtiles.h +++ b/include/pmtiles.h @@ -55,8 +55,6 @@ class PMTiles { std::unordered_map tinyCache; void appendWithRLE(std::vector &entries, pmtiles::entryv3 &entry); - void appendTileEntry(uint64_t tileId, TileOffset offset, std::vector &rootEntries, std::vector &entries); - void flushEntries(std::vector &rootEntries, std::vector &entries); }; #endif //_PMTILES_H diff --git a/src/pmtiles.cpp b/src/pmtiles.cpp index c9561b59..055e4301 100644 --- a/src/pmtiles.cpp +++ b/src/pmtiles.cpp @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include "pmtiles.h" #include "helpers.h" @@ -25,24 +27,49 @@ void PMTiles::open(std::string &filename) { void PMTiles::close(std::string &metadata) { std::cout << "\nClosing pmtiles file" << std::flush; - // add all tiles to directories, writing leaf directories as we go + // Add all tiles to directories: low zooms to the root, the rest to leaf directories of up to + // LEAF_DIRECTORY_SIZE entries. The leaves are serialized and compressed on all cores (one after + // the other that took ~9 s for the planet), then written in order, so the file is the same. std::vector rootEntries; - std::vector entries; - leafStart = outputStream.tellp(); - if (isSparse) { - for (auto it : sparseIndex) { - appendTileEntry(it.first, it.second, rootEntries, entries); + std::vector> leaves(1); + auto append = [&](uint64_t tileId, TileOffset offset) { + pmtiles::entryv3 entry = pmtiles::entryv3(tileId, offset.offset, offset.length, 1); // 1=RLE + if (tileId=LEAF_DIRECTORY_SIZE) leaves.emplace_back(); } + }; + if (isSparse) { + for (auto it : sparseIndex) append(it.first, it.second); } else { for (size_t tileId=0; tileId compressedLeaves(leaves.size()); + std::atomic next(0); + auto compressLeaves = [&]() { + for (size_t i = next++; i < leaves.size(); i = next++) + compressedLeaves[i] = compress_string(pmtiles::serialize_directory(leaves[i]), Z_DEFAULT_COMPRESSION, true); + }; + std::vector threads; + const size_t numThreads = std::min(leaves.size(), std::max(1u, std::thread::hardware_concurrency())); + for (size_t t = 0; t < numThreads; t++) threads.emplace_back(compressLeaves); + for (auto &t : threads) t.join(); + for (size_t i = 0; i < leaves.size(); i++) { + // write the leaf directory, and add a reference to it to the root directory + const uint64_t location = outputStream.tellp(); + outputStream.write(compressedLeaves[i].c_str(), compressedLeaves[i].size()); + rootEntries.emplace_back(pmtiles::entryv3(leaves[i][0].tile_id, location-leafStart, compressedLeaves[i].size(), 0)); + } } uint64_t leafLength = static_cast(outputStream.tellp()) - leafStart; @@ -87,18 +114,7 @@ void PMTiles::close(std::string &metadata) { } // Add an entry either to the current leaf directory, or (for lowzoom) the root directory -void PMTiles::appendTileEntry(uint64_t tileId, TileOffset offset, std::vector &rootEntries, std::vector &entries) { - pmtiles::entryv3 entry = pmtiles::entryv3(tileId, offset.offset, offset.length, 1); // 1=RLE - if (tileId=LEAF_DIRECTORY_SIZE) flushEntries(rootEntries, entries); - } -} - -// Handle run-length encoding for appendTileEntry +// Handle run-length encoding for the directory entries void PMTiles::appendWithRLE(std::vector &entries, pmtiles::entryv3 &entry) { if (entries.empty() || entries.back().offset != entry.offset || @@ -110,25 +126,6 @@ void PMTiles::appendWithRLE(std::vector &entries, pmtiles::ent entries.back().run_length++; } -// Write a leaf directory to file, and add a reference to it to the root directory -void PMTiles::flushEntries(std::vector &rootEntries, std::vector &entries) { - if (entries.size()==0) return; - uint64_t startId = entries[0].tile_id; - std::string directory = pmtiles::serialize_directory(entries); - std::string compressed = compress_string(directory, Z_DEFAULT_COMPRESSION, true); - entries.clear(); - - // write the leaf directory to disk - std::lock_guard lock(fileMutex); - uint64_t location = outputStream.tellp(); - uint64_t length = compressed.size(); - outputStream.write(compressed.c_str(), compressed.size()); - - // append reference to the root directory - pmtiles::entryv3 rootEntry = pmtiles::entryv3(startId, location-leafStart, length, 0); - rootEntries.emplace_back(rootEntry); -} - // Write a tile to file and store it in the index // - if the tile is small and has already been written, reuse that instead // - we're a bit fussy about mutexs because compress_string is expensive