diff --git a/include/pmtiles.h b/include/pmtiles.h index 7e8040c8..73a64384 100644 --- a/include/pmtiles.h +++ b/include/pmtiles.h @@ -55,8 +55,6 @@ class PMTiles { std::unordered_map tinyCache; void appendWithRLE(std::vector &entries, pmtiles::entryv3 &entry); - void appendTileEntry(uint64_t tileId, TileOffset offset, std::vector &rootEntries, std::vector &entries); - void flushEntries(std::vector &rootEntries, std::vector &entries); }; #endif //_PMTILES_H diff --git a/src/pmtiles.cpp b/src/pmtiles.cpp index c9561b59..055e4301 100644 --- a/src/pmtiles.cpp +++ b/src/pmtiles.cpp @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include "pmtiles.h" #include "helpers.h" @@ -25,24 +27,49 @@ void PMTiles::open(std::string &filename) { void PMTiles::close(std::string &metadata) { std::cout << "\nClosing pmtiles file" << std::flush; - // add all tiles to directories, writing leaf directories as we go + // Add all tiles to directories: low zooms to the root, the rest to leaf directories of up to + // LEAF_DIRECTORY_SIZE entries. The leaves are serialized and compressed on all cores (one after + // the other that took ~9 s for the planet), then written in order, so the file is the same. std::vector rootEntries; - std::vector entries; - leafStart = outputStream.tellp(); - if (isSparse) { - for (auto it : sparseIndex) { - appendTileEntry(it.first, it.second, rootEntries, entries); + std::vector> leaves(1); + auto append = [&](uint64_t tileId, TileOffset offset) { + pmtiles::entryv3 entry = pmtiles::entryv3(tileId, offset.offset, offset.length, 1); // 1=RLE + if (tileId=LEAF_DIRECTORY_SIZE) leaves.emplace_back(); } + }; + if (isSparse) { + for (auto it : sparseIndex) append(it.first, it.second); } else { for (size_t tileId=0; tileId compressedLeaves(leaves.size()); + std::atomic next(0); + auto compressLeaves = [&]() { + for (size_t i = next++; i < leaves.size(); i = next++) + compressedLeaves[i] = compress_string(pmtiles::serialize_directory(leaves[i]), Z_DEFAULT_COMPRESSION, true); + }; + std::vector threads; + const size_t numThreads = std::min(leaves.size(), std::max(1u, std::thread::hardware_concurrency())); + for (size_t t = 0; t < numThreads; t++) threads.emplace_back(compressLeaves); + for (auto &t : threads) t.join(); + for (size_t i = 0; i < leaves.size(); i++) { + // write the leaf directory, and add a reference to it to the root directory + const uint64_t location = outputStream.tellp(); + outputStream.write(compressedLeaves[i].c_str(), compressedLeaves[i].size()); + rootEntries.emplace_back(pmtiles::entryv3(leaves[i][0].tile_id, location-leafStart, compressedLeaves[i].size(), 0)); + } } uint64_t leafLength = static_cast(outputStream.tellp()) - leafStart; @@ -87,18 +114,7 @@ void PMTiles::close(std::string &metadata) { } // Add an entry either to the current leaf directory, or (for lowzoom) the root directory -void PMTiles::appendTileEntry(uint64_t tileId, TileOffset offset, std::vector &rootEntries, std::vector &entries) { - pmtiles::entryv3 entry = pmtiles::entryv3(tileId, offset.offset, offset.length, 1); // 1=RLE - if (tileId=LEAF_DIRECTORY_SIZE) flushEntries(rootEntries, entries); - } -} - -// Handle run-length encoding for appendTileEntry +// Handle run-length encoding for the directory entries void PMTiles::appendWithRLE(std::vector &entries, pmtiles::entryv3 &entry) { if (entries.empty() || entries.back().offset != entry.offset || @@ -110,25 +126,6 @@ void PMTiles::appendWithRLE(std::vector &entries, pmtiles::ent entries.back().run_length++; } -// Write a leaf directory to file, and add a reference to it to the root directory -void PMTiles::flushEntries(std::vector &rootEntries, std::vector &entries) { - if (entries.size()==0) return; - uint64_t startId = entries[0].tile_id; - std::string directory = pmtiles::serialize_directory(entries); - std::string compressed = compress_string(directory, Z_DEFAULT_COMPRESSION, true); - entries.clear(); - - // write the leaf directory to disk - std::lock_guard lock(fileMutex); - uint64_t location = outputStream.tellp(); - uint64_t length = compressed.size(); - outputStream.write(compressed.c_str(), compressed.size()); - - // append reference to the root directory - pmtiles::entryv3 rootEntry = pmtiles::entryv3(startId, location-leafStart, length, 0); - rootEntries.emplace_back(rootEntry); -} - // Write a tile to file and store it in the index // - if the tile is small and has already been written, reuse that instead // - we're a bit fussy about mutexs because compress_string is expensive