From 36c946614d64754621a62acf7008ce975c745621 Mon Sep 17 00:00:00 2001 From: dktue <2878623+dktue@users.noreply.github.com> Date: Sat, 7 Nov 2026 14:48:56 +0100 Subject: [PATCH 1/8] test: master + all open performance PRs + #981 + #942 + #983 (combined snapshot) For testing only, not for merging. One commit with the combined tree that built the planet in 14 min 03 s (in memory, 96 threads) for #977: master 2789651 + #933, #959, #966, #968, #970, #971, #972, #973, #974, #975, #976, #978, #979 (open performance PRs), #981 (deterministic tile content), #942 (deterministic POI order) and #983 (ordered .pmtiles/.mbtiles writer, 64 MB per thread). Each change is reviewed in its own PR. Co-Authored-By: Claude Opus 5.5 (1M context) --- Makefile | 27 +++++ include/append_vector.h | 4 + include/attribute_store.h | 27 ++++- include/coordinates_geom.h | 1 + include/geometry/correct.hpp | 92 +++++++++----- include/mbtiles.h | 6 + include/ordered_batches.h | 113 ++++++++++++++++++ include/osm_store.h | 15 ++- include/pmtiles.h | 20 +++- include/shp_mem_tiles.h | 3 +- include/stable_hash.h | 42 +++++++ include/tile_coordinates_set.h | 6 + include/tile_data.h | 157 +++++++++++++++++------- resources/process-debug.lua | 7 +- resources/process-openmaptiles.lua | 7 +- src/attribute_store.cpp | 56 +++++++++ src/coordinates_geom.cpp | 29 ++++- src/geom.cpp | 148 ++++++++++++++++++----- src/mbtiles.cpp | 40 ++++++- src/mmap_allocator.cpp | 23 ++++ src/osm_store.cpp | 55 ++++++++- src/output_object.cpp | 6 +- src/pmtiles.cpp | 59 ++++----- src/shp_mem_tiles.cpp | 61 ++++++++-- src/shp_processor.cpp | 20 ++-- src/tile_coordinates_set.cpp | 56 +++++++++ src/tile_data.cpp | 185 ++++++++++++++++++++++++++--- src/tile_sorting.cpp | 73 ++++++++++-- src/tilemaker.cpp | 149 ++++++++++++++--------- test/append_vector.test.cpp | 6 + test/correct.test.cpp | 72 +++++++++++ test/geom.test.cpp | 39 ++++++ test/ordered_batches.test.cpp | 89 ++++++++++++++ test/shp_mem_tiles.test.cpp | 66 ++++++++++ test/tile_coordinates_set.test.cpp | 61 ++++++++++ 35 files changed, 1566 insertions(+), 254 deletions(-) create mode 100644 include/ordered_batches.h create mode 100644 include/stable_hash.h create mode 100644 test/correct.test.cpp create mode 100644 test/ordered_batches.test.cpp create mode 100644 test/shp_mem_tiles.test.cpp diff --git a/Makefile b/Makefile index 57227d4c..b03576c2 100644 --- a/Makefile +++ b/Makefile @@ -156,13 +156,16 @@ test: \ test_append_vector \ test_attribute_store \ test_config_validator \ + test_correct \ test_deque_map \ test_geom \ test_helpers \ test_options_parser \ + test_ordered_batches \ test_pbf_reader \ test_pooled_string \ test_relation_roles \ + test_shp_mem_tiles \ test_significant_tags \ test_sorted_node_store \ test_sorted_way_store \ @@ -193,6 +196,10 @@ src/config_schema.h: resources/config-schema.json src/config_validator.o: src/config_schema.h +test_correct: \ + test/correct.test.o + $(CXX) $(CXXFLAGS) -o test.correct $^ $(INC) $(LIB) $(LDFLAGS) && ./test.correct + test_deque_map: \ test/deque_map.test.o $(CXX) $(CXXFLAGS) -o test.deque_map $^ $(INC) $(LIB) $(LDFLAGS) && ./test.deque_map @@ -223,6 +230,10 @@ test_options_parser: \ test/options_parser.test.o $(CXX) $(CXXFLAGS) -o test.options_parser $^ $(INC) $(LIB) $(LDFLAGS) && ./test.options_parser +test_ordered_batches: \ + test/ordered_batches.test.o + $(CXX) $(CXXFLAGS) -o test.ordered_batches $^ $(INC) $(LIB) $(LDFLAGS) && ./test.ordered_batches + test_osm_store: \ test/osm_store.test.o $(CXX) $(CXXFLAGS) -o test.osm_store $^ $(INC) $(LIB) $(LDFLAGS) && ./test.osm_store @@ -262,6 +273,22 @@ test_sorted_way_store: \ test/sorted_way_store.test.o $(CXX) $(CXXFLAGS) -o test.sorted_way_store $^ $(INC) $(LIB) $(LDFLAGS) && ./test.sorted_way_store +test_shp_mem_tiles: \ + src/attribute_store.o \ + src/coordinates.o \ + src/coordinates_geom.o \ + src/declutter.o \ + src/geom.o \ + src/mmap_allocator.o \ + src/output_object.o \ + src/pooled_string.o \ + src/shp_mem_tiles.o \ + src/tile_coordinates_set.o \ + src/tile_data.o \ + src/tile_sorting.o \ + test/shp_mem_tiles.test.o + $(CXX) $(CXXFLAGS) -o test.shp_mem_tiles $^ $(INC) $(LIB) $(LDFLAGS) && ./test.shp_mem_tiles + test_tile_coordinates_set: \ src/tile_coordinates_set.o \ test/tile_coordinates_set.test.o diff --git a/include/append_vector.h b/include/append_vector.h index 28b29a8b..8a66add9 100644 --- a/include/append_vector.h +++ b/include/append_vector.h @@ -162,6 +162,10 @@ namespace AppendVectorNS { return el; } + const T& operator [](int idx) const { + return vecs[idx / APPEND_VECTOR_SIZE][idx % APPEND_VECTOR_SIZE]; + } + Iterator begin() { return Iterator(*this, 0, 0); } diff --git a/include/attribute_store.h b/include/attribute_store.h index 295c495c..849b3889 100644 --- a/include/attribute_store.h +++ b/include/attribute_store.h @@ -9,6 +9,8 @@ #include #include #include +#include +#include #include #include #include "pooled_string.h" @@ -30,11 +32,16 @@ class AttributeKeyStore { uint16_t key2index(const std::string& key); const std::string& getKey(uint16_t index) const; const std::string& getKeyUnsafe(uint16_t index) const; - void finalize() { finalized = true; } + // Position of the key in alphabetical order. Unlike the key index, which is + // handed out in first-use order (i.e. depends on thread scheduling), it is + // the same in every run. Only valid after finalize(). + uint16_t getRankUnsafe(uint16_t index) const { return ranks[index]; } + void finalize(); std::atomic keys2indexSize; private: bool finalized; + std::vector ranks; mutable std::mutex keys2indexMutex; // NB: we use a deque, not a vector, because a deque never invalidates // pointers to its members as long as you only push_back @@ -439,6 +446,24 @@ struct AttributeStore { } attrSet->forEachPair(fn); } + // As forEachPairIndexUnsafe, but in an order that is the same in every run + // (by key name), not by pair index, which depends on thread scheduling. + template + void forEachPairIndexCanonicalUnsafe(AttributeIndex index, F&& fn) const { + boost::container::small_vector order; + forEachPairIndexUnsafe(index, [&](uint32_t pairIndex) { order.push_back(pairIndex); }); + if (order.size() > 1) + std::sort(order.begin(), order.end(), [this](uint32_t a, uint32_t b) { + return comparePairsUnsafe(a, b) < 0; + }); + for (const uint32_t pairIndex : order) + fn(pairIndex); + } + // Order and hash attribute pairs/sets by content (key names and values), not by + // index, so that they are the same in every run. Only valid after finalize(). + int comparePairsUnsafe(uint32_t a, uint32_t b) const; + int compareContentUnsafe(AttributeIndex a, AttributeIndex b) const; + uint64_t contentHashUnsafe(AttributeIndex index) const; void reset(); // used for testing size_t size() const; void reportSize() const; diff --git a/include/coordinates_geom.h b/include/coordinates_geom.h index f233ad3e..acf47b9f 100644 --- a/include/coordinates_geom.h +++ b/include/coordinates_geom.h @@ -28,6 +28,7 @@ class TileBbox { std::pair scaleLatpLon(double latp, double lon) const; void scaleRing(Ring &dst, Ring const &src) const; Ring scaleRing(Ring const &src) const; + void scaleRingNoBacktrack(Ring &dst, Ring const &src) const; void scaleGeometry(MultiPolygon &dst, MultiPolygon const &src) const; MultiPolygon scaleGeometry(MultiPolygon const &src) const; std::pair floorLatpLon(double latp, double lon) const; diff --git a/include/geometry/correct.hpp b/include/geometry/correct.hpp index fba96c69..b26c74af 100644 --- a/include/geometry/correct.hpp +++ b/include/geometry/correct.hpp @@ -10,6 +10,8 @@ * ---------------------------------------------------------------------------- */ +#include +#include #include #include #include @@ -23,13 +25,18 @@ namespace geometry { namespace impl { -template -static inline void result_combine(C &result, T &&new_element) +// boxes[i] is the envelope of result[i]. Polygons whose envelopes are disjoint +// cannot intersect, so the box test skips most of the costly intersects() calls +// (otherwise quadratic in the number of rings, e.g. a lake with 14000 islands). +template +static inline void result_combine(C &result, std::vector &boxes, T &&new_element) { result.push_back(std::forward(new_element)); + boxes.push_back(boost::geometry::return_envelope(result.back())); for(std::size_t i = 0; i < result.size() - 1; ) { - if(!boost::geometry::intersects(result[i], result.back())) { + if(boost::geometry::disjoint(boxes[i], boxes.back()) || + !boost::geometry::intersects(result[i], result.back())) { ++i; continue; } @@ -43,8 +50,10 @@ static inline void result_combine(C &result, T &&new_element) } result.back() = std::move(union_result[0]); + boxes.back() = boost::geometry::return_envelope(result.back()); result.erase(result.begin() + i); - } + boxes.erase(boxes.begin() + i); + } } struct pseudo_vertice_key @@ -199,27 +208,40 @@ static inline std::vector dissolve_generate_rings( // Generate all polygons by tracing all the intersections // Perform union to combine all polygons into single polygon again - while(!start_keys.empty()) { - ring_t new_ring; - - // Store point in generated polygon - auto push_point = [&new_ring](auto const &p) { - if(new_ring.empty() || boost::geometry::comparable_distance(new_ring.back(), p) > 0) - new_ring.push_back(p); - }; + // Position of each point in the ring being traced. Its points are distinct until + // it closes, so a point already seen closes it: looking that up here instead of + // scanning the whole ring for every new point keeps tracing linear, not quadratic. + struct point_hash { + std::size_t operator()(std::pair const &p) const { + return std::hash()(p.first) * 31 + std::hash()(p.second); + } + }; + std::unordered_map, std::size_t, point_hash> seen; - auto is_closed = [](ring_t &ring) { - if(ring.size() < 2) return false; - for(std::size_t i = 0; i < ring.size() - 1; ++i) { - if(boost::geometry::comparable_distance(ring[i], ring.back()) == 0) { - ring.erase(ring.begin(), ring.begin() + i); - return true; - } + while(!start_keys.empty()) { + ring_t new_ring; + seen.clear(); + bool closed = false; + + // Store point in generated polygon; when it repeats an earlier point, the + // ring is closed and the points before that one are dropped + auto push_point = [&new_ring, &seen, &closed](auto const &p) { + if(!new_ring.empty() && !(boost::geometry::comparable_distance(new_ring.back(), p) > 0)) + return; + new_ring.push_back(p); + // + 0.0 turns -0.0 into 0.0, which compares equal to it + std::pair key(boost::geometry::get<0>(p) + 0.0, boost::geometry::get<1>(p) + 0.0); + auto found = seen.find(key); + if(found == seen.end()) { + seen.emplace(key, new_ring.size() - 1); + } else { + new_ring.erase(new_ring.begin(), new_ring.begin() + found->second); + closed = true; } - - return false; }; + auto is_closed = [&closed](ring_t &) { return closed; }; + auto start_iter = pseudo_vertices.find(*start_keys.begin()); auto i = start_iter; @@ -302,13 +324,15 @@ template< > struct combine_non_zero_winding { - inline void operator()(multi_polygon_t &combined_outers, multi_polygon_t &combined_inners, polygon_t &poly) + using box_t = boost::geometry::model::box; + + inline void operator()(multi_polygon_t &combined_outers, std::vector &outer_boxes, multi_polygon_t &combined_inners, std::vector &inner_boxes, polygon_t &poly) { if(boost::geometry::area(poly) > 0) - result_combine(combined_outers, std::move(poly)); + result_combine(combined_outers, outer_boxes, std::move(poly)); else { std::reverse(poly.outer().begin(), poly.outer().end()); - result_combine(combined_inners, std::move(poly)); + result_combine(combined_inners, inner_boxes, std::move(poly)); } } }; @@ -320,7 +344,9 @@ template< > struct combine_odd_even { - inline void operator()(multi_polygon_t &combined_outers, multi_polygon_t &combined_inners, polygon_t &poly) + using box_t = boost::geometry::model::box; + + inline void operator()(multi_polygon_t &combined_outers, std::vector &, multi_polygon_t &combined_inners, std::vector &, polygon_t &poly) { if(boost::geometry::area(poly) < 0) std::reverse(poly.outer().begin(), poly.outer().end()); @@ -346,11 +372,12 @@ static inline void correct(polygon_t const &input, multi_polygon_t &output, doub // Calculate all outers and combine them if possible multi_polygon_t combined_outers; multi_polygon_t combined_inners; + std::vector> outer_boxes, inner_boxes; for(auto &ring: outer_rings) { polygon_t poly; poly.outer() = std::move(ring); - combine(combined_outers, combined_inners, poly); + combine(combined_outers, outer_boxes, combined_inners, inner_boxes, poly); } // Calculate all inners and combine them if possible @@ -362,8 +389,8 @@ static inline void correct(polygon_t const &input, multi_polygon_t &output, doub correct(poly, new_inners, remove_spike_min_area, combine); for(auto &poly: new_inners) { - result_combine(combined_inners, std::move(poly)); - } + result_combine(combined_inners, inner_boxes, std::move(poly)); + } } // Cut out all inners from all the outers @@ -379,13 +406,18 @@ template< > static inline void correct(multi_polygon_t const &input, multi_polygon_t &output, double remove_spike_min_area, combine_function_t combine) { + using box_t = boost::geometry::model::box; + std::vector output_boxes; + for(auto const &polygon: output) + output_boxes.push_back(boost::geometry::return_envelope(polygon)); + for(auto const &polygon: input) { multi_polygon_t new_polygons; correct(polygon, new_polygons, remove_spike_min_area, combine); - for(auto &new_polygon: new_polygons) - result_combine(output, std::move(new_polygon)); + for(auto &new_polygon: new_polygons) + result_combine(output, output_boxes, std::move(new_polygon)); } } diff --git a/include/mbtiles.h b/include/mbtiles.h index ca671b07..8181fafa 100644 --- a/include/mbtiles.h +++ b/include/mbtiles.h @@ -3,9 +3,11 @@ #define _MBTILES_H #include +#include #include #include #include "external/sqlite_modern_cpp.h" +#include "ordered_batches.h" struct PendingStatement { int zoom; @@ -27,6 +29,7 @@ class MBTiles { std::shared_ptr> pendingStatements1, pendingStatements2; std::mutex pendingStatementsMutex; + std::atomic pendingBytes; void insertOrReplace(int zoom, int x, int y, const std::string& data, bool isMerge); void flushPendingStatements(); @@ -39,6 +42,9 @@ class MBTiles { void saveTile(int zoom, int x, int y, std::string *data, bool isMerge); void closeForWriting(); + // Tiles are written in batch order, so the file doesn't depend on thread scheduling + OrderedBatches batches; + void openForReading(std::string &filename); void readBoundingBox(double &minLon, double &maxLon, double &minLat, double &maxLat); void readTileList(std::vector> &tileList); diff --git a/include/ordered_batches.h b/include/ordered_batches.h new file mode 100644 index 00000000..2d2b3b90 --- /dev/null +++ b/include/ordered_batches.h @@ -0,0 +1,113 @@ +/*! \file */ +#ifndef _ORDERED_BATCHES_H +#define _ORDERED_BATCHES_H + +#include +#include +#include +#include +#include +#include +#include + +// Writes batches of output in the order they were handed out, not the order threads +// finish them, so the output file doesn't depend on thread scheduling. +// +// A thread collects a batch's items between begin() and end(). end() writes the batch +// if it's the next one, followed by any batches that were waiting for it; otherwise it +// holds the batch. Batches must be numbered 0, 1, 2, ... in the order they're handed out, +// and a thread calls waitForRoom() before it takes its next batch: that keeps what is +// held below the limit set with setMaxHeldBytes(). The batch everyone waits for is always +// being worked on by a thread that isn't waiting, so waitForRoom() always returns. +// +// writeItem is called by one thread at a time, with writeMutex() held, in batch order. +template +class OrderedBatches { +public: + OrderedBatches(std::function writeItem, std::function itemBytes) + : writeItem(writeItem), itemBytes(itemBytes) { } + + void setMaxHeldBytes(uint64_t bytes) { + std::lock_guard lock(m); + maxHeldBytes = bytes; + } + + bool inBatch() const { return current().active; } + + void begin(uint64_t seq) { + Current& c = current(); + c.active = true; + c.seq = seq; + c.items.clear(); + } + + void add(T&& item) { current().items.emplace_back(std::move(item)); } + + // Hand the batch over. If it's the next one to be written and no other thread is + // writing, this thread writes it and then any held batches that follow it; meanwhile + // other threads just hand their batches over and carry on. + void end() { + Current& c = current(); + c.active = false; + uint64_t bytes = 0; + for (const auto& item : c.items) bytes += itemBytes(item); + std::unique_lock lock(m); + heldBytes += bytes; + held.emplace(c.seq, std::make_pair(bytes, std::move(c.items))); + c.items = std::vector(); + if (draining) return; + draining = true; + while (!held.empty() && held.begin()->first == nextSeq) { + auto batch = std::move(held.begin()->second); + held.erase(held.begin()); + lock.unlock(); + { + std::lock_guard writeLock(w); + for (auto& item : batch.second) writeItem(item); + } + lock.lock(); + heldBytes -= batch.first; + nextSeq++; + heldShrunk.notify_all(); + } + draining = false; + } + + void waitForRoom() { + std::unique_lock lock(m); + heldShrunk.wait(lock, [&]() { return heldBytes <= maxHeldBytes; }); + } + + bool allWritten() { + std::lock_guard lock(m); + return held.empty(); + } + + // Held while items are written; lock it to write anything outside a batch + std::mutex& writeMutex() { return w; } + +private: + struct Current { + bool active = false; + uint64_t seq = 0; + std::vector items; + }; + // The batch this thread is working on (one per thread and item type) + static Current& current() { + static thread_local Current c; + return c; + } + + std::function writeItem; + std::function itemBytes; + uint64_t maxHeldBytes = 1ull << 30; + std::mutex m; // guards the fields below + std::mutex w; // held while writing + std::condition_variable heldShrunk; + bool draining = false; + uint64_t nextSeq = 0; + uint64_t heldBytes = 0; + std::map>> held; // seq -> (bytes, items) +}; + +#endif //_ORDERED_BATCHES_H diff --git a/include/osm_store.h b/include/osm_store.h index 4f50cd5f..1f9915ec 100644 --- a/include/osm_store.h +++ b/include/osm_store.h @@ -154,22 +154,31 @@ class RelationScanStore { RelationScanStore(): relationsForWays(128), relationsForNodes(128), relationTags(128), mutex(128) {} + // Keep each member's parent relations ordered by relation id: relations are scanned on + // several threads, so appending would make the order NextRelation() returns depend on + // thread scheduling. Entries of the same relation keep their member order. + static void insertByRelation(std::vector>& list, RelationID relid, uint16_t roleId) { + const auto it = std::upper_bound(list.begin(), list.end(), relid, + [](RelationID id, const std::pair& entry) { return id < entry.first; }); + list.emplace(it, relid, roleId); + } + void relation_contains_way(RelationID relid, WayID wayid, std::string role) { uint16_t roleId = relationRoles.getOrAddRole(role); const size_t shard = wayid % mutex.size(); std::lock_guard lock(mutex[shard]); - relationsForWays[shard][wayid].emplace_back(std::make_pair(relid, roleId)); + insertByRelation(relationsForWays[shard][wayid], relid, roleId); } void relation_contains_node(RelationID relid, NodeID nodeId, std::string role) { uint16_t roleId = relationRoles.getOrAddRole(role); const size_t shard = nodeId % mutex.size(); std::lock_guard lock(mutex[shard]); - relationsForNodes[shard][nodeId].emplace_back(std::make_pair(relid, roleId)); + insertByRelation(relationsForNodes[shard][nodeId], relid, roleId); } void relation_contains_relation(RelationID relid, RelationID relationId, std::string role) { uint16_t roleId = relationRoles.getOrAddRole(role); std::lock_guard lock(mutex[0]); - relationsForRelations[relationId].emplace_back(std::make_pair(relid, roleId)); + insertByRelation(relationsForRelations[relationId], relid, roleId); } void store_relation_tags(RelationID relid, const tag_map_t &tags) { const size_t shard = relid % mutex.size(); diff --git a/include/pmtiles.h b/include/pmtiles.h index 7e8040c8..bcc701f1 100644 --- a/include/pmtiles.h +++ b/include/pmtiles.h @@ -3,7 +3,12 @@ #define _PMTILES_H #include +#include +#include +#include +#include #include "external/pmtiles.hpp" +#include "ordered_batches.h" struct TileOffset { uint64_t offset : 40; @@ -28,6 +33,14 @@ struct TileOffset { // PMTiles v3 tile type for MapLibre Tiles (not yet in external/pmtiles.hpp) #define TILETYPE_MLT 0x6 +// A tile waiting to be written: tiny tiles keep their raw data, so that a repeat +// can reuse the earlier copy, and are only compressed if they're written +struct PendingTile { + uint64_t tileId; + bool isTiny; + std::string data; +}; + class PMTiles { public: @@ -42,10 +55,12 @@ class PMTiles { void saveTile(int zoom, int x, int y, std::string &data); void close(std::string &metadata); + // Tiles are written in batch order, so the file doesn't depend on thread scheduling + OrderedBatches batches; + private: std::ofstream outputStream; - std::mutex fileMutex; // guards file writes, numTilesWritten - std::mutex indexMutex; // guards access to sparseIndex, denseIndex, tinyCache, numTilesAddressed + std::mutex fileMutex; // guards leaf directory writes in close() uint64_t leafStart = 0; uint64_t numTilesWritten = 0; uint64_t numTilesAddressed = 0; @@ -54,6 +69,7 @@ class PMTiles { std::vector denseIndex; std::unordered_map tinyCache; + void writeTile(PendingTile &tile); // with batches.writeMutex() held void appendWithRLE(std::vector &entries, pmtiles::entryv3 &entry); void appendTileEntry(uint64_t tileId, TileOffset offset, std::vector &rootEntries, std::vector &entries); void flushEntries(std::vector &rootEntries, std::vector &entries); diff --git a/include/shp_mem_tiles.h b/include/shp_mem_tiles.h index a5cf4f3f..0bab561c 100644 --- a/include/shp_mem_tiles.h +++ b/include/shp_mem_tiles.h @@ -27,7 +27,8 @@ class ShpMemTiles : public TileDataSource const std::string& name, uint minzoom, int32_t score, - AttributeIndex attrIdx + AttributeIndex attrIdx, + bool knownValid = false // polygon: caller has checked it is valid ); std::vector QueryMatchingGeometries( diff --git a/include/stable_hash.h b/include/stable_hash.h new file mode 100644 index 00000000..e1af029f --- /dev/null +++ b/include/stable_hash.h @@ -0,0 +1,42 @@ +/*! \file */ +#ifndef _STABLE_HASH_H +#define _STABLE_HASH_H + +#include +#include +#include + +// Hash functions that give the same result on every platform, compiler and +// library version (unlike std::hash or boost::hash), for orderings that the +// output depends on. + +inline uint64_t stableMix(uint64_t x) { + // splitmix64 finaliser + x += 0x9e3779b97f4a7c15ull; + x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9ull; + x = (x ^ (x >> 27)) * 0x94d049bb133111ebull; + return x ^ (x >> 31); +} + +inline uint64_t stableCombine(uint64_t seed, uint64_t value) { + return stableMix(seed ^ stableMix(value)); +} + +inline uint64_t stableCombine(uint64_t seed, double value) { + if (value == 0) value = 0; // -0.0 and 0.0 compare equal, so hash them equally + uint64_t bits; + std::memcpy(&bits, &value, sizeof(bits)); + return stableCombine(seed, bits); +} + +inline uint64_t stableCombine(uint64_t seed, const char* data, std::size_t length) { + // FNV-1a over the bytes + uint64_t h = 0xcbf29ce484222325ull; + for (std::size_t i = 0; i < length; i++) { + h ^= static_cast(data[i]); + h *= 0x100000001b3ull; + } + return stableCombine(stableCombine(seed, static_cast(length)), h); +} + +#endif //_STABLE_HASH_H diff --git a/include/tile_coordinates_set.h b/include/tile_coordinates_set.h index 3d45c50e..21f14649 100644 --- a/include/tile_coordinates_set.h +++ b/include/tile_coordinates_set.h @@ -14,6 +14,8 @@ class TileCoordinatesSet { virtual void set(TileCoordinate x, TileCoordinate y) = 0; virtual size_t size() const = 0; virtual size_t zoom() const = 0; + // Set every tile with minX <= x <= maxX and minY <= y <= maxY. + virtual void setBox(TileCoordinate minX, TileCoordinate maxX, TileCoordinate minY, TileCoordinate maxY); // Visit every set tile in x-major, y-minor order. virtual void forEachSet(const std::function& f) const; }; @@ -30,7 +32,10 @@ class PreciseTileCoordinatesSet : public TileCoordinatesSet { size_t size() const override; size_t zoom() const override; void set(TileCoordinate x, TileCoordinate y) override; + void setBox(TileCoordinate minX, TileCoordinate maxX, TileCoordinate minY, TileCoordinate maxY) override; void forEachSet(const std::function& f) const override; + // Add the tiles set in other, a set of the same zoom. + void merge(const PreciseTileCoordinatesSet& other); private: unsigned int zoom_; @@ -50,6 +55,7 @@ class LossyTileCoordinatesSet : public TileCoordinatesSet { size_t size() const override; size_t zoom() const override; void set(TileCoordinate x, TileCoordinate y) override; + void forEachSet(const std::function& f) const override; private: unsigned int zoom_; diff --git a/include/tile_data.h b/include/tile_data.h index 3e8d20e9..729ef009 100644 --- a/include/tile_data.h +++ b/include/tile_data.h @@ -2,10 +2,15 @@ #ifndef _TILE_DATA_H #define _TILE_DATA_H +#include +#include #include +#include #include #include #include +#include +#include #include "append_vector.h" #include "clip_cache.h" #include "mmap_allocator.h" @@ -31,77 +36,134 @@ template void finalizeObjects( typename std::vector>::iterator end, typename std::vector>& lowZoom ) { - size_t z6OffsetDivisor = indexZoom >= CLUSTER_ZOOM ? (1 << (indexZoom - CLUSTER_ZOOM)) : 1; #ifdef CLOCK_MONOTONIC timespec startTs, endTs; clock_gettime(CLOCK_MONOTONIC, &startTs); #endif - - int i = -1; - for (auto it = begin; it != end; it++) { - i++; - if (it->size() > 0 || i % 50 == 0 || i == 4095) { - std::cout << "\r" << name << ": finalizing z6 tile " << (i + 1) << "/" << CLUSTER_ZOOM_AREA; - + const size_t shards = end - begin; + std::mutex progressMutex; + size_t done = 0; + auto reportDone = [&](size_t n) { + std::lock_guard lock(progressMutex); + done += n; + std::cout << "\r" << name << ": finalizing z6 tile " << done << "/" << CLUSTER_ZOOM_AREA; #ifdef CLOCK_MONOTONIC - clock_gettime(CLOCK_MONOTONIC, &endTs); - uint64_t elapsedNs = 1e9 * (endTs.tv_sec - startTs.tv_sec) + endTs.tv_nsec - startTs.tv_nsec; - std::cout << " (" << std::to_string((uint32_t)(elapsedNs / 1e6)) << " ms)"; + clock_gettime(CLOCK_MONOTONIC, &endTs); + uint64_t elapsedNs = 1e9 * (endTs.tv_sec - startTs.tv_sec) + endTs.tv_nsec - startTs.tv_nsec; + std::cout << " (" << std::to_string((uint32_t)(elapsedNs / 1e6)) << " ms)"; #endif - std::cout << std::flush; - } - if (it->size() == 0) - continue; + std::cout << std::flush; + }; + auto finalizeShard = [&](size_t i, size_t sortThreads) { + auto it = begin + i; // We track a separate copy of low zoom objects to avoid scanning large // lists of objects that may be on slow disk storage. for (auto objectIt = it->begin(); objectIt != it->end(); objectIt++) if (objectIt->oo.minZoom < CLUSTER_ZOOM) lowZoom[i].push_back(*objectIt); - sortOutputObjects(indexZoom, threadNum, it->begin(), it->end()); + sortOutputObjects(indexZoom, sortThreads, it->begin(), it->end()); + }; + + // Shards are independent. A shard with more than 1/threadNum of all objects + // (e.g. one of the few populated z6 tiles of a small extract) is sorted on its + // own using all threads. The others are sorted in parallel, one thread each, + // largest first. + size_t total = 0, empty = 0; + for (auto it = begin; it != end; it++) total += it->size(); + std::vector big, small; + for (size_t i = 0; i < shards; i++) { + const size_t n = (begin + i)->size(); + if (n == 0) empty++; + else if (threadNum <= 1 || n * threadNum > total) big.push_back(i); + else small.push_back(i); + } + std::sort(small.begin(), small.end(), [&](size_t a, size_t b) { return (begin + a)->size() > (begin + b)->size(); }); + reportDone(empty); + + for (size_t i : big) { + finalizeShard(i, threadNum); + reportDone(1); + } + + if (!small.empty()) { + // Each sort copies its shard into a buffer, so cap the objects being + // sorted at once (a single shard may always run). + const size_t budget = std::max(total / 8, (begin + small.front())->size()); + size_t inFlight = 0; + std::mutex budgetMutex; + std::condition_variable budgetFreed; + boost::asio::thread_pool pool(threadNum); + for (size_t i : small) { + boost::asio::post(pool, [&, i]() { + const size_t n = (begin + i)->size(); + { + std::unique_lock lock(budgetMutex); + budgetFreed.wait(lock, [&]() { return inFlight == 0 || inFlight + n <= budget; }); + inFlight += n; + } + finalizeShard(i, 1); + { + std::lock_guard lock(budgetMutex); + inFlight -= n; + } + budgetFreed.notify_all(); + reportDone(1); + }); + } + pool.join(); } std::cout << std::endl; } -template void collectTilesWithObjectsAtZoomTemplate( +// Set the tiles holding the objects of z6 tile i (shard), and their parent tiles. +template void collectTilesWithObjectsInShardTemplate( const unsigned int& indexZoom, - const typename std::vector>::iterator objects, - const size_t size, + AppendVectorNS::AppendVector& shard, + const size_t i, std::vector>& zooms ) { size_t maxZoom = zooms.size() - 1; uint16_t z6OffsetDivisor = indexZoom >= CLUSTER_ZOOM ? (1 << (indexZoom - CLUSTER_ZOOM)) : 1; int64_t lastX = -1; int64_t lastY = -1; - for (size_t i = 0; i < size; i++) { - const size_t z6x = i / CLUSTER_ZOOM_WIDTH; - const size_t z6y = i % CLUSTER_ZOOM_WIDTH; - - for (size_t j = 0; j < objects[i].size(); j++) { - // Compute the x, y at the base zoom level - TileCoordinate baseX = z6x * z6OffsetDivisor + objects[i][j].x; - TileCoordinate baseY = z6y * z6OffsetDivisor + objects[i][j].y; - - // Translate the x, y at the requested zoom level - TileCoordinate x = baseX / (1 << (indexZoom - maxZoom)); - TileCoordinate y = baseY / (1 << (indexZoom - maxZoom)); - - if (lastX != x || lastY != y) { - lastX = x; - lastY = y; - - for (int zoom = maxZoom; zoom >= 0; zoom--) { - zooms[zoom]->set(x, y); - x /= 2; - y /= 2; - } + const size_t z6x = i / CLUSTER_ZOOM_WIDTH; + const size_t z6y = i % CLUSTER_ZOOM_WIDTH; + + for (size_t j = 0; j < shard.size(); j++) { + // Compute the x, y at the base zoom level + TileCoordinate baseX = z6x * z6OffsetDivisor + shard[j].x; + TileCoordinate baseY = z6y * z6OffsetDivisor + shard[j].y; + + // Translate the x, y at the requested zoom level + TileCoordinate x = baseX / (1 << (indexZoom - maxZoom)); + TileCoordinate y = baseY / (1 << (indexZoom - maxZoom)); + + if (lastX != x || lastY != y) { + lastX = x; + lastY = y; + + for (int zoom = maxZoom; zoom >= 0; zoom--) { + zooms[zoom]->set(x, y); + x /= 2; + y /= 2; } } } } +template void collectTilesWithObjectsAtZoomTemplate( + const unsigned int& indexZoom, + const typename std::vector>::iterator objects, + const size_t size, + std::vector>& zooms +) { + for (size_t i = 0; i < size; i++) + collectTilesWithObjectsInShardTemplate(indexZoom, objects[i], i, zooms); +} + template OutputObjectID outputObjectWithId(const OO& input) { return OutputObjectID({ input.oo, 0 }); @@ -245,7 +307,7 @@ template void collectObjectsForTileTemplate( class TileDataSource { public: // Store for generated geometries - using point_store_t = std::vector; + using point_store_t = AppendVectorNS::AppendVector; using linestring_t = boost::geometry::model::linestring; using linestring_store_t = std::vector; @@ -310,6 +372,8 @@ class TileDataSource { TileDataSource(size_t threadNum, unsigned int indexZoom, bool includeID); void collectTilesWithObjectsAtZoom(std::vector>& zooms); + // The same for the objects of one z6 tile (0 <= shard < CLUSTER_ZOOM_AREA) + void collectTilesWithObjectsInShard(size_t shard, std::vector>& zooms); void collectTilesWithLargeObjectsAtZoom(std::vector>& zooms); @@ -348,9 +412,11 @@ class TileDataSource { std::vector getObjectsForTile( const std::vector& sortOrders, + const AttributeStore& attributeStore, unsigned int zoom, TileCoordinates coordinates ); + void orderStoredGeometryTies(std::vector& data) const; virtual Geometry buildWayGeometry(OutputGeometryType const geomType, NodeID const objectID, const TileBbox &bbox); virtual LatpLon buildNodeGeometry(NodeID const objectID, const TileBbox &bbox) const; @@ -398,8 +464,8 @@ class TileDataSource { const auto& shardId = getShard(id); const auto& shard = pointStores[shardId]; const auto offset = getId(id); - if (offset > shard.size()) throw std::out_of_range("Could not find generated node with id " + std::to_string(id) + ", shard " + std::to_string(shardId) + ", offset=" + std::to_string(offset)); - return shard.at(offset); + if (offset >= shard.size()) throw std::out_of_range("Could not find generated node with id " + std::to_string(id) + ", shard " + std::to_string(shardId) + ", offset=" + std::to_string(offset)); + return shard[offset]; } NodeID storeLinestring(const Linestring& src); @@ -436,7 +502,8 @@ class TileDataSource { void populateTilesAtZoom( const std::vector& sources, - std::vector>& zooms + std::vector>& zooms, + size_t threadNum = 1 ); #endif //_TILE_DATA_H diff --git a/resources/process-debug.lua b/resources/process-debug.lua index 4ea5e629..5fc399f1 100644 --- a/resources/process-debug.lua +++ b/resources/process-debug.lua @@ -155,6 +155,10 @@ poiTags = { aerialway = Set { "station" }, sport = Set { "american_football", "archery", "athletics", "australian_football", "badminton", "baseball", "basketball", "beachvolleyball", "billiards", "bmx", "boules", "bowls", "boxing", "canadian_football", "canoe", "chess", "climbing", "climbing_adventure", "cricket", "cricket_nets", "croquet", "curling", "cycling", "disc_golf", "diving", "dog_racing", "equestrian", "fatsal", "field_hockey", "free_flying", "gaelic_games", "golf", "gymnastics", "handball", "hockey", "horse_racing", "horseshoes", "ice_hockey", "ice_stock", "judo", "karting", "korfball", "long_jump", "model_aerodrome", "motocross", "motor", "multi", "netball", "orienteering", "paddle_tennis", "paintball", "paragliding", "pelota", "racquet", "rc_car", "rowing", "rugby", "rugby_league", "rugby_union", "running", "sailing", "scuba_diving", "shooting", "shooting_range", "skateboard", "skating", "skiing", "soccer", "surfing", "swimming", "table_soccer", "table_tennis", "team_handball", "tennis", "toboggan", "volleyball", "water_ski", "yoga" }, tourism = Set { "alpine_hut", "aquarium", "artwork", "attraction", "bed_and_breakfast", "camp_site", "caravan_site", "chalet", "gallery", "guest_house", "hostel", "hotel", "information", "motel", "museum", "picnic_site", "theme_park", "viewpoint", "zoo" }, waterway = Set { "dock" } } +-- poiTags keys in alphabetical order: GetPOIRank tries them in this order, and the first key an object has decides its class +poiTagOrder = {} +for k in pairs(poiTags) do poiTagOrder[#poiTagOrder+1] = k end +table.sort(poiTagOrder) -- POI "class" values: based on https://github.com/openmaptiles/openmaptiles/blob/master/layers/poi/poi.yaml poiClasses = { townhall="town_hall", public_building="town_hall", courthouse="town_hall", community_centre="town_hall", @@ -504,7 +508,8 @@ function GetPOIRank(obj) local k,list,v,class,rank -- Can we find the tag? - for k,list in pairs(poiTags) do + for _,k in ipairs(poiTagOrder) do + list = poiTags[k] if list[Find(k)] then v = Find(k) -- k/v are the OSM tag pair class = poiClasses[v] or v diff --git a/resources/process-openmaptiles.lua b/resources/process-openmaptiles.lua index 74892add..7591e60f 100644 --- a/resources/process-openmaptiles.lua +++ b/resources/process-openmaptiles.lua @@ -271,6 +271,10 @@ poiTags = { aerialway = Set { "station" }, sport = Set { "american_football", "archery", "athletics", "australian_football", "badminton", "baseball", "basketball", "beachvolleyball", "billiards", "bmx", "boules", "bowls", "boxing", "canadian_football", "canoe", "chess", "climbing", "climbing_adventure", "cricket", "cricket_nets", "croquet", "curling", "cycling", "disc_golf", "diving", "dog_racing", "equestrian", "fatsal", "field_hockey", "free_flying", "gaelic_games", "golf", "gymnastics", "handball", "hockey", "horse_racing", "horseshoes", "ice_hockey", "ice_stock", "judo", "karting", "korfball", "long_jump", "model_aerodrome", "motocross", "motor", "multi", "netball", "orienteering", "paddle_tennis", "paintball", "paragliding", "pelota", "racquet", "rc_car", "rowing", "rugby", "rugby_league", "rugby_union", "running", "sailing", "scuba_diving", "shooting", "shooting_range", "skateboard", "skating", "skiing", "soccer", "surfing", "swimming", "table_soccer", "table_tennis", "team_handball", "tennis", "toboggan", "volleyball", "water_ski", "yoga" }, tourism = Set { "alpine_hut", "aquarium", "artwork", "attraction", "bed_and_breakfast", "camp_site", "caravan_site", "chalet", "gallery", "guest_house", "hostel", "hotel", "information", "motel", "museum", "picnic_site", "theme_park", "viewpoint", "zoo" }, waterway = Set { "dock" } } +-- poiTags keys in alphabetical order: GetPOIRank tries them in this order, and the first key an object has decides its class +poiTagOrder = {} +for k in pairs(poiTags) do poiTagOrder[#poiTagOrder+1] = k end +table.sort(poiTagOrder) -- POI "class" values: based on https://github.com/openmaptiles/openmaptiles/blob/master/layers/poi/poi.yaml poiClasses = { townhall="town_hall", public_building="town_hall", courthouse="town_hall", community_centre="town_hall", @@ -912,7 +916,8 @@ function GetPOIRank() local k,list,v,class,rank -- Can we find the tag? - for k,list in pairs(poiTags) do + for _,k in ipairs(poiTagOrder) do + list = poiTags[k] if list[Find(k)] then v = Find(k) -- k/v are the OSM tag pair class = poiClasses[v] or poiTagsAsClass[k] or v diff --git a/src/attribute_store.cpp b/src/attribute_store.cpp index 6e661c8d..f2a3d77d 100644 --- a/src/attribute_store.cpp +++ b/src/attribute_store.cpp @@ -1,4 +1,5 @@ #include "attribute_store.h" +#include "stable_hash.h" #include #include @@ -55,6 +56,15 @@ const std::string& AttributeKeyStore::getKeyUnsafe(uint16_t index) const { return keys[index]; } +void AttributeKeyStore::finalize() { + finalized = true; + // keys2index is ordered by key string + ranks.assign(keys.size(), 0); + uint16_t rank = 0; + for (const auto& it : keys2index) + ranks[it.second] = ++rank; +} + // AttributePair void AttributePair::ensureStringIsOwned() { // Before we store an AttributePair in our long-term storage, we need @@ -557,6 +567,52 @@ void AttributeStore::reset() { pairIndexGeneration++; } +int AttributeStore::comparePairsUnsafe(uint32_t a, uint32_t b) const { + if (a == b) return 0; + const AttributePair& p = pairStore.getPairUnsafe(a); + const AttributePair& q = pairStore.getPairUnsafe(b); + const uint16_t pRank = keyStore.getRankUnsafe(p.keyIndex), qRank = keyStore.getRankUnsafe(q.keyIndex); + if (pRank != qRank) return pRank < qRank ? -1 : 1; + if (p.minzoom != q.minzoom) return p.minzoom < q.minzoom ? -1 : 1; + if (p.valueType != q.valueType) return p.valueType < q.valueType ? -1 : 1; + if (p.hasStringValue()) { + if (p.pooledString() < q.pooledString()) return -1; + if (q.pooledString() < p.pooledString()) return 1; + return 0; + } + if (p.doubleValue_ < q.doubleValue_) return -1; + if (q.doubleValue_ < p.doubleValue_) return 1; + return 0; +} + +int AttributeStore::compareContentUnsafe(AttributeIndex a, AttributeIndex b) const { + if (a == b) return 0; + boost::container::small_vector x, y; + forEachPairIndexCanonicalUnsafe(a, [&](uint32_t pairIndex) { x.push_back(pairIndex); }); + forEachPairIndexCanonicalUnsafe(b, [&](uint32_t pairIndex) { y.push_back(pairIndex); }); + for (size_t i = 0; i < x.size() && i < y.size(); i++) { + const int c = comparePairsUnsafe(x[i], y[i]); + if (c != 0) return c; + } + if (x.size() != y.size()) return x.size() < y.size() ? -1 : 1; + return 0; +} + +uint64_t AttributeStore::contentHashUnsafe(AttributeIndex index) const { + // Sum of the pair hashes, so the (index dependent) pair order doesn't matter + uint64_t rv = 0; + forEachPairUnsafe(index, [&](const AttributePair& p) { + uint64_t h = stableCombine(keyStore.getRankUnsafe(p.keyIndex), static_cast(p.minzoom)); + h = stableCombine(h, static_cast(p.valueType)); + if (p.hasStringValue()) + h = stableCombine(h, p.pooledString().data(), p.pooledString().size()); + else + h = stableCombine(h, p.doubleValue_); + rv += h; + }); + return rv; +} + void AttributeStore::finalize() { finalized = true; keyStore.finalize(); diff --git a/src/coordinates_geom.cpp b/src/coordinates_geom.cpp index 16887856..d6c35d86 100644 --- a/src/coordinates_geom.cpp +++ b/src/coordinates_geom.cpp @@ -50,6 +50,24 @@ void TileBbox::scaleRing(Ring &points, Ring const &src) const { } } +// Scaling that only drops points repeating their immediate predecessor. Used as +// a fallback for rings that scaleRing() collapses below 4 points: its +// backtracking window is positional, so a duplicated FIRST vertex shifts every +// later vertex by one, the closing vertex then matches that duplicate at j==4, +// and resize() truncates the whole ring. Without the backtracking there is +// nothing to mis-align, and the ring survives with its shape intact. +void TileBbox::scaleRingNoBacktrack(Ring &points, Ring const &src) const { + points.clear(); + points.reserve(src.size()); + for(auto const &i: src) { + auto scaled = scaleLatpLon(i.y(), i.x()); + if (!points.empty() && + points.back().x()==scaled.first && points.back().y()==scaled.second) + continue; + points.push_back(Point(scaled.first,scaled.second)); + } +} + Ring TileBbox::scaleRing(Ring const &src) const { Ring points; scaleRing(points, src); @@ -66,8 +84,15 @@ void TileBbox::scaleGeometry(MultiPolygon &dst, MultiPolygon const &src) const { // Copy the outer ring scaleRing(p.outer(), poly.outer()); - if (p.outer().size()<4) - continue; + if (p.outer().size()<4) { + // A collapsed outer ring means the whole feature disappears from the + // tile, at any size - so before giving up, retry without the + // backtracking that can truncate a ring to two points. Every ring that + // already survives is left untouched. + scaleRingNoBacktrack(p.outer(), poly.outer()); + if (p.outer().size()<4) + continue; + } // Copy the inner rings if (p.inners().size() < poly.inners().size()) diff --git a/src/geom.cpp b/src/geom.cpp index 43d7d079..a3221c2b 100644 --- a/src/geom.cpp +++ b/src/geom.cpp @@ -178,13 +178,17 @@ struct PrecheckBox { }; struct PrecheckScratch { std::vector segs; - std::vector boxes, holeBoxes; + std::vector boxes; std::vector> pairs; + std::vector bandStart, bandFill, bandSegs, crossed; void trim() { if (segs.capacity() > kPrecheckRetained) std::vector().swap(segs); if (boxes.capacity() > kPrecheckRetained) std::vector().swap(boxes); - if (holeBoxes.capacity() > kPrecheckRetained) std::vector().swap(holeBoxes); if (pairs.capacity() > kPrecheckRetained) std::vector>().swap(pairs); + if (bandStart.capacity() > kPrecheckRetained) std::vector().swap(bandStart); + if (bandFill.capacity() > kPrecheckRetained) std::vector().swap(bandFill); + if (bandSegs.capacity() > kPrecheckRetained) std::vector().swap(bandSegs); + if (crossed.capacity() > kPrecheckRetained) std::vector().swap(crossed); } }; thread_local PrecheckScratch precheckScratch; @@ -331,20 +335,27 @@ bool segmentsClear(std::vector &segs, double tol) { return true; } +// Whether the ray from h to +x crosses segment a->b: 1 = yes, -1 = no, 0 = unsure. +// Vertices are classed as above (y > hy) or not with exact comparisons (the +// half-open rule), so a ray through a vertex is counted consistently and the +// crossing parity is exact; only the side of h is decided with a tolerance. +inline int rayClearlyCrosses(const Point &h, const Point &a, const Point &b, double tol) { + const double hx = h.x(), hy = h.y(); + if ((a.y() > hy) == (b.y() > hy)) return -1; + if (std::max(a.x(), b.x()) < hx) return -1; + if (std::min(a.x(), b.x()) > hx) return 1; + const int o = clearOrient(a, b, h, tol); + if (o == 0) return 0; + return (b.y() > a.y()) == (o > 0) ? 1 : -1; +} + // Crossing-number test: 1 = h clearly inside r, -1 = clearly outside, 0 = unsure. int pointClearlyInRing(const Point &h, const Ring &r, double tol) { bool inside = false; - const double hx = h.x(), hy = h.y(); for (std::size_t i = 0; i + 1 < r.size(); i++) { - const Point &a = r[i], &b = r[i + 1]; - if ((a.y() > hy + tol && b.y() > hy + tol) || (a.y() < hy - tol && b.y() < hy - tol)) continue; - if (std::fabs(a.y() - hy) <= tol || std::fabs(b.y() - hy) <= tol) return 0; - // a and b are clearly on opposite sides of the horizontal line through h - if (std::max(a.x(), b.x()) < hx - tol) continue; - if (std::min(a.x(), b.x()) > hx + tol) { inside = !inside; continue; } - const int o = clearOrient(a, b, h, tol); - if (o == 0) return 0; - if ((b.y() > a.y()) == (o > 0)) inside = !inside; // the ray to +x crosses a->b + const int c = rayClearlyCrosses(h, r[i], r[i + 1], tol); + if (c == 0) return 0; + if (c > 0) inside = !inside; } return inside ? 1 : -1; } @@ -360,12 +371,91 @@ int ringClearlyInRing(const Ring &inner, const Ring &r, double tol) { return 0; } +// Holes lie inside the outer ring (ring 0) and outside each other. `segs` holds the +// segments of all rings of p, known not to touch. They are bucketed into horizontal +// bands, so that a ray from a vertex of each hole is tested against the segments of +// that vertex's band only, counting crossings per ring (odd = inside that ring). +bool holesClearlyPlaced(const Polygon &p, PrecheckScratch &scratch, double tol) { + const auto &inners = p.inners(); + const std::vector &segs = scratch.segs; + if (inners.empty()) return true; + if (segs.size() >= std::numeric_limits::max()) return false; + + double minY = std::numeric_limits::infinity(), maxY = -minY; + for (const PrecheckSeg &s : segs) { minY = std::min(minY, s.minY); maxY = std::max(maxY, s.maxY); } + minY -= tol; maxY += tol; + // About 4 segments per band; fewer bands while tall segments would make the index much bigger than segs + std::size_t bands = std::max(1, segs.size() / 4); + double scale = 0; + auto bandOf = [&](double y) { + const double b = (y - minY) * scale; + return b <= 0 ? std::size_t(0) : std::min(bands - 1, static_cast(b)); + }; + std::size_t entries; + for (;;) { + scale = bands / (maxY - minY); + entries = 0; + for (const PrecheckSeg &s : segs) entries += bandOf(s.maxY + tol) - bandOf(s.minY - tol) + 1; + if (entries <= 4 * segs.size() || bands == 1) break; + bands = (bands + 1) / 2; + } + if (entries >= std::numeric_limits::max()) return false; + + // Segments of band b: bandSegs[bandStart[b] .. bandStart[b+1]) + std::vector &start = scratch.bandStart, &fill = scratch.bandFill, &items = scratch.bandSegs; + start.assign(bands + 1, 0); + for (const PrecheckSeg &s : segs) + for (std::size_t b = bandOf(s.minY - tol), hi = bandOf(s.maxY + tol); b <= hi; b++) start[b + 1]++; + for (std::size_t b = 0; b < bands; b++) start[b + 1] += start[b]; + fill.assign(start.begin(), start.end() - 1); + items.resize(entries); + for (std::size_t k = 0; k < segs.size(); k++) + for (std::size_t b = bandOf(segs[k].minY - tol), hi = bandOf(segs[k].maxY + tol); b <= hi; b++) + items[fill[b]++] = static_cast(k); + + // 1 = h inside the outer ring and no other hole, -1 = not, 0 = unsure + std::vector &crossed = scratch.crossed; + auto locate = [&](const Point &h, uint32_t ownRing) { + crossed.clear(); + const std::size_t b = bandOf(h.y()); + for (uint32_t k = start[b]; k < start[b + 1]; k++) { + const PrecheckSeg &s = segs[items[k]]; + if (s.ring == ownRing) continue; + const int c = rayClearlyCrosses(h, s.a[0], s.a[1], tol); + if (c == 0) return 0; + if (c > 0) crossed.push_back(s.ring); + } + std::sort(crossed.begin(), crossed.end()); + bool inOuter = false; + for (std::size_t i = 0; i < crossed.size(); ) { + std::size_t j = i; + while (j < crossed.size() && crossed[j] == crossed[i]) j++; + if ((j - i) % 2 == 1) { + if (crossed[i] != 0) return -1; // inside another hole + inOuter = true; + } + i = j; + } + return inOuter ? 1 : -1; + }; + + // A ring known not to touch the others lies wherever any of its vertices does + for (std::size_t i = 0; i < inners.size(); i++) { + const Ring &hole = inners[i]; + const std::size_t m = hole.size() - 1; + const std::size_t tries = std::min(m, 8); + int loc = 0; + for (std::size_t k = 0; k < tries && loc == 0; k++) + loc = locate(hole[k * m / tries], static_cast(i + 1)); + if (loc != 1) return false; + } + return true; +} + bool polygonClear(const Polygon &p, PrecheckScratch &scratch, PrecheckBox &outerBox) { const Ring &outer = p.outer(); const auto &inners = p.inners(); if (inners.size() >= std::numeric_limits::max()) return false; - // Every hole is located by walking the outer ring - if (inners.size() * outer.size() > kPrecheckMaxWork) return false; double maxAbs = 0; if (!ringFinite(outer, maxAbs)) return false; for (const auto &inner : inners) @@ -374,29 +464,16 @@ bool polygonClear(const Polygon &p, PrecheckScratch &scratch, PrecheckBox &outer std::vector &segs = scratch.segs; segs.clear(); - std::vector &holeBoxes = scratch.holeBoxes; - holeBoxes.resize(inners.size()); if (!ringClear(outer, false, tol, 0, segs, outerBox)) return false; - for (std::size_t i = 0; i < inners.size(); i++) { - if (!ringClear(inners[i], true, tol, static_cast(i + 1), segs, holeBoxes[i])) return false; - holeBoxes[i].id = static_cast(i); - } + PrecheckBox holeBox; + for (std::size_t i = 0; i < inners.size(); i++) + if (!ringClear(inners[i], true, tol, static_cast(i + 1), segs, holeBox)) return false; // No two rings touch, and no ring touches itself. if (!segmentsClear(segs, tol)) return false; // Holes lie inside the outer ring and are not nested in each other. - for (std::size_t i = 0; i < inners.size(); i++) { - if (boxesApart(holeBoxes[i], outerBox, tol)) return false; - if (ringClearlyInRing(inners[i], outer, tol) != 1) return false; - } - std::size_t work = 0; - return forOverlappingBoxes(holeBoxes, tol, [&](uint32_t i, uint32_t j) { - work += inners[i].size() + inners[j].size(); - return work <= kPrecheckMaxWork && - ringClearlyInRing(inners[i], inners[j], tol) == -1 && - ringClearlyInRing(inners[j], inners[i], tol) == -1; - }); + return holesClearlyPlaced(p, scratch, tol); } // a's outer ring, known not to touch any ring of b, lies outside b's interior: @@ -839,6 +916,11 @@ struct RectClipper { bool inside(const Point &p) const { return p.x() >= x0 && p.x() <= x1 && p.y() >= y0 && p.y() <= y1; } + // Both end points beyond the same edge: the segment can't reach the box + bool beyondOneEdge(const Point &a, const Point &b) const { + return (a.x() < x0 && b.x() < x0) || (a.x() > x1 && b.x() > x1) || + (a.y() < y0 && b.y() < y0) || (a.y() > y1 && b.y() > y1); + } bool strictlyInside(const Point &p) const { return p.x() > x0 && p.x() < x1 && p.y() > y0 && p.y() < y1; } @@ -965,7 +1047,9 @@ RingClip clipRing(const RectClipper &rc, const Ring &ring, std::vector t0; + // Most segments of a large polygon lie far from a small tile's box: reject + // those with four comparisons before the clip computation + const bool hit = !rc.beyondOneEdge(a, b) && rc.clipSegment(a, b, t0, t1, pa, pb) && t1 > t0; if (!hit || (building && t0 > 0)) { if (building && !finish()) return RingClip::Fail; if (!hit) continue; diff --git a/src/mbtiles.cpp b/src/mbtiles.cpp index c056d3a6..1d2e653f 100644 --- a/src/mbtiles.cpp +++ b/src/mbtiles.cpp @@ -2,15 +2,30 @@ #include "helpers.h" #include #include +#include using namespace sqlite; using namespace std; MBTiles::MBTiles(): pendingStatements1(std::make_shared>()), - pendingStatements2(std::make_shared>()) + pendingStatements2(std::make_shared>()), + pendingBytes(0), + batches([this](PendingStatement& stmt) { + const std::lock_guard lock(m); + insertOrReplace(stmt.zoom, stmt.x, stmt.y, stmt.data, stmt.isMerge); + }, + [](const PendingStatement& stmt) { return sizeof(PendingStatement) + stmt.data.size(); }) {} +// At most this much is buffered while SQLite is busy; beyond it, workers wait +// for the writer instead (a planet run otherwise buffered ~15M tiles, over 3 GB). +static const size_t MAX_PENDING_BYTES = 256 << 20; + +static size_t pendingCost(const std::string& data) { + return sizeof(PendingStatement) + data.size(); +} + MBTiles::~MBTiles() { if (db && inTransaction) db << "COMMIT;"; // commit all the changes if open } @@ -71,6 +86,7 @@ void MBTiles::flushPendingStatements() { while(!pendingStatements2->empty()) { const PendingStatement& stmt = pendingStatements2->back(); insertOrReplace(stmt.zoom, stmt.x, stmt.y, stmt.data, stmt.isMerge); + pendingBytes -= pendingCost(stmt.data); pendingStatements2->pop_back(); } @@ -80,19 +96,37 @@ void MBTiles::flushPendingStatements() { } void MBTiles::saveTile(int zoom, int x, int y, string *data, bool isMerge) { + if (batches.inBatch()) { + batches.add({zoom, x, y, *data, isMerge}); + return; + } + // If the lock is available, write directly to SQLite. if (m.try_lock()) { insertOrReplace(zoom, x, y, *data, isMerge); flushPendingStatements(); m.unlock(); - } else { + return; + } + + { // Else buffer the write for later, copying its binary blob. const std::lock_guard lock(pendingStatementsMutex); - pendingStatements1->push_back({zoom, x, y, *data, isMerge}); + if (pendingBytes < MAX_PENDING_BYTES) { + pendingBytes += pendingCost(*data); + pendingStatements1->push_back({zoom, x, y, *data, isMerge}); + return; + } } + + // The buffer is full: wait for SQLite, then help drain the buffer. + const std::lock_guard lock(m); + insertOrReplace(zoom, x, y, *data, isMerge); + flushPendingStatements(); } void MBTiles::closeForWriting() { + if (!batches.allWritten()) throw std::runtime_error("MBTiles: a batch of tiles was never written - please file an issue"); flushPendingStatements(); preparedStatements[0].used(true); preparedStatements[1].used(true); diff --git a/src/mmap_allocator.cpp b/src/mmap_allocator.cpp index 65818873..4daf0c00 100644 --- a/src/mmap_allocator.cpp +++ b/src/mmap_allocator.cpp @@ -1,9 +1,13 @@ #include "mmap_allocator.h" +#include +#include #include #include #include #include #include +#include +#include #include #include @@ -191,6 +195,25 @@ bool mmap_dir_t::is_open() { if(!mmap_dir_filename.empty()) { try { + // Unmapping and deleting a big store (planet: ~140 GB in ~300 files) + // takes half a minute on one thread, so release the files from several. + const size_t threads = std::min(files.size(), std::min(16u, std::max(1u, std::thread::hardware_concurrency()))); + if (threads > 1) { + std::atomic next(0); + std::vector workers; + for (size_t t = 0; t < threads; t++) { + try { + workers.emplace_back([this, &next]() { + for (size_t i; (i = next++) < files.size(); ) + files[i].reset(); + }); + } catch (const std::system_error &) { + break; // fewer threads; files.clear() below releases the rest + } + } + for (auto &worker : workers) + worker.join(); + } files.clear(); if(mmap_dir_created) { diff --git a/src/osm_store.cpp b/src/osm_store.cpp index 5ec7a81c..3bb086f2 100644 --- a/src/osm_store.cpp +++ b/src/osm_store.cpp @@ -6,6 +6,7 @@ #include #include +#include #include "node_store.h" #include "way_store.h" @@ -16,6 +17,42 @@ static inline bool isClosed(const std::vector& way) { return !way.empty() && way.front() == way.back(); } +// geom::within(inner, outer) for many inner rings against one big outer ring. +// Boost analyses the outer ring afresh on every call (a reservoir with 2500 +// islands and a 600,000-point shore takes 47 s); here its segments are indexed +// once. An inner ring that shares no point with the outer ring lies wholly +// inside or wholly outside it, so one of its vertices decides; when the rings +// touch or cross, Boost decides as before. +class RingWithinIndex { + using Segment = bg::model::segment; + using Value = std::pair; + const Ring &outer; + bg::index::rtree> segments; + + static std::vector segmentBoxes(const Ring &ring) { + std::vector values; + for (std::size_t i = 0; i + 1 < ring.size(); i++) + values.emplace_back(bg::return_envelope(Segment(ring[i], ring[i + 1])), i); + return values; + } + +public: + RingWithinIndex(const Ring &outer) : outer(outer), segments(segmentBoxes(outer)) { } + + bool within(const Ring &inner) const { + if (inner.size() < 4) return geom::within(inner, outer); + for (std::size_t j = 0; j + 1 < inner.size(); j++) { + const Segment s(inner[j], inner[j + 1]); + for (auto it = segments.qbegin(bg::index::intersects(bg::return_envelope(s))); it != segments.qend(); ++it) { + const std::size_t i = it->second; + if (bg::intersects(s, Segment(outer[i], outer[i + 1]))) + return geom::within(inner, outer); + } + } + return geom::within(inner.front(), outer); + } +}; + UsedObjects::UsedObjects(Status status): status(status), mutex(256), ids(256 * 1024) { } @@ -76,17 +113,31 @@ MultiPolygon OSMStore::wayListMultiPolygon(WayVec::const_iterator outerBegin, Wa // add all inners and outers to the multipolygon std::vector filledInners; + std::vector innerBoxes; for (auto it = inners.begin(); it != inners.end(); ++it) { Ring inner; fillPoints(inner, it->begin(), it->end()); + innerBoxes.emplace_back(geom::return_envelope(inner)); filledInners.emplace_back(inner); } bool onlyOneOuter = outers.size()==1; for (auto ot = outers.begin(); ot != outers.end(); ot++) { Polygon poly; fillPoints(poly.outer(), ot->begin(), ot->end()); - for (auto it = filledInners.begin(); it != filledInners.end(); ++it) { - if (onlyOneOuter || geom::within(*it, poly.outer())) { poly.inners().emplace_back(*it); } + const Box outerBox = geom::return_envelope(poly.outer()); + // Index big outer rings that many inner rings must be tested against + std::unique_ptr index; + if (!onlyOneOuter && filledInners.size() >= 16 && poly.outer().size() >= 4096) + index.reset(new RingWithinIndex(poly.outer())); + for (std::size_t i = 0; i < filledInners.size(); i++) { + const Ring &inner = filledInners[i]; + // A ring within the outer has its bounding box within the outer's, + // so the (costly) within test is only needed when it does + if (onlyOneOuter || + ((inner.empty() || geom::covered_by(innerBoxes[i], outerBox)) && + (index ? index->within(inner) : geom::within(inner, poly.outer())))) { + poly.inners().emplace_back(inner); + } } mp.emplace_back(move(poly)); } diff --git a/src/output_object.cpp b/src/output_object.cpp index 3d5703ca..bc4c1855 100644 --- a/src/output_object.cpp +++ b/src/output_object.cpp @@ -42,8 +42,8 @@ static void writeAttributesTo( FeatureBuilder& fbuilder, char zoom ) { - attributeStore.forEachPairUnsafe(attributes, [&](const AttributePair& pair) { - const AttributePair* it = &pair; + attributeStore.forEachPairIndexCanonicalUnsafe(attributes, [&](uint32_t pairIndex) { + const AttributePair* it = &attributeStore.pairStore.getPairUnsafe(pairIndex); if (it->minzoom > zoom) return; // TODO: consider taking a data view that is stable @@ -179,7 +179,7 @@ const uint32_t* MvtAttributeMemo::resolve(const AttributeStore& attributeStore, if (pairs.used >= MEMO_MAX_PAIRS) pairs.clear(); const uint32_t begin = entries.size(); - attributeStore.forEachPairIndexUnsafe(index, [&](uint32_t pairIndex) { + attributeStore.forEachPairIndexCanonicalUnsafe(index, [&](uint32_t pairIndex) { uint32_t key, value; if (const Table::Slot* s = pairs.find(pairIndex)) { key = s->a; diff --git a/src/pmtiles.cpp b/src/pmtiles.cpp index c9561b59..fa520f34 100644 --- a/src/pmtiles.cpp +++ b/src/pmtiles.cpp @@ -10,7 +10,9 @@ #include "helpers.h" TileOffset::TileOffset() { } -PMTiles::PMTiles() { } +PMTiles::PMTiles() : + batches([this](PendingTile &tile) { writeTile(tile); }, + [](const PendingTile &tile) { return tile.data.size(); }) { } PMTiles::~PMTiles() { } void PMTiles::open(std::string &filename) { @@ -24,6 +26,7 @@ void PMTiles::open(std::string &filename) { // Finish writing the .pmtiles file void PMTiles::close(std::string &metadata) { std::cout << "\nClosing pmtiles file" << std::flush; + if (!batches.allWritten()) { throw std::runtime_error("PMTiles: a batch of tiles was never written - please file an issue"); } // add all tiles to directories, writing leaf directories as we go std::vector rootEntries; @@ -129,34 +132,42 @@ void PMTiles::flushEntries(std::vector &rootEntries, std::vect rootEntries.emplace_back(rootEntry); } -// Write a tile to file and store it in the index -// - if the tile is small and has already been written, reuse that instead -// - we're a bit fussy about mutexs because compress_string is expensive +// Save a tile: compress it here (compress_string is expensive), but leave tiny tiles +// raw until they're written, since most of them (e.g. sea) repeat an earlier one void PMTiles::saveTile(int zoom, int x, int y, std::string &data) { - TileOffset offset; - bool isNew = false; - uint64_t tileId = pmtiles::zxy_to_tileid(zoom,x,y); - - // if it's a tiny tile (e.g. sea), see if we've written it already - std::unique_lock indexLock1(indexMutex); - if (data.size() lock(batches.writeMutex()); + writeTile(tile); +} - // otherwise, compress it and write +// Write a tile to file and store it in the index +// - if the tile is tiny and has already been written, reuse that instead +void PMTiles::writeTile(PendingTile &tile) { + TileOffset offset; + auto cached = tile.isTiny ? tinyCache.find(tile.data) : tinyCache.end(); + if (cached != tinyCache.end()) { + offset = cached->second; } else { - indexLock1.unlock(); - std::string compressed = compress_string(data, Z_DEFAULT_COMPRESSION, true); - std::lock_guard lock(fileMutex); - // write to file + std::string compressed = tile.isTiny ? compress_string(tile.data, Z_DEFAULT_COMPRESSION, true) : std::move(tile.data); offset = TileOffset(static_cast(outputStream.tellp()) - HEADER_ROOT, compressed.size()); outputStream.write(compressed.c_str(), compressed.size()); numTilesWritten++; - isNew = true; + // add to cache if tiny + if (tile.isTiny) { + if (tinyCache.size()>TINY_MAX_SIZE) tinyCache.clear(); + tinyCache.insert({ tile.data, offset }); + } } - + // store in index - std::lock_guard indexLock2(indexMutex); + uint64_t tileId = tile.tileId; numTilesAddressed++; if (isSparse) { sparseIndex[tileId] = offset; @@ -164,10 +175,4 @@ void PMTiles::saveTile(int zoom, int x, int y, std::string &data) { if (tileId >= denseIndex.size()) denseIndex.resize(tileId+10000, { 0, 0xffffff }); denseIndex[tileId] = offset; } - - // add to cache if tiny - if (isNew && data.size()TINY_MAX_SIZE) tinyCache.clear(); - tinyCache.insert({ data, offset }); - } } diff --git a/src/shp_mem_tiles.cpp b/src/shp_mem_tiles.cpp index e5c69e52..fd673e65 100644 --- a/src/shp_mem_tiles.cpp +++ b/src/shp_mem_tiles.cpp @@ -78,14 +78,58 @@ bool ShpMemTiles::mayIntersect(const std::string& layerName, const Box& box) con uint32_t y1 = latp2tiley(latp1, spatialIndexZoom); uint32_t y2 = latp2tiley(latp2, spatialIndexZoom); - for (int x = std::min(x1, x2); x <= std::min((1u << spatialIndexZoom) - 1u, std::max(x1, x2)); x++) { - for (int y = std::min(y1, y2); y <= std::min((1u << spatialIndexZoom) - 1u, std::max(y1, y2)); y++) { - uint32_t z6x = x / (1u << (spatialIndexZoom - CLUSTER_ZOOM)); - uint32_t z6y = y / (1u << (spatialIndexZoom - CLUSTER_ZOOM)); + const uint32_t xMin = std::min(x1, x2), xMax = std::min((1u << spatialIndexZoom) - 1u, std::max(x1, x2)); + const uint32_t yMin = std::min(y1, y2), yMax = std::min((1u << spatialIndexZoom) - 1u, std::max(y1, y2)); + const uint32_t divisor = 1u << (spatialIndexZoom - CLUSTER_ZOOM); + + // First pass, without any geometry checks: is a tile in the box already known to + // intersect, and how many tiles still need the exact check below? + size_t unchecked = 0; + for (uint32_t x = xMin; x <= xMax; x++) { + for (uint32_t y = yMin; y <= yMax; y++) { + uint32_t z6x = x / divisor, z6y = y / divisor; + auto& bitvec = sparseLayerVector[z6x * CLUSTER_ZOOM_WIDTH + z6y]; + uint64_t index = 2u * ((x - z6x * divisor) * divisor + (y - z6y * divisor)); + if (!bitvec.empty() && bitvec[index]) { + if (bitvec[index + 1]) return true; + unchecked++; + } + } + } + if (unchecked == 0) return false; + + // Each tile's exact check intersects the tile with every candidate geometry, so its + // cost grows with the size of those geometries. A large query box that lies inside + // the bounding box of a huge geometry without touching it (say, an offshore marine + // reserve inside a country's bounding box) would check every one of thousands of + // tiles, and find nothing. Instead, check the query box once: if no geometry + // intersects it, none intersects any tile in it. Otherwise the caller's exact check + // decides. + const size_t maxTileChecks = 16; + if (unchecked > maxTileChecks) { + Box queryBox = box; + return !QueryMatchingGeometries( + layerName, + true, + queryBox, + [&](const RTree &rtree) { // indexQuery + vector results; + rtree.query(geom::index::intersects(queryBox), back_inserter(results)); + return results; + }, + [&](OutputObject const &oo) { // checkQuery + return geom::intersects(queryBox, retrieveMultiPolygon(oo.objectID)); + } + ).empty(); + } + + for (uint32_t x = xMin; x <= xMax; x++) { + for (uint32_t y = yMin; y <= yMax; y++) { + uint32_t z6x = x / divisor; + uint32_t z6y = y / divisor; auto& bitvec = sparseLayerVector[z6x * CLUSTER_ZOOM_WIDTH + z6y]; - uint32_t divisor = 1u << (spatialIndexZoom - CLUSTER_ZOOM); uint64_t index = 2u * ((x - z6x * divisor) * divisor + (y - z6y * divisor)); if (!bitvec.empty() && bitvec[index]) { @@ -137,7 +181,8 @@ void ShpMemTiles::StoreGeometry( const std::string& name, uint minzoom, int32_t score, - AttributeIndex attrIdx + AttributeIndex attrIdx, + bool knownValid ) { geom::model::box box; @@ -175,9 +220,9 @@ void ShpMemTiles::StoreGeometry( case POLYGON_: { - // Checked once here, so tiles can clip valid polygons on the fast path + // Checked once here (unless the caller has), so tiles can clip valid polygons on the fast path const MultiPolygon &mp = boost::get(geometry); - NodeID oid = storeMultiPolygon(mp, multiPolygonDefinitelyValid(mp) || geom::is_valid(mp)); + NodeID oid = storeMultiPolygon(mp, knownValid || multiPolygonDefinitelyValid(mp) || geom::is_valid(mp)); oo = std::make_shared(geomType, layerNum, oid, attrIdx, minzoom); std::vector oolist { *oo }; addGeometryToIndex(boost::get(geometry), oolist, 0); diff --git a/src/shp_processor.cpp b/src/shp_processor.cpp index f3b3c4f2..b32291b2 100644 --- a/src/shp_processor.cpp +++ b/src/shp_processor.cpp @@ -366,8 +366,8 @@ void ShpProcessor::processShapeGeometry(SHPObject* shape, AttributeIndex attrIdx // Make valid if needs be string reason; - const bool precheckValid = multiPolygonDefinitelyValid(multi); - if (!precheckValid && !geom::is_valid(multi, reason)) { + const bool valid = multiPolygonDefinitelyValid(multi) || geom::is_valid(multi, reason); + if (!valid) { if (verbose) cerr << "Shapefile entity " << shape->nShapeId << " type " << shapeType << " is invalid. Parts:" << shape->nParts << ". Reason:" << reason; make_valid(multi); if (verbose) { @@ -376,11 +376,17 @@ void ShpProcessor::processShapeGeometry(SHPObject* shape, AttributeIndex attrIdx cerr << endl; } } - // clip to bounding box - MultiPolygon out; - geom::intersection(multi, clippingBox, out); - if (boost::size(out)>0) { - shpMemTiles.StoreGeometry(layerNum, layer.name, POLYGON_, out, layer.indexed, hasName, name, minzoom, score, attrIdx); + // clip to bounding box (a valid polygon wholly inside it is stored as it is) + Box envelope; + geom::envelope(multi, envelope); + if (valid && geom::covered_by(envelope, clippingBox)) { + shpMemTiles.StoreGeometry(layerNum, layer.name, POLYGON_, multi, layer.indexed, hasName, name, minzoom, score, attrIdx, true); + } else { + MultiPolygon out; + geom::intersection(multi, clippingBox, out); + if (boost::size(out)>0) { + shpMemTiles.StoreGeometry(layerNum, layer.name, POLYGON_, out, layer.indexed, hasName, name, minzoom, score, attrIdx); + } } } else { diff --git a/src/tile_coordinates_set.cpp b/src/tile_coordinates_set.cpp index e6d6e78e..7a25a1ba 100644 --- a/src/tile_coordinates_set.cpp +++ b/src/tile_coordinates_set.cpp @@ -40,6 +40,12 @@ void TileCoordinatesSet::forEachSet(const std::function> 12] |= (uint64_t)1 << ((w >> 6) & 63); } +// Fills each row of the box word by word, rather than tile by tile. +void PreciseTileCoordinatesSet::setBox(TileCoordinate minX, TileCoordinate maxX, TileCoordinate minY, TileCoordinate maxY) { + const uint64_t dim = 1ull << zoom_; + if (minX > maxX || minY > maxY || minX >= dim || minY >= dim) + return; + maxX = std::min(maxX, dim - 1); + maxY = std::min(maxY, dim - 1); + for (uint64_t x = minX; x <= maxX; x++) { + const uint64_t first = x * dim + minY, last = x * dim + maxY; + const uint64_t w0 = first >> 6, w1 = last >> 6; + for (uint64_t w = w0; w <= w1; w++) { + uint64_t mask = ~0ull; + if (w == w0) mask &= ~0ull << (first & 63); + if (w == w1) mask &= ~0ull >> (63 - (last & 63)); + tiles[w] |= mask; + } + for (uint64_t block = w0 >> 6; block <= (w1 >> 6); block++) + summary[block >> 6] |= (uint64_t)1 << (block & 63); + } +} + +void PreciseTileCoordinatesSet::merge(const PreciseTileCoordinatesSet& other) { + if (other.zoom_ != zoom_) + throw std::invalid_argument("PreciseTileCoordinatesSet::merge: zooms differ"); + forEachMarkedWord(other.summary, numWords, [&](size_t w) { tiles[w] |= other.tiles[w]; }); + for (size_t s = 0; s < summary.size(); s++) + summary[s] |= other.summary[s]; +} + void PreciseTileCoordinatesSet::forEachSet(const std::function& f) const { // loc = x * 2^zoom + y, so ascending loc order is x-major, y-minor. // Only blocks marked in the summary can hold set bits; visiting them in @@ -134,3 +169,24 @@ void LossyTileCoordinatesSet::set(TileCoordinate x, TileCoordinate y) { throw std::logic_error("LossyTileCoordinatesSet::set() is not implemented; LossyTileCoordinatesSet is read-only"); } +void LossyTileCoordinatesSet::forEachSet(const std::function& f) const { + // Expand the underlying set instead of testing all 4^zoom tiles (2^34 at z17). + // Its tiles arrive x-major, y-minor; one column of them becomes `scale` + // columns here, which keeps this set's order x-major, y-minor too. + std::vector column; + TileCoordinate columnX = 0; + auto flush = [&]() { + for (TileCoordinate dx = 0; dx < scale; dx++) + for (TileCoordinate y : column) + for (TileCoordinate dy = 0; dy < scale; dy++) + f(columnX * scale + dx, y * scale + dy); + column.clear(); + }; + tiles.forEachSet([&](TileCoordinate x, TileCoordinate y) { + if (!column.empty() && x != columnX) flush(); + columnX = x; + column.push_back(y); + }); + if (!column.empty()) flush(); +} + diff --git a/src/tile_data.cpp b/src/tile_data.cpp index 89b367e0..c1fcc02e 100644 --- a/src/tile_data.cpp +++ b/src/tile_data.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -6,6 +7,7 @@ #include "tile_data.h" #include "coordinates_geom.h" #include "leased_store.h" +#include "stable_hash.h" #include using namespace std; @@ -166,6 +168,13 @@ void TileDataSource::collectTilesWithObjectsAtZoom(std::vector(indexZoom, objectsWithIds.begin(), objectsWithIds.size(), zooms); } +void TileDataSource::collectTilesWithObjectsInShard(size_t shard, std::vector>& zooms) { + if (shard < objects.size()) + collectTilesWithObjectsInShardTemplate(indexZoom, objects[shard], shard, zooms); + if (shard < objectsWithIds.size()) + collectTilesWithObjectsInShardTemplate(indexZoom, objectsWithIds[shard], shard, zooms); +} + void addCoveredTilesToOutput(const uint indexZoom, std::vector>& zooms, const Box& box) { size_t maxZoom = zooms.size() - 1; // std::cout << "addCoveredTilesToOutput maxZoom=" << maxZoom << ", indexZoom - maxZoom = " << (indexZoom - maxZoom) << std::endl; @@ -174,16 +183,11 @@ void addCoveredTilesToOutput(const uint indexZoom, std::vector= 0; zoom--) { - zooms[zoom]->set(zx, zy); - zx /= 2; - zy /= 2; - } - } + // The parents of the tiles in the box are the tiles in the box scaled down + for (int zoom = maxZoom; zoom >= 0; zoom--) { + zooms[zoom]->setBox(minx, maxx, miny, maxy); + minx /= 2; maxx /= 2; + miny /= 2; maxy /= 2; } } @@ -385,7 +389,13 @@ Geometry TileDataSource::buildWayGeometry(OutputGeometryType const geomType, } MultiPolygon mp; - if (cachedClip == nullptr) + // rect_clip below needs the input as it was before fast_clip. For a large valid + // polygon, keep it rather than load it from the store again for every tile. + std::size_t inputPoints = 0; + if (cachedClip == nullptr && inputKnownValid) + for (auto const &p : uncached) inputPoints += geom::num_points(p); + const bool keepUncached = inputPoints > 1000; + if (cachedClip == nullptr && !keepUncached) mp = std::move(uncached); else geom::assign(mp, input); @@ -405,9 +415,9 @@ Geometry TileDataSource::buildWayGeometry(OutputGeometryType const geomType, // instead, which is much cheaper than the is_valid/intersection/repair // path below; rect_clip bails out on anything it can't handle. MultiPolygon original; - if (cachedClip == nullptr) populateMultiPolygon(original, objectID); + if (cachedClip == nullptr && !keepUncached) populateMultiPolygon(original, objectID); MultiPolygon clipped; - if (rect_clip(cachedClip == nullptr ? original : input, box, clipped) && + if (rect_clip(cachedClip == nullptr && !keepUncached ? original : input, box, clipped) && (multiPolygonDefinitelyValid(clipped) || geom::is_valid(clipped))) { multiPolygonClipCache.add(bbox, objectID, clipped, true); return clipped; @@ -498,35 +508,172 @@ void TileDataSource::reportSize() const { void populateTilesAtZoom( const std::vector& sources, - std::vector>& zooms + std::vector>& zooms, + size_t threadNum ) { if (zooms.size() > 15) throw std::out_of_range("populateTilesAtZoom: expected at most z14 zooms (15), but found " + std::to_string(zooms.size()) + " vectors"); - for(size_t i=0; icollectTilesWithObjectsAtZoom(zooms); - sources[i]->collectTilesWithLargeObjectsAtZoom(zooms); + std::vector precise; + for (const auto& zoom : zooms) + precise.push_back(dynamic_cast(zoom.get())); + const bool allPrecise = std::find(precise.begin(), precise.end(), nullptr) == precise.end(); + + if (threadNum <= 1 || !allPrecise) { + for (size_t i = 0; i < sources.size(); i++) + sources[i]->collectTilesWithObjectsAtZoom(zooms); + } else { + // Objects are kept per z6 tile. Each worker takes whole columns of z6 + // tiles (so that it touches only its part of the bitmaps), sets bits in + // bitmaps of its own, and these are merged at the end. + std::atomic nextColumn(0); + std::vector>> local(threadNum); + boost::asio::thread_pool pool(threadNum); + for (size_t t = 0; t < threadNum; t++) { + boost::asio::post(pool, [&, t]() { + auto& mine = local[t]; + for (size_t zoom = 0; zoom < zooms.size(); zoom++) + mine.push_back(std::make_shared(zoom)); + for (size_t column; (column = nextColumn++) < CLUSTER_ZOOM_WIDTH; ) + for (size_t row = 0; row < CLUSTER_ZOOM_WIDTH; row++) + for (const auto source : sources) + source->collectTilesWithObjectsInShard(column * CLUSTER_ZOOM_WIDTH + row, mine); + }); + } + pool.join(); + for (const auto& mine : local) + for (size_t zoom = 0; zoom < mine.size(); zoom++) + precise[zoom]->merge(static_cast(*mine[zoom])); } + + for (size_t i = 0; i < sources.size(); i++) + sources[i]->collectTilesWithLargeObjectsAtZoom(zooms); } void sortOutputObjectIDs( const std::vector& sortOrders, + const AttributeStore& attributeStore, std::vector& data ); std::vector TileDataSource::getObjectsForTile( const std::vector& sortOrders, + const AttributeStore& attributeStore, unsigned int zoom, TileCoordinates coordinates ) { std::vector data; collectObjectsForTile(zoom, coordinates, data); collectLargeObjectsForTile(zoom, coordinates, data); - sortOutputObjectIDs(sortOrders, data); + sortOutputObjectIDs(sortOrders, attributeStore, data); data.erase(unique(data.begin(), data.end()), data.end()); + orderStoredGeometryTies(data); return data; } +namespace { +struct GeometryTieKey { + double x, y; // first point + uint64_t points; // number of points + uint64_t hash; // of all points, only computed when the above are equal + OutputObjectID o; +}; + +// Adds a ring/linestring to the key: first point and point count, or (withHash) the hash of all points +template +void addToTieKey(GeometryTieKey& key, const R& ring, bool withHash) { + if (withHash) { + key.hash = stableCombine(key.hash, static_cast(ring.size())); + for (const auto& p : ring) { + key.hash = stableCombine(key.hash, static_cast(p.x())); + key.hash = stableCombine(key.hash, static_cast(p.y())); + } + return; + } + if (key.points == 0 && !ring.empty()) { + key.x = ring.front().x(); + key.y = ring.front().y(); + } + key.points += ring.size(); +} + +void geometryTieKey(const TileDataSource& source, GeometryTieKey& key, bool withHash) { + const NodeID objectID = key.o.oo.objectID; + switch (key.o.oo.geomType) { + case POINT_: { + const Point& p = source.retrievePoint(objectID); + addToTieKey(key, std::array{p}, withHash); + break; + } + case LINESTRING_: + addToTieKey(key, source.retrieveLinestring(objectID), withHash); + break; + case MULTILINESTRING_: + for (const auto& ls : source.retrieveMultiLinestring(objectID)) + addToTieKey(key, ls, withHash); + break; + case POLYGON_: + for (const auto& poly : source.retrieveMultiPolygon(objectID)) { + addToTieKey(key, poly.outer(), withHash); + for (const auto& inner : poly.inners()) + addToTieKey(key, inner, withHash); + } + break; + } +} +} + +// Geometries stored at runtime (relations, shapefiles, ...) have an objectID that +// depends on which thread stored them, so the sort ties objects that share layer, +// z_order, type, attributes and id in an arbitrary order. Order such runs by +// geometry instead. OSM ways and nodes carry their OSM id in the objectID, and +// sort after stored geometries, so they never need this. +void TileDataSource::orderStoredGeometryTies(std::vector& data) const { + const NodeID storedLimit = 1ull << TILE_DATA_ID_SIZE; + auto tied = [](const OutputObjectID& a, const OutputObjectID& b) { + return a.oo.layer == b.oo.layer && a.oo.z_order == b.oo.z_order && a.oo.geomType == b.oo.geomType && + a.oo.attributes == b.oo.attributes && a.id == b.id; + }; + + std::vector keys; + size_t i = 0; + while (i < data.size()) { + size_t j = i + 1; + if (data[i].oo.objectID < storedLimit) + while (j < data.size() && data[j].oo.objectID < storedLimit && tied(data[i], data[j])) j++; + if (j - i < 2) { i = j; continue; } + + keys.clear(); + for (size_t k = i; k < j; k++) { + keys.push_back(GeometryTieKey{0, 0, 0, 0, data[k]}); + geometryTieKey(*this, keys.back(), false); + } + auto byPoint = [](const GeometryTieKey& a, const GeometryTieKey& b) { + if (a.x != b.x) return a.x < b.x; + if (a.y != b.y) return a.y < b.y; + return a.points < b.points; + }; + std::sort(keys.begin(), keys.end(), byPoint); + + // Still tied: hash all points (identical geometries may stay in any order) + for (size_t a = 0; a < keys.size(); ) { + size_t b = a + 1; + while (b < keys.size() && !byPoint(keys[a], keys[b])) b++; + if (b - a > 1) { + for (size_t k = a; k < b; k++) geometryTieKey(*this, keys[k], true); + std::sort(keys.begin() + a, keys.begin() + b, [](const GeometryTieKey& p, const GeometryTieKey& q) { + if (p.hash != q.hash) return p.hash < q.hash; + return p.o.oo.objectID < q.o.oo.objectID; + }); + } + a = b; + } + + for (size_t k = i; k < j; k++) data[k] = keys[k - i].o; + i = j; + } +} + // ------------------------------------ // Add geometries to tile/large indices @@ -660,7 +807,7 @@ NodeID TileDataSource::storePoint(const Point& input) { const auto& store = pointStore.get(this); NodeID offset = store.second->size(); - store.second->emplace_back(input); + store.second->push_back(input); NodeID rv = (store.first << (TILE_DATA_ID_SIZE - shardBits)) + offset; return rv; } diff --git a/src/tile_sorting.cpp b/src/tile_sorting.cpp index 64588163..cc79e20d 100644 --- a/src/tile_sorting.cpp +++ b/src/tile_sorting.cpp @@ -208,26 +208,75 @@ template void sortOutputObjects( typename AppendVectorNS::AppendVector::Iterator end ); +namespace { +struct AttributeKeyCacheEntry { + AttributeIndex index; + uint64_t key; +}; +// Direct-mapped, per thread: neighbouring tiles share most attribute sets +const size_t ATTRIBUTE_KEY_CACHE_SIZE = 1 << 12; +thread_local std::vector tlsAttributeKeys; +thread_local const AttributeStore* tlsAttributeKeysStore = nullptr; + +struct KeyedOutputObjectID { + uint64_t attributeKey; + OutputObjectID o; +}; + +uint64_t attributeKey(const AttributeStore& attributeStore, AttributeIndex index) { + if (tlsAttributeKeysStore != &attributeStore) { + tlsAttributeKeys.assign(ATTRIBUTE_KEY_CACHE_SIZE, AttributeKeyCacheEntry{~0u, 0}); + tlsAttributeKeysStore = &attributeStore; + } + uint32_t slot = index; + slot ^= slot >> 16; slot *= 0x7feb352dU; slot ^= slot >> 15; + AttributeKeyCacheEntry& entry = tlsAttributeKeys[slot & (ATTRIBUTE_KEY_CACHE_SIZE - 1)]; + if (entry.index != index) + entry = AttributeKeyCacheEntry{index, attributeStore.contentHashUnsafe(index)}; + return entry.key; +} +} + void sortOutputObjectIDs( const std::vector& sortOrders, + const AttributeStore& attributeStore, std::vector& data ) { - // Lexicographic comparison, with the order of: layer, geomType, attributes, and objectID. + // Lexicographic comparison, with the order of: layer, geomType, attributes, id and objectID. // Note that attributes is preferred to objectID. // It is to arrange objects with the identical attributes continuously. // Such objects will be merged into one object, to reduce the size of output. - boost::sort::pdqsort(data.begin(), data.end(), [&sortOrders](const OutputObjectID& x, const OutputObjectID& y) -> bool { - if (x.oo.layer < y.oo.layer) return true; - if (x.oo.layer > y.oo.layer) return false; - if (x.oo.z_order < y.oo.z_order) return sortOrders[x.oo.layer]; - if (x.oo.z_order > y.oo.z_order) return !sortOrders[x.oo.layer]; - if (x.oo.geomType < y.oo.geomType) return true; - if (x.oo.geomType > y.oo.geomType) return false; - if (x.oo.attributes < y.oo.attributes) return true; - if (x.oo.attributes > y.oo.attributes) return false; - if (x.oo.objectID < y.oo.objectID) return true; - return false; + // + // Attribute sets are compared by content, not by AttributeIndex: the index is + // handed out in insertion order, which depends on thread scheduling. The same + // goes for the objectID of stored geometries, so the OSM id goes first; + // TileDataSource::orderStoredGeometryTies handles the remaining ties. + std::vector keyed; + keyed.reserve(data.size()); + for (const OutputObjectID& o : data) + keyed.push_back(KeyedOutputObjectID{attributeKey(attributeStore, o.oo.attributes), o}); + + boost::sort::pdqsort(keyed.begin(), keyed.end(), [&sortOrders, &attributeStore](const KeyedOutputObjectID& a, const KeyedOutputObjectID& b) -> bool { + const OutputObject& x = a.o.oo; + const OutputObject& y = b.o.oo; + if (x.layer < y.layer) return true; + if (x.layer > y.layer) return false; + if (x.z_order < y.z_order) return sortOrders[x.layer]; + if (x.z_order > y.z_order) return !sortOrders[x.layer]; + if (x.geomType < y.geomType) return true; + if (x.geomType > y.geomType) return false; + if (x.attributes != y.attributes) { + if (a.attributeKey != b.attributeKey) return a.attributeKey < b.attributeKey; + const int c = attributeStore.compareContentUnsafe(x.attributes, y.attributes); + if (c != 0) return c < 0; + return x.attributes < y.attributes; + } + if (a.o.id != b.o.id) return a.o.id < b.o.id; + return x.objectID < y.objectID; }); + + for (size_t i = 0; i < keyed.size(); i++) + data[i] = keyed[i].o; } void sortTileCoordinates( diff --git a/src/tilemaker.cpp b/src/tilemaker.cpp index 7a7ae1e5..bdecc1d4 100644 --- a/src/tilemaker.cpp +++ b/src/tilemaker.cpp @@ -97,11 +97,19 @@ int main(const int argc, const char* argv[]) { #ifdef __GLIBC__ // glibc raises its mmap threshold dynamically (up to 32/64 MB) once a large // mmapped chunk is freed; multi-MB buffers then land in per-thread arenas - // that are never trimmed, inflating peak RSS. Setting the trim threshold - // explicitly also disables that dynamic adjustment (the mmap threshold - // stays at its 128 KB default), unless the user tunes malloc via the environment. - if (!getenv("MALLOC_TRIM_THRESHOLD_") && !getenv("MALLOC_MMAP_THRESHOLD_") && !getenv("GLIBC_TUNABLES")) { - mallopt(M_TRIM_THRESHOLD, 4 << 20); + // that are never trimmed, inflating peak RSS. Setting malloc parameters + // explicitly turns that dynamic adjustment off, so both are set here: + // - a 16 MB top pad makes the heap grow and shrink in large steps; with many + // threads allocating, the default 128 KB steps cost many brk() calls and + // page faults + // - a fixed 8 MB mmap threshold: otherwise it would stay at 128 KB, and every + // larger buffer would be a fresh mmap with zeroed pages. Boost's validity + // check on a huge polygon grows such buffers over and over (one relation + // with 1.9M points took 3x as long). + // Users who tune malloc via the environment keep their settings. + if (!getenv("MALLOC_TRIM_THRESHOLD_") && !getenv("MALLOC_MMAP_THRESHOLD_") && !getenv("MALLOC_TOP_PAD_") && !getenv("GLIBC_TUNABLES")) { + mallopt(M_TOP_PAD, 16 << 20); + mallopt(M_MMAP_THRESHOLD, 8 << 20); } #endif @@ -446,7 +454,7 @@ int main(const int argc, const char* argv[]) { clock_gettime(CLOCK_MONOTONIC, &start); #endif std::cout << "collecting tiles" << std::flush; - populateTilesAtZoom(sources, zoomResults); + populateTilesAtZoom(sources, zoomResults, options.threadNum); #ifdef CLOCK_MONOTONIC clock_gettime(CLOCK_MONOTONIC, &end); uint64_t tileNs = 1e9 * (end.tv_sec - start.tv_sec) + end.tv_nsec - start.tv_nsec; @@ -504,8 +512,15 @@ int main(const int argc, const char* argv[]) { std::atomic batchCounter(0); const uint cutZoom = std::max(12, config.startZoom); - std::size_t batchSize = 0; - for(std::size_t startIndex = 0; startIndex < tileCoordinates.size(); startIndex += batchSize) { + // Batches are handed out as threads become free rather than all posted up + // front: a planet has ~14M of them, and their queued handlers held GBs. + std::mutex batchMutex; + std::size_t nextBatchStart = 0; + uint64_t nextBatchSeq = 0; + auto nextBatch = [&](std::size_t &startIndex, std::size_t &batchSize, uint64_t &seq) { + const std::lock_guard lock(batchMutex); + startIndex = nextBatchStart; + seq = nextBatchSeq++; // Compute how many tiles should be assigned to this batch -- // higher-zoom tiles are cheaper to compute, lower-zoom tiles more expensive. batchSize = 0; @@ -529,65 +544,89 @@ int main(const int argc, const char* argv[]) { batchSize++; } + nextBatchStart += batchSize; + return batchSize > 0; + }; - boost::asio::post(pool, [=, &tileCoordinates, &pool, &sharedData, &sources, &attributeStore, &io_mutex, &tilesWritten, &lastTilesWritten, &batchCounter]() { - clipCacheBatchId = ++batchCounter; - // A single-tile batch has no descendants to reuse its clips. - clipCacheNoStore = batchSize == 1; - std::vector tileTimings; - std::size_t endIndex = std::min(tileCoordinates.size(), startIndex + batchSize); - for(std::size_t i = startIndex; i < endIndex; ++i) { - unsigned int zoom = tileCoordinates[i].first; - TileCoordinates coords = tileCoordinates[i].second; + // .pmtiles and .mbtiles are written in batch order, so the file doesn't depend on thread scheduling + const bool toPMTiles = options.outputMode == OptionsParser::OutputMode::PMTiles; + const bool toMBTiles = options.outputMode == OptionsParser::OutputMode::MBTiles; + // While an earlier batch is still being worked on, each thread may run ahead by about + // 64 MB of finished tiles. With less, threads often wait for a slow batch (mostly a + // dense z12 area), above all when two threads share a core and each runs at half speed. + const uint64_t maxHeldBytes = (64ull << 20) * options.threadNum; + if (toPMTiles) sharedData.pmtiles.batches.setMaxHeldBytes(maxHeldBytes); + if (toMBTiles) sharedData.mbtiles.batches.setMaxHeldBytes(maxHeldBytes); + for (unsigned int t = 0; t < options.threadNum; t++) { + boost::asio::post(pool, [&]() { + std::size_t startIndex, batchSize; + uint64_t seq; + while (true) { + if (toPMTiles) sharedData.pmtiles.batches.waitForRoom(); + if (toMBTiles) sharedData.mbtiles.batches.waitForRoom(); + if (!nextBatch(startIndex, batchSize, seq)) break; + if (toPMTiles) sharedData.pmtiles.batches.begin(seq); + if (toMBTiles) sharedData.mbtiles.batches.begin(seq); + clipCacheBatchId = ++batchCounter; + // A single-tile batch has no descendants to reuse its clips. + clipCacheNoStore = batchSize == 1; + std::vector tileTimings; + std::size_t endIndex = std::min(tileCoordinates.size(), startIndex + batchSize); + for(std::size_t i = startIndex; i < endIndex; ++i) { + unsigned int zoom = tileCoordinates[i].first; + TileCoordinates coords = tileCoordinates[i].second; #ifdef CLOCK_MONOTONIC - timespec start, end; - if (options.logTileTimings) - clock_gettime(CLOCK_MONOTONIC, &start); + timespec start, end; + if (options.logTileTimings) + clock_gettime(CLOCK_MONOTONIC, &start); #endif - std::vector> data; - for (auto source : sources) { - data.emplace_back(source->getObjectsForTile(sortOrders, zoom, coords)); - } - outputProc(sharedData, sources, attributeStore, data, coords, zoom); + std::vector> data; + for (auto source : sources) { + data.emplace_back(source->getObjectsForTile(sortOrders, attributeStore, zoom, coords)); + } + outputProc(sharedData, sources, attributeStore, data, coords, zoom); #ifdef CLOCK_MONOTONIC - if (options.logTileTimings) { - clock_gettime(CLOCK_MONOTONIC, &end); - uint64_t tileNs = 1e9 * (end.tv_sec - start.tv_sec) + end.tv_nsec - start.tv_nsec; - std::string output = "z" + std::to_string(zoom) + "/" + std::to_string(coords.x) + "/" + std::to_string(coords.y) + " took " + std::to_string(tileNs/1e6) + " ms"; - tileTimings.push_back(output); - } + if (options.logTileTimings) { + clock_gettime(CLOCK_MONOTONIC, &end); + uint64_t tileNs = 1e9 * (end.tv_sec - start.tv_sec) + end.tv_nsec - start.tv_nsec; + std::string output = "z" + std::to_string(zoom) + "/" + std::to_string(coords.x) + "/" + std::to_string(coords.y) + " took " + std::to_string(tileNs/1e6) + " ms"; + tileTimings.push_back(output); + } #endif - } + } - if (options.logTileTimings) { - const std::lock_guard lock(io_mutex); - std::cout << std::endl; - for (const auto& output : tileTimings) - std::cout << output << std::endl; - } + if (options.logTileTimings) { + const std::lock_guard lock(io_mutex); + std::cout << std::endl; + for (const auto& output : tileTimings) + std::cout << output << std::endl; + } - tilesWritten += (endIndex - startIndex); - - if (io_mutex.try_lock()) { - uint64_t written = tilesWritten.load(); - - if (written >= lastTilesWritten + tileCoordinates.size() / 100 || ISATTY) { - lastTilesWritten = written; - // Show progress grouped by z6 (or lower) - size_t z = tileCoordinates[startIndex].first; - size_t x = tileCoordinates[startIndex].second.x; - size_t y = tileCoordinates[startIndex].second.y; - if (z > CLUSTER_ZOOM) { - x = x / (1 << (z - CLUSTER_ZOOM)); - y = y / (1 << (z - CLUSTER_ZOOM)); - z = CLUSTER_ZOOM; + if (toPMTiles) sharedData.pmtiles.batches.end(); + if (toMBTiles) sharedData.mbtiles.batches.end(); + tilesWritten += (endIndex - startIndex); + + if (io_mutex.try_lock()) { + uint64_t written = tilesWritten.load(); + + if (written >= lastTilesWritten + tileCoordinates.size() / 100 || ISATTY) { + lastTilesWritten = written; + // Show progress grouped by z6 (or lower) + size_t z = tileCoordinates[startIndex].first; + size_t x = tileCoordinates[startIndex].second.x; + size_t y = tileCoordinates[startIndex].second.y; + if (z > CLUSTER_ZOOM) { + x = x / (1 << (z - CLUSTER_ZOOM)); + y = y / (1 << (z - CLUSTER_ZOOM)); + z = CLUSTER_ZOOM; + } + cout << "z" << z << "/" << x << "/" << y << ", writing tile " << written << " of " << tileCoordinates.size() << " \r" << std::flush; } - cout << "z" << z << "/" << x << "/" << y << ", writing tile " << written << " of " << tileCoordinates.size() << " \r" << std::flush; + io_mutex.unlock(); } - io_mutex.unlock(); } }); } diff --git a/test/append_vector.test.cpp b/test/append_vector.test.cpp index b3104428..f0d5d704 100644 --- a/test/append_vector.test.cpp +++ b/test/append_vector.test.cpp @@ -21,6 +21,12 @@ MU_TEST(test_append_vector) { mu_check(vec[25] == 25); + const AppendVector& constVec = vec; + mu_check(constVec[25] == 25); + mu_check(constVec[8191] == 8191); + mu_check(constVec[8192] == 8192); + mu_check(constVec[9999] == 9999); + const AppendVector::Iterator& it = vec.begin(); mu_check(*it == 0); mu_check(*(it + 1) == 1); diff --git a/test/correct.test.cpp b/test/correct.test.cpp new file mode 100644 index 00000000..7178d173 --- /dev/null +++ b/test/correct.test.cpp @@ -0,0 +1,72 @@ +#include +#include +#include +#include "external/minunit.h" +#include "geometry/correct.hpp" + +typedef boost::geometry::model::d2::point_xy point_t; +typedef boost::geometry::model::polygon polygon_t; +typedef boost::geometry::model::multi_polygon multi_polygon_t; + +static bool near(double a, double b) { return std::abs(a - b) < 1e-9; } + +MU_TEST(test_correct_merges_overlapping_holes) { + // Two overlapping holes are merged into one; a disjoint hole is kept as is + polygon_t input; + boost::geometry::read_wkt( + "POLYGON((0 0,0 10,10 10,10 0,0 0)," + "(1 1,3 1,3 3,1 3,1 1),(2 2,4 2,4 4,2 4,2 2),(6 6,7 6,7 7,6 7,6 6))", input); + multi_polygon_t output; + geometry::correct(input, output); + + mu_check(output.size() == 1); + mu_check(output[0].inners().size() == 2); + mu_check(near(boost::geometry::area(output), 100 - 7 - 1)); + mu_check(boost::geometry::is_valid(output)); +} + +MU_TEST(test_correct_merges_overlapping_outers) { + multi_polygon_t input; + boost::geometry::read_wkt( + "MULTIPOLYGON(((0 0,0 2,2 2,2 0,0 0)),((1 1,1 3,3 3,3 1,1 1)),((5 5,5 6,6 6,6 5,5 5)))", input); + multi_polygon_t output; + geometry::correct(input, output); + + mu_check(output.size() == 2); + mu_check(near(boost::geometry::area(output), 4 + 4 - 1 + 1)); + mu_check(boost::geometry::is_valid(output)); +} + +MU_TEST(test_correct_many_disjoint_holes) { + // 400 disjoint holes, e.g. a lake with many islands: all are kept + std::string wkt = "POLYGON((0 0,0 100,100 100,100 0,0 0)"; + for (int x = 0; x < 20; x++) + for (int y = 0; y < 20; y++) { + double x0 = 1 + x * 5, y0 = 1 + y * 5; + wkt += ",(" + std::to_string(x0) + " " + std::to_string(y0) + "," + std::to_string(x0 + 2) + " " + std::to_string(y0) + "," + + std::to_string(x0 + 2) + " " + std::to_string(y0 + 2) + "," + std::to_string(x0) + " " + std::to_string(y0 + 2) + "," + + std::to_string(x0) + " " + std::to_string(y0) + ")"; + } + wkt += ")"; + polygon_t input; + boost::geometry::read_wkt(wkt, input); + multi_polygon_t output; + geometry::correct(input, output); + + mu_check(output.size() == 1); + mu_check(output[0].inners().size() == 400); + mu_check(near(boost::geometry::area(output), 100 * 100 - 400 * 4)); + mu_check(boost::geometry::is_valid(output)); +} + +MU_TEST_SUITE(test_suite_correct) { + MU_RUN_TEST(test_correct_merges_overlapping_holes); + MU_RUN_TEST(test_correct_merges_overlapping_outers); + MU_RUN_TEST(test_correct_many_disjoint_holes); +} + +int main() { + MU_RUN_SUITE(test_suite_correct); + MU_REPORT(); + return MU_EXIT_CODE; +} diff --git a/test/geom.test.cpp b/test/geom.test.cpp index e33b5990..2a31d97d 100644 --- a/test/geom.test.cpp +++ b/test/geom.test.cpp @@ -116,6 +116,44 @@ MU_TEST(test_precheck_many_holes) { checkPrecheck(mp, false); } +static Ring squareRing(double x, double y, double size) { + Ring r; + geom::append(r, Point(x, y)); geom::append(r, Point(x + size, y)); + geom::append(r, Point(x + size, y + size)); geom::append(r, Point(x, y + size)); + geom::append(r, Point(x, y)); + return r; +} + +MU_TEST(test_precheck_large_polygon) { + // A 40000-vertex outer ring (a circle) with 3600 holes: far more than walking the + // outer ring for every hole would allow (the old 2^26 work cap) + Polygon p; + const int n = 40000; + for (int i = 0; i <= n; i++) { + const double a = 2 * M_PI * (i % n) / n; + geom::append(p.outer(), Point(100 + 100 * std::cos(a), 100 + 100 * std::sin(a))); + } + for (int i = 0; i < 60; i++) + for (int j = 0; j < 60; j++) + p.inners().push_back(squareRing(40 + i * 2, 40 + j * 2, 1)); + MultiPolygon mp; + mp.push_back(p); + geom::correct(mp); + checkPrecheck(mp, true); + + // A hole outside the outer ring + MultiPolygon outside = mp; + outside[0].inners().push_back(squareRing(300, 100, 1)); + geom::correct(outside); + checkPrecheck(outside, false); + + // A hole nested in a hole on the other side of the polygon + MultiPolygon nested = mp; + nested[0].inners().insert(nested[0].inners().begin(), squareRing(40.25, 40.25, 0.5)); + geom::correct(nested); + checkPrecheck(nested, false); +} + MU_TEST_SUITE(test_suite_geom) { MU_RUN_TEST(test_rect_clip_concave); MU_RUN_TEST(test_rect_clip_holes); @@ -124,6 +162,7 @@ MU_TEST_SUITE(test_suite_geom) { MU_RUN_TEST(test_rect_clip_touch_from_inside); MU_RUN_TEST(test_precheck_overlapping_bboxes); MU_RUN_TEST(test_precheck_many_holes); + MU_RUN_TEST(test_precheck_large_polygon); } int main() { diff --git a/test/ordered_batches.test.cpp b/test/ordered_batches.test.cpp new file mode 100644 index 00000000..5fe47787 --- /dev/null +++ b/test/ordered_batches.test.cpp @@ -0,0 +1,89 @@ +#include +#include +#include +#include +#include +#include +#include "external/minunit.h" +#include "ordered_batches.h" + +MU_TEST(test_ordered_batches_single_thread) { + std::vector written; + OrderedBatches batches([&](int& item) { written.push_back(item); }, [](const int&) { return 1; }); + + // batch 1 finishes first, so it's held until batch 0 is written + batches.begin(1); + mu_check(batches.inBatch()); + batches.add(10); + batches.add(11); + batches.end(); + mu_check(!batches.inBatch()); + mu_check(written.empty()); + mu_check(!batches.allWritten()); + + batches.begin(0); + batches.add(0); + batches.end(); + mu_check(written == std::vector({ 0, 10, 11 })); + mu_check(batches.allWritten()); + + // an empty batch still moves the order on + batches.begin(3); + batches.add(30); + batches.end(); + batches.begin(2); + batches.end(); + mu_check(written == std::vector({ 0, 10, 11, 30 })); + mu_check(batches.allWritten()); +} + +MU_TEST(test_ordered_batches_threads) { + // Batches are handed out in order and finish in any order; with a small limit on what + // may be held, threads also have to wait for room + const uint64_t numBatches = 2000, itemsPerBatch = 5; + std::vector written; + OrderedBatches batches([&](uint64_t& item) { written.push_back(item); }, [](const uint64_t&) { return 1; }); + batches.setMaxHeldBytes(20); + + std::mutex handOut; + uint64_t nextSeq = 0; + std::vector threads; + for (int t = 0; t < 8; t++) { + threads.emplace_back([&, t]() { + std::mt19937 rng(t); + while (true) { + batches.waitForRoom(); + uint64_t seq; + { + std::lock_guard lock(handOut); + if (nextSeq == numBatches) return; + seq = nextSeq++; + } + batches.begin(seq); + for (uint64_t i = 0; i < itemsPerBatch; i++) { + if (rng() % 4 == 0) std::this_thread::yield(); + batches.add(seq * itemsPerBatch + i); + } + batches.end(); + } + }); + } + for (auto& thread : threads) thread.join(); + + mu_check(batches.allWritten()); + mu_check(written.size() == numBatches * itemsPerBatch); + bool inOrder = true; + for (uint64_t i = 0; i < written.size(); i++) if (written[i] != i) inOrder = false; + mu_check(inOrder); +} + +MU_TEST_SUITE(test_suite_ordered_batches) { + MU_RUN_TEST(test_ordered_batches_single_thread); + MU_RUN_TEST(test_ordered_batches_threads); +} + +int main() { + MU_RUN_SUITE(test_suite_ordered_batches); + MU_REPORT(); + return MU_EXIT_CODE; +} diff --git a/test/shp_mem_tiles.test.cpp b/test/shp_mem_tiles.test.cpp new file mode 100644 index 00000000..a0cbcb2f --- /dev/null +++ b/test/shp_mem_tiles.test.cpp @@ -0,0 +1,66 @@ +#include +#include "external/minunit.h" +#include "shp_mem_tiles.h" + +bool verbose = false; + +namespace { + MultiPolygon square(double x, double y, double size) { + MultiPolygon mp; + Polygon p; + geom::append(p.outer(), Point(x, y)); + geom::append(p.outer(), Point(x, y + size)); + geom::append(p.outer(), Point(x + size, y + size)); + geom::append(p.outer(), Point(x + size, y)); + geom::append(p.outer(), Point(x, y)); + mp.push_back(p); + return mp; + } + + Box box(double x1, double y1, double x2, double y2) { + return Box(Point(x1, y1), Point(x2, y2)); + } +} + +MU_TEST(test_may_intersect) { + Declutter declutter; + ShpMemTiles shp(1, 14, declutter); + shp.CreateNamedLayerIndex("layer"); + + // Two small squares at opposite corners: the shape's bounding box, (0,0)-(10,10), + // is almost all empty, like that of a country with far-flung islands. + MultiPolygon mp = square(0, 0, 1); + mp.push_back(square(9, 9, 1)[0]); + shp.StoreGeometry(0, "layer", POLYGON_, mp, true, true, "corners", 0, 0, 0); + + // Outside the shape's bounding box + mu_check(!shp.mayIntersect("layer", box(20, 20, 21, 21))); + + // A small box (a few z15 tiles) in the empty middle: checked tile by tile + mu_check(!shp.mayIntersect("layer", box(5, 5, 5.001, 5.001))); + // A large box (tens of thousands of z15 tiles) in the empty middle: checked at once + mu_check(!shp.mayIntersect("layer", box(3, 3, 7, 7))); + + // Small and large boxes that touch a square + mu_check(shp.mayIntersect("layer", box(0.5, 0.5, 0.501, 0.501))); + mu_check(shp.mayIntersect("layer", box(0.5, 0.5, 5, 5))); + mu_check(shp.mayIntersect("layer", box(5, 5, 9.5, 9.5))); + + // A large box that surrounds a square without containing a vertex of it + mu_check(shp.mayIntersect("layer", box(-1, -1, 2, 2))); + + // Answers stay the same once tiles have been checked and cached + mu_check(!shp.mayIntersect("layer", box(5, 5, 5.001, 5.001))); + mu_check(!shp.mayIntersect("layer", box(3, 3, 7, 7))); + mu_check(shp.mayIntersect("layer", box(0.5, 0.5, 0.501, 0.501))); +} + +MU_TEST_SUITE(test_suite_shp_mem_tiles) { + MU_RUN_TEST(test_may_intersect); +} + +int main() { + MU_RUN_SUITE(test_suite_shp_mem_tiles); + MU_REPORT(); + return MU_EXIT_CODE; +} diff --git a/test/tile_coordinates_set.test.cpp b/test/tile_coordinates_set.test.cpp index 87a054b8..973cc13c 100644 --- a/test/tile_coordinates_set.test.cpp +++ b/test/tile_coordinates_set.test.cpp @@ -45,10 +45,71 @@ MU_TEST(test_tile_coordinates_set) { mu_check(z2.test(1, 1) == true); mu_check(z2.test(2, 2) == false); } + + // forEachSet on a wrapped set visits the same tiles as a full scan, in x-major, y-minor order + { + PreciseTileCoordinatesSet z3(3); + z3.set(0, 0); z3.set(0, 5); z3.set(2, 1); z3.set(2, 7); z3.set(7, 3); + LossyTileCoordinatesSet z5(5, z3); + + std::vector> expected, visited; + for (TileCoordinate x = 0; x < 32; x++) + for (TileCoordinate y = 0; y < 32; y++) + if (z5.test(x, y)) expected.push_back({x, y}); + z5.forEachSet([&](TileCoordinate x, TileCoordinate y) { visited.push_back({x, y}); }); + + mu_check(expected.size() == 5 * 16); + mu_check(visited == expected); + } +} + +static bool sameTiles(const TileCoordinatesSet& a, const TileCoordinatesSet& b) { + std::vector> ta, tb; + a.forEachSet([&](TileCoordinate x, TileCoordinate y) { ta.emplace_back(x, y); }); + b.forEachSet([&](TileCoordinate x, TileCoordinate y) { tb.emplace_back(x, y); }); + return ta == tb && a.size() == b.size(); +} + +MU_TEST(test_tile_coordinates_set_box) { + // setBox must set exactly the tiles that set() would, including boxes that + // start and end mid-word, span several words, or are clipped at the edge + const TileCoordinate boxes[][4] = { + { 0, 0, 0, 0 }, { 3, 9, 5, 70 }, { 100, 101, 0, 1023 }, { 7, 7, 60, 200 }, + { 1000, 1023, 990, 1023 }, { 1020, 1023, 1000, 1023 } + }; + for (const auto& b : boxes) { + PreciseTileCoordinatesSet bySet(10), byBox(10); + for (TileCoordinate x = b[0]; x <= b[1]; x++) + for (TileCoordinate y = b[2]; y <= b[3]; y++) + bySet.set(x, y); + byBox.setBox(b[0], b[1], b[2], b[3]); + mu_check(sameTiles(bySet, byBox)); + } + // Partly outside the 1024x1024 grid: the part inside is set + PreciseTileCoordinatesSet edge(10); + edge.setBox(1020, 1030, 1022, 1030); + mu_check(edge.size() == 4 * 2); + mu_check(edge.test(1023, 1023)); +} + +MU_TEST(test_tile_coordinates_set_merge) { + PreciseTileCoordinatesSet a(12), b(12), both(12); + for (TileCoordinate i = 0; i < 4096; i += 37) { + a.set(i, (i * 7) % 4096); + both.set(i, (i * 7) % 4096); + } + for (TileCoordinate i = 0; i < 4096; i += 53) { + b.set((i * 3) % 4096, i); + both.set((i * 3) % 4096, i); + } + a.merge(b); + mu_check(sameTiles(a, both)); } MU_TEST_SUITE(test_suite_tile_coordinates_set) { MU_RUN_TEST(test_tile_coordinates_set); + MU_RUN_TEST(test_tile_coordinates_set_box); + MU_RUN_TEST(test_tile_coordinates_set_merge); } int main() { From a5c518cc02609bb2413074f9a7e1440f9fcbed87 Mon Sep 17 00:00:00 2001 From: dktue <2878623+dktue@users.noreply.github.com> Date: Sat, 7 Nov 2026 18:18:30 +0100 Subject: [PATCH 2/8] test: add #986 (sort the tile list with at most 16 threads) Same change as #986. With it, the sort before the tile phase takes ~13 s instead of 2-5 minutes at 96 threads, so planet times no longer vary by minutes between runs (14:12 / 14:08 in memory at 96 threads). Co-Authored-By: Claude Opus 5.5 (1M context) --- src/tile_sorting.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/tile_sorting.cpp b/src/tile_sorting.cpp index cc79e20d..3bb9442f 100644 --- a/src/tile_sorting.cpp +++ b/src/tile_sorting.cpp @@ -335,5 +335,7 @@ void sortTileCoordinates( return false; }, - threadNum); + // block_indirect_sort slows down badly with many threads: on the planet (346M tiles) it took + // 200-300 s at 96 threads instead of about 7 s at 16, with the threads spinning on its internal lock. + std::min(threadNum, 16)); } \ No newline at end of file From 154852c167a75c0ac5f70c591583be19f547e207 Mon Sep 17 00:00:00 2001 From: dktue <2878623+dktue@users.noreply.github.com> Date: Sun, 8 Nov 2026 10:09:13 +0100 Subject: [PATCH 3/8] perf: wake threads waiting for room only once there is room end() woke every thread waiting in waitForRoom() after each batch it wrote, although most of the time the held tiles were still over the limit. At 96 threads that meant dozens of threads waking, finding no room and sleeping again for every batch: 125M context switches and 7,700 s of system time on the planet. Now they are only woken once the held tiles are back under the limit. Planet in memory, 96 threads, all open performance PRs: 14:05 -> 12:45 (-9.5%), system time 7,708 -> 2,868 s, context switches 125M -> 35M. Output unchanged (same batches, same order). Co-Authored-By: Claude Opus 5.5 (1M context) --- include/ordered_batches.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/include/ordered_batches.h b/include/ordered_batches.h index 2d2b3b90..d67dff4a 100644 --- a/include/ordered_batches.h +++ b/include/ordered_batches.h @@ -68,7 +68,10 @@ class OrderedBatches { lock.lock(); heldBytes -= batch.first; nextSeq++; - heldShrunk.notify_all(); + // Wake the waiting threads only once there is room. Woken for every batch + // written, they all found no room and went back to sleep, again and again: + // most of the system time of a 96-thread planet run. + if (heldBytes <= maxHeldBytes) heldShrunk.notify_all(); } draining = false; } From 89208f38bcc1d8a2206721e54895832cfb83aee6 Mon Sep 17 00:00:00 2001 From: dktue <2878623+dktue@users.noreply.github.com> Date: Sun, 8 Nov 2026 10:09:13 +0100 Subject: [PATCH 4/8] perf: clip large polygons once per z8 tile and share the piece Every tile batch that needs a large polygon copied it from the store and clipped it from scratch, at every zoom. Now polygons with at least 10,000 points are clipped once per z8 tile, before the tiles are written (all threads, largest polygon first), and tiles below clip from that piece. After that the pieces are only read, so there is no lock while tiles are written; a z6 tile's pieces are freed once no tile in it is still to come. A piece only depends on the polygon and the z8 tile, so the output stays the same for any thread count. A piece whose clip needed the repair path is not used: once one came out empty and a lake vanished from all tiles below. Planet in memory, 96 threads, all open performance PRs + #983's wake-up fix: 12:45 -> 10:55 (-14.4%), user CPU -27%, peak RSS +7.5 GB. Germany byte-identical at 32 and 8 threads; planet sample: same feature counts, area within 0.003%. Co-Authored-By: Claude Opus 5.5 (1M context) --- include/clip_cache.h | 7 ++- include/osm_mem_tiles.h | 2 + include/tile_data.h | 21 +++++++ src/tile_data.cpp | 126 ++++++++++++++++++++++++++++++++++++++++ src/tilemaker.cpp | 42 ++++++++++++++ 5 files changed, 197 insertions(+), 1 deletion(-) diff --git a/include/clip_cache.h b/include/clip_cache.h index 66a400ab..e650671a 100644 --- a/include/clip_cache.h +++ b/include/clip_cache.h @@ -16,6 +16,9 @@ class TileBbox; extern thread_local uint64_t clipCacheBatchId; // Set when the current batch can't reuse clips (a single tile), so add() is skipped. extern thread_local bool clipCacheNoStore; +// Set while a shared piece of a large polygon is computed (TileDataSource::precomputeSharedPieces): +// the batch cache is then neither read nor written. +extern thread_local bool clipCacheBypass; template class ClipCache { @@ -36,6 +39,8 @@ class ClipCache { } Entry getEntry(uint zoom, TileCoordinate x, TileCoordinate y, NodeID objectID) const { + if (clipCacheBypass) + return Entry(); // Look for a previously clipped version at z-1, z-2, ... (nearest ancestor first) LocalStore& s = local(zoom, TileCoordinates(x, y)); Entry rv; @@ -59,7 +64,7 @@ class ClipCache { void add(const TileBbox& bbox, const NodeID objectID, const T& output, bool valid = false) { // The point of caching is to reuse the clip, so caching at the terminal zoom is // pointless. - if (bbox.zoom == baseZoom || clipCacheNoStore) + if (bbox.zoom == baseZoom || clipCacheNoStore || clipCacheBypass) return; // Levels beyond MAX_LEVELS could never be looked up by a child tile anyway diff --git a/include/osm_mem_tiles.h b/include/osm_mem_tiles.h index 1966bc4b..5ab90fc1 100644 --- a/include/osm_mem_tiles.h +++ b/include/osm_mem_tiles.h @@ -57,6 +57,8 @@ class OsmMemTiles : public TileDataSource { void populateLinestring(Linestring& ls, NodeID objectID) const; Linestring& getOrBuildLinestring(NodeID objectID) const; void populateMultiPolygon(MultiPolygon& dst, NodeID objectID) override; + // Polygons from ways are built from the way store, only relations are in the multipolygon store + bool storesMultiPolygon(NodeID objectID) const override { return objectID < OSM_THRESHOLD; } // Way-store polygons are only emitted by OsmLuaProcessing::Layer when // CorrectGeometry found them Valid (or Reversed: valid after reversing the ring) // on the same lon/latp coordinates. diff --git a/include/tile_data.h b/include/tile_data.h index 729ef009..b7ceaf71 100644 --- a/include/tile_data.h +++ b/include/tile_data.h @@ -8,7 +8,10 @@ #include #include #include +#include +#include #include +#include #include #include #include "append_vector.h" @@ -419,6 +422,24 @@ class TileDataSource { void orderStoredGeometryTies(std::vector& data) const; virtual Geometry buildWayGeometry(OutputGeometryType const geomType, NodeID const objectID, const TileBbox &bbox); + + // Large polygons are clipped once per SHARED_PIECE_ZOOM tile before the tiles are written, + // and deeper tiles clip from that piece instead of the whole polygon (see tile_data.cpp). + static constexpr unsigned SHARED_PIECE_ZOOM = 8; + // present(x, y): whether any tile below SHARED_PIECE_ZOOM tile x/y will be written + void precomputeSharedPieces(const std::function &present, unsigned threadNum); + // Drop the shared pieces inside a z6 tile, once no tile in it is still to be written + void releaseSharedPieces(TileCoordinate z6x, TileCoordinate z6y); + // Whether objectID's polygon is kept in the multipolygon store (retrieveMultiPolygon) + virtual bool storesMultiPolygon(NodeID objectID) const { return true; } + struct SharedPiece { + std::shared_ptr geom; // null: no usable piece, clip the whole polygon + bool valid = false; + }; + // Filled by precomputeSharedPieces, then only read while the tiles are written (no lock). + // releaseSharedPieces resets the pieces of a z6 tile no thread reads any more. + std::unordered_set pieceObjects; + std::map, std::shared_ptr> sharedPieces; virtual LatpLon buildNodeGeometry(NodeID const objectID, const TileBbox &bbox) const; void open() { diff --git a/src/tile_data.cpp b/src/tile_data.cpp index c1fcc02e..cdee86a3 100644 --- a/src/tile_data.cpp +++ b/src/tile_data.cpp @@ -90,6 +90,118 @@ TileDataSource::TileDataSource(size_t threadNum, unsigned int indexZoom, bool in thread_local std::vector>* tlsPendingSmallIndexObjects = nullptr; thread_local uint64_t clipCacheBatchId = 0; thread_local bool clipCacheNoStore = false; +thread_local bool clipCacheBypass = false; +thread_local bool clipNeededRepair = false; // set when a POLYGON_ clip went down the repair path +thread_local bool clipResultValid = false; // whether the last POLYGON_ clip is known to be valid + +// Shared pieces. Every tile batch that needs a large polygon (often 100,000+ points) used to copy +// it from the store and clip it from scratch, at every zoom. Instead, polygons with at least +// SHARED_PIECE_MIN_POINTS points are clipped once per SHARED_PIECE_ZOOM tile before the tiles are +// written, and tiles below clip from that piece. A piece only depends on the polygon and the tile, +// never on the thread or the order, so the output stays the same for any thread count. +// +// The pieces are computed up front, all threads at once and largest polygon first: computed on +// demand, a slow piece (up to ~15 s on the planet) held up every thread whose tiles needed it, +// a third of the tile phase on a 96-thread planet run. After that they are only read, so the +// tile threads look them up without a lock. +#define SHARED_PIECE_MIN_POINTS 10000 + +void TileDataSource::precomputeSharedPieces(const std::function &present, unsigned threadNum) { + const unsigned zoom = SHARED_PIECE_ZOOM; + if (zoom > indexZoom) return; + const unsigned shift = indexZoom - zoom; + // Candidates: polygons in the large-object index (those spanning 16+ index tiles), with the + // index tiles they cover + std::map boxes; + auto add = [&](const Box &box, const OutputObject &oo) { + if (oo.geomType != POLYGON_ || !storesMultiPolygon(oo.objectID)) return; + auto it = boxes.find(oo.objectID); + if (it == boxes.end()) boxes.emplace(oo.objectID, box); + else boost::geometry::expand(it->second, box); + }; + for (auto const &r : boxRtree) add(r.first, r.second); + for (auto const &r : boxRtreeWithIds) add(r.first, r.second.oo); + if (boxes.empty()) return; + const std::vector> candidates(boxes.begin(), boxes.end()); + + // The pieces each candidate needs: tiles below which a tile is written, that its envelope reaches + struct Job { std::tuple key; uint64_t points; }; + std::vector jobs; + std::mutex jobsMutex; + std::atomic next(0); + auto findJobs = [&]() { + std::vector local; + for (size_t i = next++; i < candidates.size(); i = next++) { + const NodeID objectID = candidates[i].first; + uint64_t points = 0; + Box envelope; + boost::geometry::assign_inverse(envelope); + for (auto const &p : retrieveMultiPolygon(objectID)) { + points += p.outer().size(); + for (auto const &inner : p.inners()) points += inner.size(); + for (auto const &pt : p.outer()) boost::geometry::expand(envelope, pt); + } + if (points < SHARED_PIECE_MIN_POINTS) continue; + const Box &box = candidates[i].second; + const TileCoordinate x0 = (TileCoordinate)box.min_corner().x() >> shift, x1 = (TileCoordinate)box.max_corner().x() >> shift; + const TileCoordinate y0 = (TileCoordinate)box.min_corner().y() >> shift, y1 = (TileCoordinate)box.max_corner().y() >> shift; + for (TileCoordinate x = x0; x <= x1; x++) + for (TileCoordinate y = y0; y <= y1; y++) + if (present(x, y) && boost::geometry::intersects(envelope, TileBbox(TileCoordinates(x, y), zoom, false, false).clippingBox)) + local.push_back(Job { std::make_tuple(objectID, x, y), points }); + } + std::lock_guard lock(jobsMutex); + jobs.insert(jobs.end(), local.begin(), local.end()); + }; + { + boost::asio::thread_pool pool(threadNum); + for (unsigned t = 0; t < threadNum; t++) boost::asio::post(pool, findJobs); + pool.join(); + } + std::sort(jobs.begin(), jobs.end(), [](const Job &a, const Job &b) { return a.points != b.points ? a.points > b.points : a.key < b.key; }); + for (const Job &job : jobs) { + pieceObjects.insert(std::get<0>(job.key)); + sharedPieces.emplace(job.key, nullptr); + } + + // Compute them, largest polygon first. Each job writes its own map entry; the map itself + // doesn't change any more. + next = 0; + auto compute = [&]() { + for (size_t i = next++; i < jobs.size(); i = next++) { + const auto &key = jobs[i].key; + auto piece = std::make_shared(); + try { + clipCacheBypass = true; + clipNeededRepair = false; + Geometry g = buildWayGeometry(POLYGON_, std::get<0>(key), TileBbox(TileCoordinates(std::get<1>(key), std::get<2>(key)), zoom, false, false)); + // A piece that needed the repair path can be wrong (once a lake came out empty), and + // every tile below it would inherit that: leave those tiles to clip the whole polygon. + if (!clipNeededRepair) { + piece->geom = std::make_shared(std::move(boost::get(g))); + // buildWayGeometry has just checked it (for a huge piece, is_valid takes seconds) + piece->valid = clipResultValid; + } + } catch (std::exception &) { + // The tiles clip the whole polygon, and report the problem there + } + clipCacheBypass = false; + sharedPieces.find(key)->second = piece; + } + }; + { + boost::asio::thread_pool pool(threadNum); + for (unsigned t = 0; t < threadNum; t++) boost::asio::post(pool, compute); + pool.join(); + } +} + +void TileDataSource::releaseSharedPieces(TileCoordinate z6x, TileCoordinate z6y) { + const unsigned shift = SHARED_PIECE_ZOOM - CLUSTER_ZOOM; + for (auto &e : sharedPieces) + if ((std::get<1>(e.first) >> shift) == z6x && (std::get<2>(e.first) >> shift) == z6y) + e.second.reset(); +} void TileDataSource::finalize(size_t threadNum) { uint64_t finalized = 0; @@ -333,6 +445,15 @@ Geometry TileDataSource::buildWayGeometry(OutputGeometryType const geomType, auto cachedEntry = multiPolygonClipCache.getEntry(bbox.zoom, bbox.index.x, bbox.index.y, objectID); std::shared_ptr cachedClip = cachedEntry.geom; bool inputKnownValid = cachedClip == nullptr ? polygonKnownValid(objectID) : cachedEntry.valid; + if (cachedClip == nullptr && bbox.zoom > SHARED_PIECE_ZOOM && pieceObjects.count(objectID)) { + // A large polygon: clip from its shared piece + const unsigned shift = bbox.zoom - SHARED_PIECE_ZOOM; + auto it = sharedPieces.find(std::make_tuple(objectID, bbox.index.x >> shift, bbox.index.y >> shift)); + if (it != sharedPieces.end() && it->second && it->second->geom) { + cachedClip = it->second->geom; + inputKnownValid = it->second->valid; + } + } MultiPolygon uncached; @@ -404,6 +525,7 @@ Geometry TileDataSource::buildWayGeometry(OutputGeometryType const geomType, // fast_clip was the identity on a geometry that already passed // is_valid, so correct() is a no-op and it is still valid. multiPolygonClipCache.add(bbox, objectID, mp, true); + clipResultValid = true; return mp; } geom::correct(mp); @@ -420,12 +542,14 @@ Geometry TileDataSource::buildWayGeometry(OutputGeometryType const geomType, if (rect_clip(cachedClip == nullptr && !keepUncached ? original : input, box, clipped) && (multiPolygonDefinitelyValid(clipped) || geom::is_valid(clipped))) { multiPolygonClipCache.add(bbox, objectID, clipped, true); + clipResultValid = true; return clipped; } } if (!valid) valid = geom::is_valid(mp,failure); if (!valid) { + clipNeededRepair = true; if (verbose) { // Build the whole line first and emit it with a single stream write: // tilemaker runs multi-threaded and chained operator<< calls are not @@ -461,6 +585,7 @@ Geometry TileDataSource::buildWayGeometry(OutputGeometryType const geomType, repairMultiPolygon(output, objectID, &outputValid); // Children of this tile can then take the fast paths for valid input multiPolygonClipCache.add(bbox, objectID, output, outputValid); + clipResultValid = outputValid; return output; } else if (!valid) { // occasionally also wrong_topological_dimension, disconnected_interior: @@ -471,6 +596,7 @@ Geometry TileDataSource::buildWayGeometry(OutputGeometryType const geomType, } multiPolygonClipCache.add(bbox, objectID, mp, valid); + clipResultValid = valid; return mp; } diff --git a/src/tilemaker.cpp b/src/tilemaker.cpp index bdecc1d4..a2643cac 100644 --- a/src/tilemaker.cpp +++ b/src/tilemaker.cpp @@ -510,6 +510,19 @@ int main(const int argc, const char* argv[]) { // Cluster tiles: breadth-first for z0..z5, depth-first for z6 sortTileCoordinates(config.baseZoom, options.threadNum, tileCoordinates); + // Shared pieces of large polygons, for the SHARED_PIECE_ZOOM tiles below which a tile is written + { + const unsigned pieceZoom = TileDataSource::SHARED_PIECE_ZOOM; + std::vector piecePresent((size_t)1 << (2 * pieceZoom), false); + for (auto const &t : tileCoordinates) + if (t.first > pieceZoom) + piecePresent[((size_t)(t.second.x >> (t.first - pieceZoom)) << pieceZoom) + (t.second.y >> (t.first - pieceZoom))] = true; + auto present = [&](TileCoordinate x, TileCoordinate y) { + return x < (1u << pieceZoom) && y < (1u << pieceZoom) && piecePresent[((size_t)x << pieceZoom) + y]; + }; + for (auto source : sources) source->precomputeSharedPieces(present, options.threadNum); + } + std::atomic batchCounter(0); const uint cutZoom = std::max(12, config.startZoom); // Batches are handed out as threads become free rather than all posted up @@ -517,6 +530,20 @@ int main(const int argc, const char* argv[]) { std::mutex batchMutex; std::size_t nextBatchStart = 0; uint64_t nextBatchSeq = 0; + // Shared pieces (TileDataSource::precomputeSharedPieces) lie inside one z6 tile. Tiles from z6 on + // are written z6 tile by z6 tile, so once the hand-out has moved past a z6 tile and none of its + // batches is still running, no tile needs its pieces any more. + std::vector z6Running(CLUSTER_ZOOM_WIDTH * CLUSTER_ZOOM_WIDTH, 0); + int64_t z6HandedOut = -1; + auto z6Of = [&](std::size_t i) -> int64_t { + const unsigned z = tileCoordinates[i].first; + if (z < CLUSTER_ZOOM) return -1; + return (int64_t)(tileCoordinates[i].second.x >> (z - CLUSTER_ZOOM)) * CLUSTER_ZOOM_WIDTH + (tileCoordinates[i].second.y >> (z - CLUSTER_ZOOM)); + }; + auto releaseZ6 = [&](int64_t b) { + for (auto source : sources) source->releaseSharedPieces(b / CLUSTER_ZOOM_WIDTH, b % CLUSTER_ZOOM_WIDTH); + }; + static thread_local std::vector batchZ6; // z6 tiles of the batch this thread is working on auto nextBatch = [&](std::size_t &startIndex, std::size_t &batchSize, uint64_t &seq) { const std::lock_guard lock(batchMutex); startIndex = nextBatchStart; @@ -545,6 +572,16 @@ int main(const int argc, const char* argv[]) { batchSize++; } nextBatchStart += batchSize; + batchZ6.clear(); + for (std::size_t i = startIndex; i < startIndex + batchSize; i++) { + const int64_t b = z6Of(i); + if (b >= 0 && (batchZ6.empty() || batchZ6.back() != b)) batchZ6.push_back(b); + } + for (int64_t b : batchZ6) z6Running[b]++; + if (!batchZ6.empty() && batchZ6.back() != z6HandedOut) { + if (z6HandedOut >= 0 && z6Running[z6HandedOut] == 0) releaseZ6(z6HandedOut); + z6HandedOut = batchZ6.back(); + } return batchSize > 0; }; @@ -605,6 +642,11 @@ int main(const int argc, const char* argv[]) { std::cout << output << std::endl; } + { + const std::lock_guard lock(batchMutex); + for (int64_t b : batchZ6) + if (--z6Running[b] == 0 && b != z6HandedOut) releaseZ6(b); + } if (toPMTiles) sharedData.pmtiles.batches.end(); if (toMBTiles) sharedData.mbtiles.batches.end(); tilesWritten += (endIndex - startIndex); From 528bb9992faf8e399f30fe3f8e7de687a4787607 Mon Sep 17 00:00:00 2001 From: dktue <2878623+dktue@users.noreply.github.com> Date: Sun, 8 Nov 2026 14:11:34 +0100 Subject: [PATCH 5/8] perf(pmtiles): place and write tiles in two ordered steps At 96 threads the one thread that writes batches in order became the bottleneck of the tile phase: busy 260 s of a 323 s tile phase, with the others waiting for room for 5,100 thread-seconds. Writing the bytes was more than half of its work (ofstream::write, and a tellp() per tile). - The writer keeps its own tile data offset instead of calling tellp() for every tile, and the stream gets a 4 MB buffer. - OrderedBatches can take a second step, flushItem: also in batch order and one thread at a time, but run by whichever thread is free, so it doesn't hold up the first step. PMTiles places tiles (offset, repeated tiny tiles, index) in the first step and writes their bytes in the second. The file is byte-identical. Planet in memory, 96 threads, all open performance PRs: 10:41 -> 9:54 (-7.3%); tile phase 323 -> 245 s, threads busy 76 -> 94 of 96 on average. New test: the second step runs in order, never twice at once, and always after the first (also clean under TSan). Co-Authored-By: Claude Opus 5.5 (1M context) --- include/ordered_batches.h | 53 +++++++++++++++++++++++++++++---- include/pmtiles.h | 7 +++-- src/pmtiles.cpp | 20 +++++++++++-- test/ordered_batches.test.cpp | 56 +++++++++++++++++++++++++++++++++++ 4 files changed, 125 insertions(+), 11 deletions(-) diff --git a/include/ordered_batches.h b/include/ordered_batches.h index d67dff4a..9531565f 100644 --- a/include/ordered_batches.h +++ b/include/ordered_batches.h @@ -3,6 +3,7 @@ #define _ORDERED_BATCHES_H #include +#include #include #include #include @@ -21,11 +22,15 @@ // being worked on by a thread that isn't waiting, so waitForRoom() always returns. // // writeItem is called by one thread at a time, with writeMutex() held, in batch order. +// If flushItem is given, a batch then goes through flushItem as well: also one thread at a time +// and in batch order, but without holding up writeItem. Whichever thread is free does it, so the +// two steps run side by side. A batch counts as held until it has been flushed. template class OrderedBatches { public: - OrderedBatches(std::function writeItem, std::function itemBytes) - : writeItem(writeItem), itemBytes(itemBytes) { } + OrderedBatches(std::function writeItem, std::function itemBytes, + std::function flushItem = nullptr) + : writeItem(writeItem), itemBytes(itemBytes), flushItem(flushItem) { } void setMaxHeldBytes(uint64_t bytes) { std::lock_guard lock(m); @@ -55,7 +60,10 @@ class OrderedBatches { heldBytes += bytes; held.emplace(c.seq, std::make_pair(bytes, std::move(c.items))); c.items = std::vector(); - if (draining) return; + if (draining) { + flushQueued(lock); + return; + } draining = true; while (!held.empty() && held.begin()->first == nextSeq) { auto batch = std::move(held.begin()->second); @@ -66,24 +74,33 @@ class OrderedBatches { for (auto& item : batch.second) writeItem(item); } lock.lock(); - heldBytes -= batch.first; nextSeq++; + if (flushItem) { + toFlush.emplace_back(std::move(batch)); + heldShrunk.notify_one(); // a waiting thread can flush it + continue; + } + heldBytes -= batch.first; // Wake the waiting threads only once there is room. Woken for every batch // written, they all found no room and went back to sleep, again and again: // most of the system time of a 96-thread planet run. if (heldBytes <= maxHeldBytes) heldShrunk.notify_all(); } draining = false; + flushQueued(lock); } void waitForRoom() { std::unique_lock lock(m); - heldShrunk.wait(lock, [&]() { return heldBytes <= maxHeldBytes; }); + while (heldBytes > maxHeldBytes) { + if (!toFlush.empty() && !flushing) flushQueued(lock); + else heldShrunk.wait(lock); + } } bool allWritten() { std::lock_guard lock(m); - return held.empty(); + return held.empty() && toFlush.empty() && !flushing; } // Held while items are written; lock it to write anything outside a batch @@ -95,6 +112,27 @@ class OrderedBatches { uint64_t seq = 0; std::vector items; }; + // Flush the batches waiting for flushItem, unless another thread already does; called and + // returns with m held + void flushQueued(std::unique_lock &lock) { + if (!flushItem || flushing) return; + flushing = true; + while (!toFlush.empty()) { + auto batch = std::move(toFlush.front()); + toFlush.pop_front(); + lock.unlock(); + for (auto& item : batch.second) flushItem(item); + batch.second = std::vector(); + lock.lock(); + heldBytes -= batch.first; + if (heldBytes <= maxHeldBytes) heldShrunk.notify_all(); + } + flushing = false; + // batches queued while finishing the last one are taken by the next thread that comes by; + // wake one in case all others wait for room + heldShrunk.notify_one(); + } + // The batch this thread is working on (one per thread and item type) static Current& current() { static thread_local Current c; @@ -103,6 +141,7 @@ class OrderedBatches { std::function writeItem; std::function itemBytes; + std::function flushItem; uint64_t maxHeldBytes = 1ull << 30; std::mutex m; // guards the fields below std::mutex w; // held while writing @@ -111,6 +150,8 @@ class OrderedBatches { uint64_t nextSeq = 0; uint64_t heldBytes = 0; std::map>> held; // seq -> (bytes, items) + std::deque>> toFlush; // written in order, waiting for flushItem + bool flushing = false; // a thread is flushing }; #endif //_ORDERED_BATCHES_H diff --git a/include/pmtiles.h b/include/pmtiles.h index bcc701f1..fb2e6682 100644 --- a/include/pmtiles.h +++ b/include/pmtiles.h @@ -38,7 +38,7 @@ struct TileOffset { struct PendingTile { uint64_t tileId; bool isTiny; - std::string data; + std::string data; // after writeTile: the bytes flushTile appends to the file (empty if a repeat) }; class PMTiles { @@ -60,6 +60,8 @@ class PMTiles { private: std::ofstream outputStream; + std::vector outputBuffer; // a large stream buffer: one thread writes all tiles + uint64_t tileDataEnd = 0; // bytes of tile data written so far (tellp() costs a flush and a seek per tile) std::mutex fileMutex; // guards leaf directory writes in close() uint64_t leafStart = 0; uint64_t numTilesWritten = 0; @@ -69,7 +71,8 @@ class PMTiles { std::vector denseIndex; std::unordered_map tinyCache; - void writeTile(PendingTile &tile); // with batches.writeMutex() held + void writeTile(PendingTile &tile); // with batches.writeMutex() held: places the tile + void flushTile(PendingTile &tile); // in the same order, by one thread at a time: writes its bytes void appendWithRLE(std::vector &entries, pmtiles::entryv3 &entry); void appendTileEntry(uint64_t tileId, TileOffset offset, std::vector &rootEntries, std::vector &entries); void flushEntries(std::vector &rootEntries, std::vector &entries); diff --git a/src/pmtiles.cpp b/src/pmtiles.cpp index fa520f34..dceb5af4 100644 --- a/src/pmtiles.cpp +++ b/src/pmtiles.cpp @@ -10,13 +10,19 @@ #include "helpers.h" TileOffset::TileOffset() { } +// Placing tiles (offset, repeats, index) and writing their bytes are two steps, each in batch order +// and one thread at a time, but run by different threads side by side: one thread doing both was +// the bottleneck of the tile phase at 96 threads. PMTiles::PMTiles() : batches([this](PendingTile &tile) { writeTile(tile); }, - [](const PendingTile &tile) { return tile.data.size(); }) { } + [](const PendingTile &tile) { return tile.data.size(); }, + [this](PendingTile &tile) { flushTile(tile); }) { } PMTiles::~PMTiles() { } void PMTiles::open(std::string &filename) { std::cout << "Creating pmtiles at " << filename << std::endl; + outputBuffer.resize(4 << 20); + outputStream.rdbuf()->pubsetbuf(outputBuffer.data(), outputBuffer.size()); outputStream.open(filename, std::ios::out | std::ios::trunc | std::ios::binary); // dummy header/root directory for now - we'll write it all later char header[HEADER_ROOT] = "PMTiles"; @@ -145,6 +151,12 @@ void PMTiles::saveTile(int zoom, int x, int y, std::string &data) { } std::lock_guard lock(batches.writeMutex()); writeTile(tile); + flushTile(tile); +} + +void PMTiles::flushTile(PendingTile &tile) { + if (!tile.data.empty()) outputStream.write(tile.data.c_str(), tile.data.size()); + tile.data = std::string(); } // Write a tile to file and store it in the index @@ -154,16 +166,18 @@ void PMTiles::writeTile(PendingTile &tile) { auto cached = tile.isTiny ? tinyCache.find(tile.data) : tinyCache.end(); if (cached != tinyCache.end()) { offset = cached->second; + tile.data = std::string(); // already in the file } else { std::string compressed = tile.isTiny ? compress_string(tile.data, Z_DEFAULT_COMPRESSION, true) : std::move(tile.data); - offset = TileOffset(static_cast(outputStream.tellp()) - HEADER_ROOT, compressed.size()); - outputStream.write(compressed.c_str(), compressed.size()); + offset = TileOffset(tileDataEnd, compressed.size()); + tileDataEnd += compressed.size(); numTilesWritten++; // add to cache if tiny if (tile.isTiny) { if (tinyCache.size()>TINY_MAX_SIZE) tinyCache.clear(); tinyCache.insert({ tile.data, offset }); } + tile.data = std::move(compressed); // flushTile writes it } // store in index diff --git a/test/ordered_batches.test.cpp b/test/ordered_batches.test.cpp index 5fe47787..b9ff9599 100644 --- a/test/ordered_batches.test.cpp +++ b/test/ordered_batches.test.cpp @@ -77,9 +77,65 @@ MU_TEST(test_ordered_batches_threads) { mu_check(inOrder); } +MU_TEST(test_ordered_batches_flush) { + // With flushItem: every item is written and then flushed, both in order, flushes never + // overlap, and a flush can be done by a thread other than the one that wrote the batch + const uint64_t numBatches = 2000, itemsPerBatch = 5; + std::vector written, flushed; + std::atomic flushingNow(0); + std::atomic overlap(false), flushedBeforeWritten(false); + std::atomic writtenCount(0); + OrderedBatches batches( + [&](uint64_t& item) { written.push_back(item); writtenCount++; }, + [](const uint64_t&) { return 1; }, + [&](uint64_t& item) { + if (flushingNow++ != 0) overlap = true; + if (item >= writtenCount) flushedBeforeWritten = true; + flushed.push_back(item); + if (item % 7 == 0) std::this_thread::yield(); + flushingNow--; + }); + batches.setMaxHeldBytes(20); + + std::mutex handOut; + uint64_t nextSeq = 0; + std::vector threads; + for (int t = 0; t < 8; t++) { + threads.emplace_back([&, t]() { + std::mt19937 rng(t); + while (true) { + batches.waitForRoom(); + uint64_t seq; + { + std::lock_guard lock(handOut); + if (nextSeq == numBatches) return; + seq = nextSeq++; + } + batches.begin(seq); + for (uint64_t i = 0; i < itemsPerBatch; i++) { + if (rng() % 4 == 0) std::this_thread::yield(); + batches.add(seq * itemsPerBatch + i); + } + batches.end(); + } + }); + } + for (auto& thread : threads) thread.join(); + + mu_check(batches.allWritten()); + mu_check(!overlap); + mu_check(!flushedBeforeWritten); + mu_check(written.size() == numBatches * itemsPerBatch); + mu_check(flushed.size() == numBatches * itemsPerBatch); + bool inOrder = true; + for (uint64_t i = 0; i < flushed.size(); i++) if (flushed[i] != i || written[i] != i) inOrder = false; + mu_check(inOrder); +} + MU_TEST_SUITE(test_suite_ordered_batches) { MU_RUN_TEST(test_ordered_batches_single_thread); MU_RUN_TEST(test_ordered_batches_threads); + MU_RUN_TEST(test_ordered_batches_flush); } int main() { From d019a904e60a626b802ff7bd3e2f08db9802671a Mon Sep 17 00:00:00 2001 From: dktue <2878623+dktue@users.noreply.github.com> Date: Sun, 8 Nov 2026 15:29:20 +0100 Subject: [PATCH 6/8] perf: let the tile threads compute the shared pieces The pieces used to be computed in a pass of their own before the tile phase, which ended with one thread on the largest piece (~6 s on the planet) while all others waited. Now the tile threads compute them before their first tile, largest polygon first, and go on to tiles once no piece is left to start; a tile that needs a piece still being computed waits for just that one (shared_future). A piece is known before the tiles start, so the map doesn't change and tiles look pieces up without a lock as before. Output byte-identical (Germany at 32 and 8 threads). Co-Authored-By: Claude Opus 5.5 (1M context) --- include/tile_data.h | 13 +++++++- src/tile_data.cpp | 79 +++++++++++++++++++++++---------------------- src/tilemaker.cpp | 2 ++ 3 files changed, 54 insertions(+), 40 deletions(-) diff --git a/include/tile_data.h b/include/tile_data.h index b7ceaf71..8d3f3d41 100644 --- a/include/tile_data.h +++ b/include/tile_data.h @@ -12,6 +12,8 @@ #include #include #include +#include +#include #include #include #include "append_vector.h" @@ -428,6 +430,8 @@ class TileDataSource { static constexpr unsigned SHARED_PIECE_ZOOM = 8; // present(x, y): whether any tile below SHARED_PIECE_ZOOM tile x/y will be written void precomputeSharedPieces(const std::function &present, unsigned threadNum); + // Compute pieces until none is left to start; the tile threads call this before their first tile + void computeSharedPieces(); // Drop the shared pieces inside a z6 tile, once no tile in it is still to be written void releaseSharedPieces(TileCoordinate z6x, TileCoordinate z6y); // Whether objectID's polygon is kept in the multipolygon store (retrieveMultiPolygon) @@ -439,7 +443,14 @@ class TileDataSource { // Filled by precomputeSharedPieces, then only read while the tiles are written (no lock). // releaseSharedPieces resets the pieces of a z6 tile no thread reads any more. std::unordered_set pieceObjects; - std::map, std::shared_ptr> sharedPieces; + std::map, std::shared_future>> sharedPieces; + struct PieceJob { + std::tuple key; + uint64_t points; + std::shared_ptr>> promise; + }; + std::vector pieceJobs; // largest polygon first + std::atomic nextPieceJob { 0 }; virtual LatpLon buildNodeGeometry(NodeID const objectID, const TileBbox &bbox) const; void open() { diff --git a/src/tile_data.cpp b/src/tile_data.cpp index cdee86a3..1f100c54 100644 --- a/src/tile_data.cpp +++ b/src/tile_data.cpp @@ -100,10 +100,11 @@ thread_local bool clipResultValid = false; // whether the last POLYGON_ clip is // written, and tiles below clip from that piece. A piece only depends on the polygon and the tile, // never on the thread or the order, so the output stays the same for any thread count. // -// The pieces are computed up front, all threads at once and largest polygon first: computed on -// demand, a slow piece (up to ~15 s on the planet) held up every thread whose tiles needed it, -// a third of the tile phase on a 96-thread planet run. After that they are only read, so the -// tile threads look them up without a lock. +// The pieces are known up front and computed by all threads, largest polygon first, before they +// start on tiles: computed on demand, a slow piece (up to ~15 s on the planet) held up every +// thread whose tiles needed it, a third of the tile phase on a 96-thread planet run. A thread +// that runs out of pieces to start goes on to tiles; only a tile that needs a piece still being +// computed waits for it. The map itself doesn't change, so the tiles look pieces up without a lock. #define SHARED_PIECE_MIN_POINTS 10000 void TileDataSource::precomputeSharedPieces(const std::function &present, unsigned threadNum) { @@ -125,7 +126,7 @@ void TileDataSource::precomputeSharedPieces(const std::function> candidates(boxes.begin(), boxes.end()); // The pieces each candidate needs: tiles below which a tile is written, that its envelope reaches - struct Job { std::tuple key; uint64_t points; }; + using Job = PieceJob; std::vector jobs; std::mutex jobsMutex; std::atomic next(0); @@ -148,7 +149,7 @@ void TileDataSource::precomputeSharedPieces(const std::function lock(jobsMutex); jobs.insert(jobs.end(), local.begin(), local.end()); @@ -159,40 +160,37 @@ void TileDataSource::precomputeSharedPieces(const std::function b.points : a.key < b.key; }); - for (const Job &job : jobs) { + for (Job &job : jobs) { pieceObjects.insert(std::get<0>(job.key)); - sharedPieces.emplace(job.key, nullptr); + job.promise = std::make_shared>>(); + sharedPieces.emplace(job.key, job.promise->get_future().share()); } + pieceJobs = std::move(jobs); + nextPieceJob = 0; +} - // Compute them, largest polygon first. Each job writes its own map entry; the map itself - // doesn't change any more. - next = 0; - auto compute = [&]() { - for (size_t i = next++; i < jobs.size(); i = next++) { - const auto &key = jobs[i].key; - auto piece = std::make_shared(); - try { - clipCacheBypass = true; - clipNeededRepair = false; - Geometry g = buildWayGeometry(POLYGON_, std::get<0>(key), TileBbox(TileCoordinates(std::get<1>(key), std::get<2>(key)), zoom, false, false)); - // A piece that needed the repair path can be wrong (once a lake came out empty), and - // every tile below it would inherit that: leave those tiles to clip the whole polygon. - if (!clipNeededRepair) { - piece->geom = std::make_shared(std::move(boost::get(g))); - // buildWayGeometry has just checked it (for a huge piece, is_valid takes seconds) - piece->valid = clipResultValid; - } - } catch (std::exception &) { - // The tiles clip the whole polygon, and report the problem there +void TileDataSource::computeSharedPieces() { + const unsigned zoom = SHARED_PIECE_ZOOM; + for (size_t i = nextPieceJob++; i < pieceJobs.size(); i = nextPieceJob++) { + const auto &key = pieceJobs[i].key; + auto piece = std::make_shared(); + try { + clipCacheBypass = true; + clipNeededRepair = false; + Geometry g = buildWayGeometry(POLYGON_, std::get<0>(key), TileBbox(TileCoordinates(std::get<1>(key), std::get<2>(key)), zoom, false, false)); + // A piece that needed the repair path can be wrong (once a lake came out empty), and + // every tile below it would inherit that: leave those tiles to clip the whole polygon. + if (!clipNeededRepair) { + piece->geom = std::make_shared(std::move(boost::get(g))); + // buildWayGeometry has just checked it (for a huge piece, is_valid takes seconds) + piece->valid = clipResultValid; } - clipCacheBypass = false; - sharedPieces.find(key)->second = piece; + } catch (std::exception &) { + // The tiles clip the whole polygon, and report the problem there } - }; - { - boost::asio::thread_pool pool(threadNum); - for (unsigned t = 0; t < threadNum; t++) boost::asio::post(pool, compute); - pool.join(); + clipCacheBypass = false; + pieceJobs[i].promise->set_value(piece); + pieceJobs[i].promise.reset(); // the map's future keeps the piece, until its z6 tile is released } } @@ -200,7 +198,7 @@ void TileDataSource::releaseSharedPieces(TileCoordinate z6x, TileCoordinate z6y) const unsigned shift = SHARED_PIECE_ZOOM - CLUSTER_ZOOM; for (auto &e : sharedPieces) if ((std::get<1>(e.first) >> shift) == z6x && (std::get<2>(e.first) >> shift) == z6y) - e.second.reset(); + e.second = std::shared_future>(); } void TileDataSource::finalize(size_t threadNum) { @@ -449,9 +447,12 @@ Geometry TileDataSource::buildWayGeometry(OutputGeometryType const geomType, // A large polygon: clip from its shared piece const unsigned shift = bbox.zoom - SHARED_PIECE_ZOOM; auto it = sharedPieces.find(std::make_tuple(objectID, bbox.index.x >> shift, bbox.index.y >> shift)); - if (it != sharedPieces.end() && it->second && it->second->geom) { - cachedClip = it->second->geom; - inputKnownValid = it->second->valid; + if (it != sharedPieces.end() && it->second.valid()) { + const std::shared_ptr &piece = it->second.get(); // waits if still being computed + if (piece && piece->geom) { + cachedClip = piece->geom; + inputKnownValid = piece->valid; + } } } diff --git a/src/tilemaker.cpp b/src/tilemaker.cpp index a2643cac..f118840d 100644 --- a/src/tilemaker.cpp +++ b/src/tilemaker.cpp @@ -598,6 +598,8 @@ int main(const int argc, const char* argv[]) { boost::asio::post(pool, [&]() { std::size_t startIndex, batchSize; uint64_t seq; + // Shared pieces first (see TileDataSource::precomputeSharedPieces) + for (auto source : sources) source->computeSharedPieces(); while (true) { if (toPMTiles) sharedData.pmtiles.batches.waitForRoom(); if (toMBTiles) sharedData.mbtiles.batches.waitForRoom(); From 697a6ff30c4ff97c91fefa3593361f35112b880c Mon Sep 17 00:00:00 2001 From: dktue <2878623+dktue@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:38:03 +0100 Subject: [PATCH 7/8] build: fix Windows and macOS builds of the combined base - polylabel.h: rename edgeDistBound's parameter `near` to `nearCell`. tile_data.h includes boost/asio since #972, which pulls in windows.h on Windows, and windows.h defines `near` as an empty macro. - test/shp_mem_tiles.test.cpp (#979): include shp_mem_tiles.h before minunit.h. minunit.h sets _POSIX_C_SOURCE, which on macOS hides ip_mreq, and boost/asio (through tile_data.h) needs it. Renames and include order only; output unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) --- include/polylabel.h | 10 +++++----- test/shp_mem_tiles.test.cpp | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/polylabel.h b/include/polylabel.h index 6ce44f16..30de7a6d 100644 --- a/include/polylabel.h +++ b/include/polylabel.h @@ -329,13 +329,13 @@ Cell getCentroidCell(const Polygon& polygon) { return Cell(area == 0 ? ring.at(0) : Point { ox + cx / area, oy + cy / area }, 0, polygon); } -// Upper bound on pointToPolygonDist(p, polygon), from the edge nearest to `near.c`. The full scan +// Upper bound on pointToPolygonDist(p, polygon), from the edge nearest to `nearCell.c`. The full scan // computes min over all edges of getSegDistSq(p, a, b) with the same arguments, so its result is // <= this edge's value; the margin covers FP contraction differences between call sites. -inline double edgeDistBound(const Point& p, const Cell& near, const Polygon& polygon) { - const auto& ring = near.kr < 0 ? polygon.outer() : polygon.inners()[near.kr]; - const Point& a = ring[near.ki]; - const Point& b = ring[near.ki == 0 ? ring.size() - 1 : near.ki - 1]; +inline double edgeDistBound(const Point& p, const Cell& nearCell, const Polygon& polygon) { + const auto& ring = nearCell.kr < 0 ? polygon.outer() : polygon.inners()[nearCell.kr]; + const Point& a = ring[nearCell.ki]; + const Point& b = ring[nearCell.ki == 0 ? ring.size() - 1 : nearCell.ki - 1]; return std::sqrt(getSegDistSq(p, a, b)) * (1 + 1e-12) + 1e-11; } diff --git a/test/shp_mem_tiles.test.cpp b/test/shp_mem_tiles.test.cpp index a0cbcb2f..b4629162 100644 --- a/test/shp_mem_tiles.test.cpp +++ b/test/shp_mem_tiles.test.cpp @@ -1,6 +1,6 @@ #include -#include "external/minunit.h" #include "shp_mem_tiles.h" +#include "external/minunit.h" bool verbose = false; From 4c0023f510cdd7da666bc9b8d2b19601ea5edb4e Mon Sep 17 00:00:00 2001 From: dktue <2878623+dktue@users.noreply.github.com> Date: Sun, 15 Nov 2026 16:21:20 +0100 Subject: [PATCH 8/8] fix: keep the allocator's region lists alive until thread-local destructors have run (macOS) On macOS the main thread's thread-local destructors run after the static destructors. Two of them touched objects that were already gone, and the unit tests aborted at exit with "mutex lock failed: Invalid argument": - mmap_allocator.cpp: thread_regions_t's destructor locks mmap_allocator_mutex and appends to free_mmap_files / free_mmap_shm_regions. These three are now allocated once and never destroyed. ~mmap_dir_t clears free_mmap_files first, so --store files are still released and deleted as before (that list used to be destroyed just before mmap_dir). - LeasedStore (tile_data.cpp): a thread gives its store leases back to the TileDataSource when it ends; the main thread does that at exit, after the sources (locals of main(), or of a unit test such as test_shp_mem_tiles) are destroyed, locking a destroyed storeMutex. The sources are now registered while alive (an immortal registry), and a lease goes back only to a live source. Sources are also numbered: a new source at the address of a destroyed one no longer gets that one's lease. No change in output. Co-Authored-By: Claude Opus 5.5 (1M context) --- include/leased_store.h | 16 ++++++++++------ include/tile_data.h | 10 ++++++++++ src/mmap_allocator.cpp | 14 +++++++++++--- src/tile_data.cpp | 29 +++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 9 deletions(-) diff --git a/include/leased_store.h b/include/leased_store.h index 0349d64d..019b110d 100644 --- a/include/leased_store.h +++ b/include/leased_store.h @@ -2,6 +2,7 @@ #define _LEASED_STORE_H #include "tile_data.h" +#include // LeasedStore implements "leases" for generated geometries. // @@ -45,23 +46,26 @@ inline std::vector>& g template class LeasedStore { - std::vector>> leases; + std::vector>> leases; // source, its leaseNumber, lease public: ~LeasedStore() { + // Only to the sources still alive (see registerLeaseSource) + std::lock_guard live(leaseSourcesMutex()); for (const auto& lease : leases) { - auto source = lease.first; + auto source = std::get<0>(lease); + if (!leaseSourceAlive(source, std::get<1>(lease))) continue; std::lock_guard lock(source->storeMutex); std::vector>& availableLeases = getAvailableLeases(source); - availableLeases.push_back(lease.second); + availableLeases.push_back(std::get<2>(lease)); } } std::pair get(TileDataSource* source) { for (const auto& lease : leases) { - if (lease.first == source) - return lease.second; + if (std::get<0>(lease) == source && std::get<1>(lease) == source->leaseNumber) + return std::get<2>(lease); } std::lock_guard lock(source->storeMutex); @@ -74,7 +78,7 @@ class LeasedStore { std::pair entry = availableLeases.back(); availableLeases.pop_back(); - leases.push_back(std::make_pair(source, entry)); + leases.push_back(std::make_tuple(source, source->leaseNumber, entry)); return entry; } diff --git a/include/tile_data.h b/include/tile_data.h index 8d3f3d41..0adf6d95 100644 --- a/include/tile_data.h +++ b/include/tile_data.h @@ -24,6 +24,14 @@ typedef std::vector SourceList; +// The TileDataSources alive, for the store leases a thread keeps until it ends (leased_store.h): the +// main thread gives its leases back at exit, after the sources are gone. A source is also told apart +// by its number, as a new source can be at the address of a destroyed one. +uint64_t registerLeaseSource(const TileDataSource *source); +void unregisterLeaseSource(const TileDataSource *source); +std::mutex &leaseSourcesMutex(); +bool leaseSourceAlive(const TileDataSource *source, uint64_t number); // with leaseSourcesMutex() held + class TileBbox; template void sortOutputObjects( @@ -375,6 +383,8 @@ class TileDataSource { public: TileDataSource(size_t threadNum, unsigned int indexZoom, bool includeID); + ~TileDataSource(); + const uint64_t leaseNumber = registerLeaseSource(this); // see registerLeaseSource void collectTilesWithObjectsAtZoom(std::vector>& zooms); // The same for the objects of one z6 tile (0 <= shard < CLUSTER_ZOOM_AREA) diff --git a/src/mmap_allocator.cpp b/src/mmap_allocator.cpp index 4daf0c00..33ced307 100644 --- a/src/mmap_allocator.cpp +++ b/src/mmap_allocator.cpp @@ -98,13 +98,15 @@ static mmap_dir_t mmap_dir; std::vector< mmap_shm_ptr > mmap_shm_regions; size_t mmap_shm::mmap_file_size = 0; -std::mutex mmap_allocator_mutex; +// Never destroyed (nor the two lists below): thread_regions_t's destructor uses them, and the +// main thread's thread-locals are destroyed after the statics on some platforms (macOS) +std::mutex &mmap_allocator_mutex = *new std::mutex; // Regions of threads that have exited, for reuse by new threads. Every thread // pool (one per .shp/GeoJSON layer, per .pbf read phase) starts new threads, and // a new region (or 1GB store file) for each of them would never be released. -std::vector free_mmap_files; -std::vector free_mmap_shm_regions; +std::vector &free_mmap_files = *new std::vector(); +std::vector &free_mmap_shm_regions = *new std::vector(); // The regions the current thread allocates from struct thread_regions_t @@ -195,6 +197,12 @@ bool mmap_dir_t::is_open() { if(!mmap_dir_filename.empty()) { try { + // The files of exited threads, kept for reuse: let them go first, so that they are + // deleted below (free_mmap_files is never destroyed) + { + std::lock_guard lock(mmap_allocator_mutex); + free_mmap_files.clear(); + } // Unmapping and deleting a big store (planet: ~140 GB in ~300 files) // takes half a minute on one thread, so release the files from several. const size_t threads = std::min(files.size(), std::min(16u, std::max(1u, std::thread::hardware_concurrency()))); diff --git a/src/tile_data.cpp b/src/tile_data.cpp index 1f100c54..866b853a 100644 --- a/src/tile_data.cpp +++ b/src/tile_data.cpp @@ -7,6 +7,7 @@ #include "tile_data.h" #include "coordinates_geom.h" #include "leased_store.h" +#include #include "stable_hash.h" #include @@ -48,6 +49,34 @@ thread_local LeasedStore linestringStore; thread_local LeasedStore multilinestringStore; thread_local LeasedStore multipolygonStore; +// Never destroyed: the thread-locals that use them may be destroyed after the statics (macOS) +static std::unordered_map &leaseSources() { + static std::unordered_map &sources = *new std::unordered_map(); + return sources; +} +std::mutex &leaseSourcesMutex() { + static std::mutex &m = *new std::mutex; + return m; +} +uint64_t registerLeaseSource(const TileDataSource *source) { + static uint64_t last = 0; + std::lock_guard lock(leaseSourcesMutex()); + const uint64_t number = ++last; + leaseSources()[source] = number; + return number; +} +void unregisterLeaseSource(const TileDataSource *source) { + std::lock_guard lock(leaseSourcesMutex()); + leaseSources().erase(source); +} +bool leaseSourceAlive(const TileDataSource *source, uint64_t number) { + const auto it = leaseSources().find(source); + return it != leaseSources().end() && it->second == number; +} +TileDataSource::~TileDataSource() { + unregisterLeaseSource(this); +} + TileDataSource::TileDataSource(size_t threadNum, unsigned int indexZoom, bool includeID) : includeID(includeID),