From 36c946614d64754621a62acf7008ce975c745621 Mon Sep 17 00:00:00 2001 From: dktue <2878623+dktue@users.noreply.github.com> Date: Sat, 7 Nov 2026 14:48:56 +0100 Subject: [PATCH 01/10] test: master + all open performance PRs + #981 + #942 + #983 (combined snapshot) For testing only, not for merging. One commit with the combined tree that built the planet in 14 min 03 s (in memory, 96 threads) for #977: master 2789651 + #933, #959, #966, #968, #970, #971, #972, #973, #974, #975, #976, #978, #979 (open performance PRs), #981 (deterministic tile content), #942 (deterministic POI order) and #983 (ordered .pmtiles/.mbtiles writer, 64 MB per thread). Each change is reviewed in its own PR. Co-Authored-By: Claude Opus 5.5 (1M context) --- Makefile | 27 +++++ include/append_vector.h | 4 + include/attribute_store.h | 27 ++++- include/coordinates_geom.h | 1 + include/geometry/correct.hpp | 92 +++++++++----- include/mbtiles.h | 6 + include/ordered_batches.h | 113 ++++++++++++++++++ include/osm_store.h | 15 ++- include/pmtiles.h | 20 +++- include/shp_mem_tiles.h | 3 +- include/stable_hash.h | 42 +++++++ include/tile_coordinates_set.h | 6 + include/tile_data.h | 157 +++++++++++++++++------- resources/process-debug.lua | 7 +- resources/process-openmaptiles.lua | 7 +- src/attribute_store.cpp | 56 +++++++++ src/coordinates_geom.cpp | 29 ++++- src/geom.cpp | 148 ++++++++++++++++++----- src/mbtiles.cpp | 40 ++++++- src/mmap_allocator.cpp | 23 ++++ src/osm_store.cpp | 55 ++++++++- src/output_object.cpp | 6 +- src/pmtiles.cpp | 59 ++++----- src/shp_mem_tiles.cpp | 61 ++++++++-- src/shp_processor.cpp | 20 ++-- src/tile_coordinates_set.cpp | 56 +++++++++ src/tile_data.cpp | 185 ++++++++++++++++++++++++++--- src/tile_sorting.cpp | 73 ++++++++++-- src/tilemaker.cpp | 149 ++++++++++++++--------- test/append_vector.test.cpp | 6 + test/correct.test.cpp | 72 +++++++++++ test/geom.test.cpp | 39 ++++++ test/ordered_batches.test.cpp | 89 ++++++++++++++ test/shp_mem_tiles.test.cpp | 66 ++++++++++ test/tile_coordinates_set.test.cpp | 61 ++++++++++ 35 files changed, 1566 insertions(+), 254 deletions(-) create mode 100644 include/ordered_batches.h create mode 100644 include/stable_hash.h create mode 100644 test/correct.test.cpp create mode 100644 test/ordered_batches.test.cpp create mode 100644 test/shp_mem_tiles.test.cpp diff --git a/Makefile b/Makefile index 57227d4c..b03576c2 100644 --- a/Makefile +++ b/Makefile @@ -156,13 +156,16 @@ test: \ test_append_vector \ test_attribute_store \ test_config_validator \ + test_correct \ test_deque_map \ test_geom \ test_helpers \ test_options_parser \ + test_ordered_batches \ test_pbf_reader \ test_pooled_string \ test_relation_roles \ + test_shp_mem_tiles \ test_significant_tags \ test_sorted_node_store \ test_sorted_way_store \ @@ -193,6 +196,10 @@ src/config_schema.h: resources/config-schema.json src/config_validator.o: src/config_schema.h +test_correct: \ + test/correct.test.o + $(CXX) $(CXXFLAGS) -o test.correct $^ $(INC) $(LIB) $(LDFLAGS) && ./test.correct + test_deque_map: \ test/deque_map.test.o $(CXX) $(CXXFLAGS) -o test.deque_map $^ $(INC) $(LIB) $(LDFLAGS) && ./test.deque_map @@ -223,6 +230,10 @@ test_options_parser: \ test/options_parser.test.o $(CXX) $(CXXFLAGS) -o test.options_parser $^ $(INC) $(LIB) $(LDFLAGS) && ./test.options_parser +test_ordered_batches: \ + test/ordered_batches.test.o + $(CXX) $(CXXFLAGS) -o test.ordered_batches $^ $(INC) $(LIB) $(LDFLAGS) && ./test.ordered_batches + test_osm_store: \ test/osm_store.test.o $(CXX) $(CXXFLAGS) -o test.osm_store $^ $(INC) $(LIB) $(LDFLAGS) && ./test.osm_store @@ -262,6 +273,22 @@ test_sorted_way_store: \ test/sorted_way_store.test.o $(CXX) $(CXXFLAGS) -o test.sorted_way_store $^ $(INC) $(LIB) $(LDFLAGS) && ./test.sorted_way_store +test_shp_mem_tiles: \ + src/attribute_store.o \ + src/coordinates.o \ + src/coordinates_geom.o \ + src/declutter.o \ + src/geom.o \ + src/mmap_allocator.o \ + src/output_object.o \ + src/pooled_string.o \ + src/shp_mem_tiles.o \ + src/tile_coordinates_set.o \ + src/tile_data.o \ + src/tile_sorting.o \ + test/shp_mem_tiles.test.o + $(CXX) $(CXXFLAGS) -o test.shp_mem_tiles $^ $(INC) $(LIB) $(LDFLAGS) && ./test.shp_mem_tiles + test_tile_coordinates_set: \ src/tile_coordinates_set.o \ test/tile_coordinates_set.test.o diff --git a/include/append_vector.h b/include/append_vector.h index 28b29a8b..8a66add9 100644 --- a/include/append_vector.h +++ b/include/append_vector.h @@ -162,6 +162,10 @@ namespace AppendVectorNS { return el; } + const T& operator [](int idx) const { + return vecs[idx / APPEND_VECTOR_SIZE][idx % APPEND_VECTOR_SIZE]; + } + Iterator begin() { return Iterator(*this, 0, 0); } diff --git a/include/attribute_store.h b/include/attribute_store.h index 295c495c..849b3889 100644 --- a/include/attribute_store.h +++ b/include/attribute_store.h @@ -9,6 +9,8 @@ #include #include #include +#include +#include #include #include #include "pooled_string.h" @@ -30,11 +32,16 @@ class AttributeKeyStore { uint16_t key2index(const std::string& key); const std::string& getKey(uint16_t index) const; const std::string& getKeyUnsafe(uint16_t index) const; - void finalize() { finalized = true; } + // Position of the key in alphabetical order. Unlike the key index, which is + // handed out in first-use order (i.e. depends on thread scheduling), it is + // the same in every run. Only valid after finalize(). + uint16_t getRankUnsafe(uint16_t index) const { return ranks[index]; } + void finalize(); std::atomic keys2indexSize; private: bool finalized; + std::vector ranks; mutable std::mutex keys2indexMutex; // NB: we use a deque, not a vector, because a deque never invalidates // pointers to its members as long as you only push_back @@ -439,6 +446,24 @@ struct AttributeStore { } attrSet->forEachPair(fn); } + // As forEachPairIndexUnsafe, but in an order that is the same in every run + // (by key name), not by pair index, which depends on thread scheduling. + template + void forEachPairIndexCanonicalUnsafe(AttributeIndex index, F&& fn) const { + boost::container::small_vector order; + forEachPairIndexUnsafe(index, [&](uint32_t pairIndex) { order.push_back(pairIndex); }); + if (order.size() > 1) + std::sort(order.begin(), order.end(), [this](uint32_t a, uint32_t b) { + return comparePairsUnsafe(a, b) < 0; + }); + for (const uint32_t pairIndex : order) + fn(pairIndex); + } + // Order and hash attribute pairs/sets by content (key names and values), not by + // index, so that they are the same in every run. Only valid after finalize(). + int comparePairsUnsafe(uint32_t a, uint32_t b) const; + int compareContentUnsafe(AttributeIndex a, AttributeIndex b) const; + uint64_t contentHashUnsafe(AttributeIndex index) const; void reset(); // used for testing size_t size() const; void reportSize() const; diff --git a/include/coordinates_geom.h b/include/coordinates_geom.h index f233ad3e..acf47b9f 100644 --- a/include/coordinates_geom.h +++ b/include/coordinates_geom.h @@ -28,6 +28,7 @@ class TileBbox { std::pair scaleLatpLon(double latp, double lon) const; void scaleRing(Ring &dst, Ring const &src) const; Ring scaleRing(Ring const &src) const; + void scaleRingNoBacktrack(Ring &dst, Ring const &src) const; void scaleGeometry(MultiPolygon &dst, MultiPolygon const &src) const; MultiPolygon scaleGeometry(MultiPolygon const &src) const; std::pair floorLatpLon(double latp, double lon) const; diff --git a/include/geometry/correct.hpp b/include/geometry/correct.hpp index fba96c69..b26c74af 100644 --- a/include/geometry/correct.hpp +++ b/include/geometry/correct.hpp @@ -10,6 +10,8 @@ * ---------------------------------------------------------------------------- */ +#include +#include #include #include #include @@ -23,13 +25,18 @@ namespace geometry { namespace impl { -template -static inline void result_combine(C &result, T &&new_element) +// boxes[i] is the envelope of result[i]. Polygons whose envelopes are disjoint +// cannot intersect, so the box test skips most of the costly intersects() calls +// (otherwise quadratic in the number of rings, e.g. a lake with 14000 islands). +template +static inline void result_combine(C &result, std::vector &boxes, T &&new_element) { result.push_back(std::forward(new_element)); + boxes.push_back(boost::geometry::return_envelope(result.back())); for(std::size_t i = 0; i < result.size() - 1; ) { - if(!boost::geometry::intersects(result[i], result.back())) { + if(boost::geometry::disjoint(boxes[i], boxes.back()) || + !boost::geometry::intersects(result[i], result.back())) { ++i; continue; } @@ -43,8 +50,10 @@ static inline void result_combine(C &result, T &&new_element) } result.back() = std::move(union_result[0]); + boxes.back() = boost::geometry::return_envelope(result.back()); result.erase(result.begin() + i); - } + boxes.erase(boxes.begin() + i); + } } struct pseudo_vertice_key @@ -199,27 +208,40 @@ static inline std::vector dissolve_generate_rings( // Generate all polygons by tracing all the intersections // Perform union to combine all polygons into single polygon again - while(!start_keys.empty()) { - ring_t new_ring; - - // Store point in generated polygon - auto push_point = [&new_ring](auto const &p) { - if(new_ring.empty() || boost::geometry::comparable_distance(new_ring.back(), p) > 0) - new_ring.push_back(p); - }; + // Position of each point in the ring being traced. Its points are distinct until + // it closes, so a point already seen closes it: looking that up here instead of + // scanning the whole ring for every new point keeps tracing linear, not quadratic. + struct point_hash { + std::size_t operator()(std::pair const &p) const { + return std::hash()(p.first) * 31 + std::hash()(p.second); + } + }; + std::unordered_map, std::size_t, point_hash> seen; - auto is_closed = [](ring_t &ring) { - if(ring.size() < 2) return false; - for(std::size_t i = 0; i < ring.size() - 1; ++i) { - if(boost::geometry::comparable_distance(ring[i], ring.back()) == 0) { - ring.erase(ring.begin(), ring.begin() + i); - return true; - } + while(!start_keys.empty()) { + ring_t new_ring; + seen.clear(); + bool closed = false; + + // Store point in generated polygon; when it repeats an earlier point, the + // ring is closed and the points before that one are dropped + auto push_point = [&new_ring, &seen, &closed](auto const &p) { + if(!new_ring.empty() && !(boost::geometry::comparable_distance(new_ring.back(), p) > 0)) + return; + new_ring.push_back(p); + // + 0.0 turns -0.0 into 0.0, which compares equal to it + std::pair key(boost::geometry::get<0>(p) + 0.0, boost::geometry::get<1>(p) + 0.0); + auto found = seen.find(key); + if(found == seen.end()) { + seen.emplace(key, new_ring.size() - 1); + } else { + new_ring.erase(new_ring.begin(), new_ring.begin() + found->second); + closed = true; } - - return false; }; + auto is_closed = [&closed](ring_t &) { return closed; }; + auto start_iter = pseudo_vertices.find(*start_keys.begin()); auto i = start_iter; @@ -302,13 +324,15 @@ template< > struct combine_non_zero_winding { - inline void operator()(multi_polygon_t &combined_outers, multi_polygon_t &combined_inners, polygon_t &poly) + using box_t = boost::geometry::model::box; + + inline void operator()(multi_polygon_t &combined_outers, std::vector &outer_boxes, multi_polygon_t &combined_inners, std::vector &inner_boxes, polygon_t &poly) { if(boost::geometry::area(poly) > 0) - result_combine(combined_outers, std::move(poly)); + result_combine(combined_outers, outer_boxes, std::move(poly)); else { std::reverse(poly.outer().begin(), poly.outer().end()); - result_combine(combined_inners, std::move(poly)); + result_combine(combined_inners, inner_boxes, std::move(poly)); } } }; @@ -320,7 +344,9 @@ template< > struct combine_odd_even { - inline void operator()(multi_polygon_t &combined_outers, multi_polygon_t &combined_inners, polygon_t &poly) + using box_t = boost::geometry::model::box; + + inline void operator()(multi_polygon_t &combined_outers, std::vector &, multi_polygon_t &combined_inners, std::vector &, polygon_t &poly) { if(boost::geometry::area(poly) < 0) std::reverse(poly.outer().begin(), poly.outer().end()); @@ -346,11 +372,12 @@ static inline void correct(polygon_t const &input, multi_polygon_t &output, doub // Calculate all outers and combine them if possible multi_polygon_t combined_outers; multi_polygon_t combined_inners; + std::vector> outer_boxes, inner_boxes; for(auto &ring: outer_rings) { polygon_t poly; poly.outer() = std::move(ring); - combine(combined_outers, combined_inners, poly); + combine(combined_outers, outer_boxes, combined_inners, inner_boxes, poly); } // Calculate all inners and combine them if possible @@ -362,8 +389,8 @@ static inline void correct(polygon_t const &input, multi_polygon_t &output, doub correct(poly, new_inners, remove_spike_min_area, combine); for(auto &poly: new_inners) { - result_combine(combined_inners, std::move(poly)); - } + result_combine(combined_inners, inner_boxes, std::move(poly)); + } } // Cut out all inners from all the outers @@ -379,13 +406,18 @@ template< > static inline void correct(multi_polygon_t const &input, multi_polygon_t &output, double remove_spike_min_area, combine_function_t combine) { + using box_t = boost::geometry::model::box; + std::vector output_boxes; + for(auto const &polygon: output) + output_boxes.push_back(boost::geometry::return_envelope(polygon)); + for(auto const &polygon: input) { multi_polygon_t new_polygons; correct(polygon, new_polygons, remove_spike_min_area, combine); - for(auto &new_polygon: new_polygons) - result_combine(output, std::move(new_polygon)); + for(auto &new_polygon: new_polygons) + result_combine(output, output_boxes, std::move(new_polygon)); } } diff --git a/include/mbtiles.h b/include/mbtiles.h index ca671b07..8181fafa 100644 --- a/include/mbtiles.h +++ b/include/mbtiles.h @@ -3,9 +3,11 @@ #define _MBTILES_H #include +#include #include #include #include "external/sqlite_modern_cpp.h" +#include "ordered_batches.h" struct PendingStatement { int zoom; @@ -27,6 +29,7 @@ class MBTiles { std::shared_ptr> pendingStatements1, pendingStatements2; std::mutex pendingStatementsMutex; + std::atomic pendingBytes; void insertOrReplace(int zoom, int x, int y, const std::string& data, bool isMerge); void flushPendingStatements(); @@ -39,6 +42,9 @@ class MBTiles { void saveTile(int zoom, int x, int y, std::string *data, bool isMerge); void closeForWriting(); + // Tiles are written in batch order, so the file doesn't depend on thread scheduling + OrderedBatches batches; + void openForReading(std::string &filename); void readBoundingBox(double &minLon, double &maxLon, double &minLat, double &maxLat); void readTileList(std::vector> &tileList); diff --git a/include/ordered_batches.h b/include/ordered_batches.h new file mode 100644 index 00000000..2d2b3b90 --- /dev/null +++ b/include/ordered_batches.h @@ -0,0 +1,113 @@ +/*! \file */ +#ifndef _ORDERED_BATCHES_H +#define _ORDERED_BATCHES_H + +#include +#include +#include +#include +#include +#include +#include + +// Writes batches of output in the order they were handed out, not the order threads +// finish them, so the output file doesn't depend on thread scheduling. +// +// A thread collects a batch's items between begin() and end(). end() writes the batch +// if it's the next one, followed by any batches that were waiting for it; otherwise it +// holds the batch. Batches must be numbered 0, 1, 2, ... in the order they're handed out, +// and a thread calls waitForRoom() before it takes its next batch: that keeps what is +// held below the limit set with setMaxHeldBytes(). The batch everyone waits for is always +// being worked on by a thread that isn't waiting, so waitForRoom() always returns. +// +// writeItem is called by one thread at a time, with writeMutex() held, in batch order. +template +class OrderedBatches { +public: + OrderedBatches(std::function writeItem, std::function itemBytes) + : writeItem(writeItem), itemBytes(itemBytes) { } + + void setMaxHeldBytes(uint64_t bytes) { + std::lock_guard lock(m); + maxHeldBytes = bytes; + } + + bool inBatch() const { return current().active; } + + void begin(uint64_t seq) { + Current& c = current(); + c.active = true; + c.seq = seq; + c.items.clear(); + } + + void add(T&& item) { current().items.emplace_back(std::move(item)); } + + // Hand the batch over. If it's the next one to be written and no other thread is + // writing, this thread writes it and then any held batches that follow it; meanwhile + // other threads just hand their batches over and carry on. + void end() { + Current& c = current(); + c.active = false; + uint64_t bytes = 0; + for (const auto& item : c.items) bytes += itemBytes(item); + std::unique_lock lock(m); + heldBytes += bytes; + held.emplace(c.seq, std::make_pair(bytes, std::move(c.items))); + c.items = std::vector(); + if (draining) return; + draining = true; + while (!held.empty() && held.begin()->first == nextSeq) { + auto batch = std::move(held.begin()->second); + held.erase(held.begin()); + lock.unlock(); + { + std::lock_guard writeLock(w); + for (auto& item : batch.second) writeItem(item); + } + lock.lock(); + heldBytes -= batch.first; + nextSeq++; + heldShrunk.notify_all(); + } + draining = false; + } + + void waitForRoom() { + std::unique_lock lock(m); + heldShrunk.wait(lock, [&]() { return heldBytes <= maxHeldBytes; }); + } + + bool allWritten() { + std::lock_guard lock(m); + return held.empty(); + } + + // Held while items are written; lock it to write anything outside a batch + std::mutex& writeMutex() { return w; } + +private: + struct Current { + bool active = false; + uint64_t seq = 0; + std::vector items; + }; + // The batch this thread is working on (one per thread and item type) + static Current& current() { + static thread_local Current c; + return c; + } + + std::function writeItem; + std::function itemBytes; + uint64_t maxHeldBytes = 1ull << 30; + std::mutex m; // guards the fields below + std::mutex w; // held while writing + std::condition_variable heldShrunk; + bool draining = false; + uint64_t nextSeq = 0; + uint64_t heldBytes = 0; + std::map>> held; // seq -> (bytes, items) +}; + +#endif //_ORDERED_BATCHES_H diff --git a/include/osm_store.h b/include/osm_store.h index 4f50cd5f..1f9915ec 100644 --- a/include/osm_store.h +++ b/include/osm_store.h @@ -154,22 +154,31 @@ class RelationScanStore { RelationScanStore(): relationsForWays(128), relationsForNodes(128), relationTags(128), mutex(128) {} + // Keep each member's parent relations ordered by relation id: relations are scanned on + // several threads, so appending would make the order NextRelation() returns depend on + // thread scheduling. Entries of the same relation keep their member order. + static void insertByRelation(std::vector>& list, RelationID relid, uint16_t roleId) { + const auto it = std::upper_bound(list.begin(), list.end(), relid, + [](RelationID id, const std::pair& entry) { return id < entry.first; }); + list.emplace(it, relid, roleId); + } + void relation_contains_way(RelationID relid, WayID wayid, std::string role) { uint16_t roleId = relationRoles.getOrAddRole(role); const size_t shard = wayid % mutex.size(); std::lock_guard lock(mutex[shard]); - relationsForWays[shard][wayid].emplace_back(std::make_pair(relid, roleId)); + insertByRelation(relationsForWays[shard][wayid], relid, roleId); } void relation_contains_node(RelationID relid, NodeID nodeId, std::string role) { uint16_t roleId = relationRoles.getOrAddRole(role); const size_t shard = nodeId % mutex.size(); std::lock_guard lock(mutex[shard]); - relationsForNodes[shard][nodeId].emplace_back(std::make_pair(relid, roleId)); + insertByRelation(relationsForNodes[shard][nodeId], relid, roleId); } void relation_contains_relation(RelationID relid, RelationID relationId, std::string role) { uint16_t roleId = relationRoles.getOrAddRole(role); std::lock_guard lock(mutex[0]); - relationsForRelations[relationId].emplace_back(std::make_pair(relid, roleId)); + insertByRelation(relationsForRelations[relationId], relid, roleId); } void store_relation_tags(RelationID relid, const tag_map_t &tags) { const size_t shard = relid % mutex.size(); diff --git a/include/pmtiles.h b/include/pmtiles.h index 7e8040c8..bcc701f1 100644 --- a/include/pmtiles.h +++ b/include/pmtiles.h @@ -3,7 +3,12 @@ #define _PMTILES_H #include +#include +#include +#include +#include #include "external/pmtiles.hpp" +#include "ordered_batches.h" struct TileOffset { uint64_t offset : 40; @@ -28,6 +33,14 @@ struct TileOffset { // PMTiles v3 tile type for MapLibre Tiles (not yet in external/pmtiles.hpp) #define TILETYPE_MLT 0x6 +// A tile waiting to be written: tiny tiles keep their raw data, so that a repeat +// can reuse the earlier copy, and are only compressed if they're written +struct PendingTile { + uint64_t tileId; + bool isTiny; + std::string data; +}; + class PMTiles { public: @@ -42,10 +55,12 @@ class PMTiles { void saveTile(int zoom, int x, int y, std::string &data); void close(std::string &metadata); + // Tiles are written in batch order, so the file doesn't depend on thread scheduling + OrderedBatches batches; + private: std::ofstream outputStream; - std::mutex fileMutex; // guards file writes, numTilesWritten - std::mutex indexMutex; // guards access to sparseIndex, denseIndex, tinyCache, numTilesAddressed + std::mutex fileMutex; // guards leaf directory writes in close() uint64_t leafStart = 0; uint64_t numTilesWritten = 0; uint64_t numTilesAddressed = 0; @@ -54,6 +69,7 @@ class PMTiles { std::vector denseIndex; std::unordered_map tinyCache; + void writeTile(PendingTile &tile); // with batches.writeMutex() held void appendWithRLE(std::vector &entries, pmtiles::entryv3 &entry); void appendTileEntry(uint64_t tileId, TileOffset offset, std::vector &rootEntries, std::vector &entries); void flushEntries(std::vector &rootEntries, std::vector &entries); diff --git a/include/shp_mem_tiles.h b/include/shp_mem_tiles.h index a5cf4f3f..0bab561c 100644 --- a/include/shp_mem_tiles.h +++ b/include/shp_mem_tiles.h @@ -27,7 +27,8 @@ class ShpMemTiles : public TileDataSource const std::string& name, uint minzoom, int32_t score, - AttributeIndex attrIdx + AttributeIndex attrIdx, + bool knownValid = false // polygon: caller has checked it is valid ); std::vector QueryMatchingGeometries( diff --git a/include/stable_hash.h b/include/stable_hash.h new file mode 100644 index 00000000..e1af029f --- /dev/null +++ b/include/stable_hash.h @@ -0,0 +1,42 @@ +/*! \file */ +#ifndef _STABLE_HASH_H +#define _STABLE_HASH_H + +#include +#include +#include + +// Hash functions that give the same result on every platform, compiler and +// library version (unlike std::hash or boost::hash), for orderings that the +// output depends on. + +inline uint64_t stableMix(uint64_t x) { + // splitmix64 finaliser + x += 0x9e3779b97f4a7c15ull; + x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9ull; + x = (x ^ (x >> 27)) * 0x94d049bb133111ebull; + return x ^ (x >> 31); +} + +inline uint64_t stableCombine(uint64_t seed, uint64_t value) { + return stableMix(seed ^ stableMix(value)); +} + +inline uint64_t stableCombine(uint64_t seed, double value) { + if (value == 0) value = 0; // -0.0 and 0.0 compare equal, so hash them equally + uint64_t bits; + std::memcpy(&bits, &value, sizeof(bits)); + return stableCombine(seed, bits); +} + +inline uint64_t stableCombine(uint64_t seed, const char* data, std::size_t length) { + // FNV-1a over the bytes + uint64_t h = 0xcbf29ce484222325ull; + for (std::size_t i = 0; i < length; i++) { + h ^= static_cast(data[i]); + h *= 0x100000001b3ull; + } + return stableCombine(stableCombine(seed, static_cast(length)), h); +} + +#endif //_STABLE_HASH_H diff --git a/include/tile_coordinates_set.h b/include/tile_coordinates_set.h index 3d45c50e..21f14649 100644 --- a/include/tile_coordinates_set.h +++ b/include/tile_coordinates_set.h @@ -14,6 +14,8 @@ class TileCoordinatesSet { virtual void set(TileCoordinate x, TileCoordinate y) = 0; virtual size_t size() const = 0; virtual size_t zoom() const = 0; + // Set every tile with minX <= x <= maxX and minY <= y <= maxY. + virtual void setBox(TileCoordinate minX, TileCoordinate maxX, TileCoordinate minY, TileCoordinate maxY); // Visit every set tile in x-major, y-minor order. virtual void forEachSet(const std::function& f) const; }; @@ -30,7 +32,10 @@ class PreciseTileCoordinatesSet : public TileCoordinatesSet { size_t size() const override; size_t zoom() const override; void set(TileCoordinate x, TileCoordinate y) override; + void setBox(TileCoordinate minX, TileCoordinate maxX, TileCoordinate minY, TileCoordinate maxY) override; void forEachSet(const std::function& f) const override; + // Add the tiles set in other, a set of the same zoom. + void merge(const PreciseTileCoordinatesSet& other); private: unsigned int zoom_; @@ -50,6 +55,7 @@ class LossyTileCoordinatesSet : public TileCoordinatesSet { size_t size() const override; size_t zoom() const override; void set(TileCoordinate x, TileCoordinate y) override; + void forEachSet(const std::function& f) const override; private: unsigned int zoom_; diff --git a/include/tile_data.h b/include/tile_data.h index 3e8d20e9..729ef009 100644 --- a/include/tile_data.h +++ b/include/tile_data.h @@ -2,10 +2,15 @@ #ifndef _TILE_DATA_H #define _TILE_DATA_H +#include +#include #include +#include #include #include #include +#include +#include #include "append_vector.h" #include "clip_cache.h" #include "mmap_allocator.h" @@ -31,77 +36,134 @@ template void finalizeObjects( typename std::vector>::iterator end, typename std::vector>& lowZoom ) { - size_t z6OffsetDivisor = indexZoom >= CLUSTER_ZOOM ? (1 << (indexZoom - CLUSTER_ZOOM)) : 1; #ifdef CLOCK_MONOTONIC timespec startTs, endTs; clock_gettime(CLOCK_MONOTONIC, &startTs); #endif - - int i = -1; - for (auto it = begin; it != end; it++) { - i++; - if (it->size() > 0 || i % 50 == 0 || i == 4095) { - std::cout << "\r" << name << ": finalizing z6 tile " << (i + 1) << "/" << CLUSTER_ZOOM_AREA; - + const size_t shards = end - begin; + std::mutex progressMutex; + size_t done = 0; + auto reportDone = [&](size_t n) { + std::lock_guard lock(progressMutex); + done += n; + std::cout << "\r" << name << ": finalizing z6 tile " << done << "/" << CLUSTER_ZOOM_AREA; #ifdef CLOCK_MONOTONIC - clock_gettime(CLOCK_MONOTONIC, &endTs); - uint64_t elapsedNs = 1e9 * (endTs.tv_sec - startTs.tv_sec) + endTs.tv_nsec - startTs.tv_nsec; - std::cout << " (" << std::to_string((uint32_t)(elapsedNs / 1e6)) << " ms)"; + clock_gettime(CLOCK_MONOTONIC, &endTs); + uint64_t elapsedNs = 1e9 * (endTs.tv_sec - startTs.tv_sec) + endTs.tv_nsec - startTs.tv_nsec; + std::cout << " (" << std::to_string((uint32_t)(elapsedNs / 1e6)) << " ms)"; #endif - std::cout << std::flush; - } - if (it->size() == 0) - continue; + std::cout << std::flush; + }; + auto finalizeShard = [&](size_t i, size_t sortThreads) { + auto it = begin + i; // We track a separate copy of low zoom objects to avoid scanning large // lists of objects that may be on slow disk storage. for (auto objectIt = it->begin(); objectIt != it->end(); objectIt++) if (objectIt->oo.minZoom < CLUSTER_ZOOM) lowZoom[i].push_back(*objectIt); - sortOutputObjects(indexZoom, threadNum, it->begin(), it->end()); + sortOutputObjects(indexZoom, sortThreads, it->begin(), it->end()); + }; + + // Shards are independent. A shard with more than 1/threadNum of all objects + // (e.g. one of the few populated z6 tiles of a small extract) is sorted on its + // own using all threads. The others are sorted in parallel, one thread each, + // largest first. + size_t total = 0, empty = 0; + for (auto it = begin; it != end; it++) total += it->size(); + std::vector big, small; + for (size_t i = 0; i < shards; i++) { + const size_t n = (begin + i)->size(); + if (n == 0) empty++; + else if (threadNum <= 1 || n * threadNum > total) big.push_back(i); + else small.push_back(i); + } + std::sort(small.begin(), small.end(), [&](size_t a, size_t b) { return (begin + a)->size() > (begin + b)->size(); }); + reportDone(empty); + + for (size_t i : big) { + finalizeShard(i, threadNum); + reportDone(1); + } + + if (!small.empty()) { + // Each sort copies its shard into a buffer, so cap the objects being + // sorted at once (a single shard may always run). + const size_t budget = std::max(total / 8, (begin + small.front())->size()); + size_t inFlight = 0; + std::mutex budgetMutex; + std::condition_variable budgetFreed; + boost::asio::thread_pool pool(threadNum); + for (size_t i : small) { + boost::asio::post(pool, [&, i]() { + const size_t n = (begin + i)->size(); + { + std::unique_lock lock(budgetMutex); + budgetFreed.wait(lock, [&]() { return inFlight == 0 || inFlight + n <= budget; }); + inFlight += n; + } + finalizeShard(i, 1); + { + std::lock_guard lock(budgetMutex); + inFlight -= n; + } + budgetFreed.notify_all(); + reportDone(1); + }); + } + pool.join(); } std::cout << std::endl; } -template void collectTilesWithObjectsAtZoomTemplate( +// Set the tiles holding the objects of z6 tile i (shard), and their parent tiles. +template void collectTilesWithObjectsInShardTemplate( const unsigned int& indexZoom, - const typename std::vector>::iterator objects, - const size_t size, + AppendVectorNS::AppendVector& shard, + const size_t i, std::vector>& zooms ) { size_t maxZoom = zooms.size() - 1; uint16_t z6OffsetDivisor = indexZoom >= CLUSTER_ZOOM ? (1 << (indexZoom - CLUSTER_ZOOM)) : 1; int64_t lastX = -1; int64_t lastY = -1; - for (size_t i = 0; i < size; i++) { - const size_t z6x = i / CLUSTER_ZOOM_WIDTH; - const size_t z6y = i % CLUSTER_ZOOM_WIDTH; - - for (size_t j = 0; j < objects[i].size(); j++) { - // Compute the x, y at the base zoom level - TileCoordinate baseX = z6x * z6OffsetDivisor + objects[i][j].x; - TileCoordinate baseY = z6y * z6OffsetDivisor + objects[i][j].y; - - // Translate the x, y at the requested zoom level - TileCoordinate x = baseX / (1 << (indexZoom - maxZoom)); - TileCoordinate y = baseY / (1 << (indexZoom - maxZoom)); - - if (lastX != x || lastY != y) { - lastX = x; - lastY = y; - - for (int zoom = maxZoom; zoom >= 0; zoom--) { - zooms[zoom]->set(x, y); - x /= 2; - y /= 2; - } + const size_t z6x = i / CLUSTER_ZOOM_WIDTH; + const size_t z6y = i % CLUSTER_ZOOM_WIDTH; + + for (size_t j = 0; j < shard.size(); j++) { + // Compute the x, y at the base zoom level + TileCoordinate baseX = z6x * z6OffsetDivisor + shard[j].x; + TileCoordinate baseY = z6y * z6OffsetDivisor + shard[j].y; + + // Translate the x, y at the requested zoom level + TileCoordinate x = baseX / (1 << (indexZoom - maxZoom)); + TileCoordinate y = baseY / (1 << (indexZoom - maxZoom)); + + if (lastX != x || lastY != y) { + lastX = x; + lastY = y; + + for (int zoom = maxZoom; zoom >= 0; zoom--) { + zooms[zoom]->set(x, y); + x /= 2; + y /= 2; } } } } +template void collectTilesWithObjectsAtZoomTemplate( + const unsigned int& indexZoom, + const typename std::vector>::iterator objects, + const size_t size, + std::vector>& zooms +) { + for (size_t i = 0; i < size; i++) + collectTilesWithObjectsInShardTemplate(indexZoom, objects[i], i, zooms); +} + template OutputObjectID outputObjectWithId(const OO& input) { return OutputObjectID({ input.oo, 0 }); @@ -245,7 +307,7 @@ template void collectObjectsForTileTemplate( class TileDataSource { public: // Store for generated geometries - using point_store_t = std::vector; + using point_store_t = AppendVectorNS::AppendVector; using linestring_t = boost::geometry::model::linestring; using linestring_store_t = std::vector; @@ -310,6 +372,8 @@ class TileDataSource { TileDataSource(size_t threadNum, unsigned int indexZoom, bool includeID); void collectTilesWithObjectsAtZoom(std::vector>& zooms); + // The same for the objects of one z6 tile (0 <= shard < CLUSTER_ZOOM_AREA) + void collectTilesWithObjectsInShard(size_t shard, std::vector>& zooms); void collectTilesWithLargeObjectsAtZoom(std::vector>& zooms); @@ -348,9 +412,11 @@ class TileDataSource { std::vector getObjectsForTile( const std::vector& sortOrders, + const AttributeStore& attributeStore, unsigned int zoom, TileCoordinates coordinates ); + void orderStoredGeometryTies(std::vector& data) const; virtual Geometry buildWayGeometry(OutputGeometryType const geomType, NodeID const objectID, const TileBbox &bbox); virtual LatpLon buildNodeGeometry(NodeID const objectID, const TileBbox &bbox) const; @@ -398,8 +464,8 @@ class TileDataSource { const auto& shardId = getShard(id); const auto& shard = pointStores[shardId]; const auto offset = getId(id); - if (offset > shard.size()) throw std::out_of_range("Could not find generated node with id " + std::to_string(id) + ", shard " + std::to_string(shardId) + ", offset=" + std::to_string(offset)); - return shard.at(offset); + if (offset >= shard.size()) throw std::out_of_range("Could not find generated node with id " + std::to_string(id) + ", shard " + std::to_string(shardId) + ", offset=" + std::to_string(offset)); + return shard[offset]; } NodeID storeLinestring(const Linestring& src); @@ -436,7 +502,8 @@ class TileDataSource { void populateTilesAtZoom( const std::vector& sources, - std::vector>& zooms + std::vector>& zooms, + size_t threadNum = 1 ); #endif //_TILE_DATA_H diff --git a/resources/process-debug.lua b/resources/process-debug.lua index 4ea5e629..5fc399f1 100644 --- a/resources/process-debug.lua +++ b/resources/process-debug.lua @@ -155,6 +155,10 @@ poiTags = { aerialway = Set { "station" }, sport = Set { "american_football", "archery", "athletics", "australian_football", "badminton", "baseball", "basketball", "beachvolleyball", "billiards", "bmx", "boules", "bowls", "boxing", "canadian_football", "canoe", "chess", "climbing", "climbing_adventure", "cricket", "cricket_nets", "croquet", "curling", "cycling", "disc_golf", "diving", "dog_racing", "equestrian", "fatsal", "field_hockey", "free_flying", "gaelic_games", "golf", "gymnastics", "handball", "hockey", "horse_racing", "horseshoes", "ice_hockey", "ice_stock", "judo", "karting", "korfball", "long_jump", "model_aerodrome", "motocross", "motor", "multi", "netball", "orienteering", "paddle_tennis", "paintball", "paragliding", "pelota", "racquet", "rc_car", "rowing", "rugby", "rugby_league", "rugby_union", "running", "sailing", "scuba_diving", "shooting", "shooting_range", "skateboard", "skating", "skiing", "soccer", "surfing", "swimming", "table_soccer", "table_tennis", "team_handball", "tennis", "toboggan", "volleyball", "water_ski", "yoga" }, tourism = Set { "alpine_hut", "aquarium", "artwork", "attraction", "bed_and_breakfast", "camp_site", "caravan_site", "chalet", "gallery", "guest_house", "hostel", "hotel", "information", "motel", "museum", "picnic_site", "theme_park", "viewpoint", "zoo" }, waterway = Set { "dock" } } +-- poiTags keys in alphabetical order: GetPOIRank tries them in this order, and the first key an object has decides its class +poiTagOrder = {} +for k in pairs(poiTags) do poiTagOrder[#poiTagOrder+1] = k end +table.sort(poiTagOrder) -- POI "class" values: based on https://github.com/openmaptiles/openmaptiles/blob/master/layers/poi/poi.yaml poiClasses = { townhall="town_hall", public_building="town_hall", courthouse="town_hall", community_centre="town_hall", @@ -504,7 +508,8 @@ function GetPOIRank(obj) local k,list,v,class,rank -- Can we find the tag? - for k,list in pairs(poiTags) do + for _,k in ipairs(poiTagOrder) do + list = poiTags[k] if list[Find(k)] then v = Find(k) -- k/v are the OSM tag pair class = poiClasses[v] or v diff --git a/resources/process-openmaptiles.lua b/resources/process-openmaptiles.lua index 74892add..7591e60f 100644 --- a/resources/process-openmaptiles.lua +++ b/resources/process-openmaptiles.lua @@ -271,6 +271,10 @@ poiTags = { aerialway = Set { "station" }, sport = Set { "american_football", "archery", "athletics", "australian_football", "badminton", "baseball", "basketball", "beachvolleyball", "billiards", "bmx", "boules", "bowls", "boxing", "canadian_football", "canoe", "chess", "climbing", "climbing_adventure", "cricket", "cricket_nets", "croquet", "curling", "cycling", "disc_golf", "diving", "dog_racing", "equestrian", "fatsal", "field_hockey", "free_flying", "gaelic_games", "golf", "gymnastics", "handball", "hockey", "horse_racing", "horseshoes", "ice_hockey", "ice_stock", "judo", "karting", "korfball", "long_jump", "model_aerodrome", "motocross", "motor", "multi", "netball", "orienteering", "paddle_tennis", "paintball", "paragliding", "pelota", "racquet", "rc_car", "rowing", "rugby", "rugby_league", "rugby_union", "running", "sailing", "scuba_diving", "shooting", "shooting_range", "skateboard", "skating", "skiing", "soccer", "surfing", "swimming", "table_soccer", "table_tennis", "team_handball", "tennis", "toboggan", "volleyball", "water_ski", "yoga" }, tourism = Set { "alpine_hut", "aquarium", "artwork", "attraction", "bed_and_breakfast", "camp_site", "caravan_site", "chalet", "gallery", "guest_house", "hostel", "hotel", "information", "motel", "museum", "picnic_site", "theme_park", "viewpoint", "zoo" }, waterway = Set { "dock" } } +-- poiTags keys in alphabetical order: GetPOIRank tries them in this order, and the first key an object has decides its class +poiTagOrder = {} +for k in pairs(poiTags) do poiTagOrder[#poiTagOrder+1] = k end +table.sort(poiTagOrder) -- POI "class" values: based on https://github.com/openmaptiles/openmaptiles/blob/master/layers/poi/poi.yaml poiClasses = { townhall="town_hall", public_building="town_hall", courthouse="town_hall", community_centre="town_hall", @@ -912,7 +916,8 @@ function GetPOIRank() local k,list,v,class,rank -- Can we find the tag? - for k,list in pairs(poiTags) do + for _,k in ipairs(poiTagOrder) do + list = poiTags[k] if list[Find(k)] then v = Find(k) -- k/v are the OSM tag pair class = poiClasses[v] or poiTagsAsClass[k] or v diff --git a/src/attribute_store.cpp b/src/attribute_store.cpp index 6e661c8d..f2a3d77d 100644 --- a/src/attribute_store.cpp +++ b/src/attribute_store.cpp @@ -1,4 +1,5 @@ #include "attribute_store.h" +#include "stable_hash.h" #include #include @@ -55,6 +56,15 @@ const std::string& AttributeKeyStore::getKeyUnsafe(uint16_t index) const { return keys[index]; } +void AttributeKeyStore::finalize() { + finalized = true; + // keys2index is ordered by key string + ranks.assign(keys.size(), 0); + uint16_t rank = 0; + for (const auto& it : keys2index) + ranks[it.second] = ++rank; +} + // AttributePair void AttributePair::ensureStringIsOwned() { // Before we store an AttributePair in our long-term storage, we need @@ -557,6 +567,52 @@ void AttributeStore::reset() { pairIndexGeneration++; } +int AttributeStore::comparePairsUnsafe(uint32_t a, uint32_t b) const { + if (a == b) return 0; + const AttributePair& p = pairStore.getPairUnsafe(a); + const AttributePair& q = pairStore.getPairUnsafe(b); + const uint16_t pRank = keyStore.getRankUnsafe(p.keyIndex), qRank = keyStore.getRankUnsafe(q.keyIndex); + if (pRank != qRank) return pRank < qRank ? -1 : 1; + if (p.minzoom != q.minzoom) return p.minzoom < q.minzoom ? -1 : 1; + if (p.valueType != q.valueType) return p.valueType < q.valueType ? -1 : 1; + if (p.hasStringValue()) { + if (p.pooledString() < q.pooledString()) return -1; + if (q.pooledString() < p.pooledString()) return 1; + return 0; + } + if (p.doubleValue_ < q.doubleValue_) return -1; + if (q.doubleValue_ < p.doubleValue_) return 1; + return 0; +} + +int AttributeStore::compareContentUnsafe(AttributeIndex a, AttributeIndex b) const { + if (a == b) return 0; + boost::container::small_vector x, y; + forEachPairIndexCanonicalUnsafe(a, [&](uint32_t pairIndex) { x.push_back(pairIndex); }); + forEachPairIndexCanonicalUnsafe(b, [&](uint32_t pairIndex) { y.push_back(pairIndex); }); + for (size_t i = 0; i < x.size() && i < y.size(); i++) { + const int c = comparePairsUnsafe(x[i], y[i]); + if (c != 0) return c; + } + if (x.size() != y.size()) return x.size() < y.size() ? -1 : 1; + return 0; +} + +uint64_t AttributeStore::contentHashUnsafe(AttributeIndex index) const { + // Sum of the pair hashes, so the (index dependent) pair order doesn't matter + uint64_t rv = 0; + forEachPairUnsafe(index, [&](const AttributePair& p) { + uint64_t h = stableCombine(keyStore.getRankUnsafe(p.keyIndex), static_cast(p.minzoom)); + h = stableCombine(h, static_cast(p.valueType)); + if (p.hasStringValue()) + h = stableCombine(h, p.pooledString().data(), p.pooledString().size()); + else + h = stableCombine(h, p.doubleValue_); + rv += h; + }); + return rv; +} + void AttributeStore::finalize() { finalized = true; keyStore.finalize(); diff --git a/src/coordinates_geom.cpp b/src/coordinates_geom.cpp index 16887856..d6c35d86 100644 --- a/src/coordinates_geom.cpp +++ b/src/coordinates_geom.cpp @@ -50,6 +50,24 @@ void TileBbox::scaleRing(Ring &points, Ring const &src) const { } } +// Scaling that only drops points repeating their immediate predecessor. Used as +// a fallback for rings that scaleRing() collapses below 4 points: its +// backtracking window is positional, so a duplicated FIRST vertex shifts every +// later vertex by one, the closing vertex then matches that duplicate at j==4, +// and resize() truncates the whole ring. Without the backtracking there is +// nothing to mis-align, and the ring survives with its shape intact. +void TileBbox::scaleRingNoBacktrack(Ring &points, Ring const &src) const { + points.clear(); + points.reserve(src.size()); + for(auto const &i: src) { + auto scaled = scaleLatpLon(i.y(), i.x()); + if (!points.empty() && + points.back().x()==scaled.first && points.back().y()==scaled.second) + continue; + points.push_back(Point(scaled.first,scaled.second)); + } +} + Ring TileBbox::scaleRing(Ring const &src) const { Ring points; scaleRing(points, src); @@ -66,8 +84,15 @@ void TileBbox::scaleGeometry(MultiPolygon &dst, MultiPolygon const &src) const { // Copy the outer ring scaleRing(p.outer(), poly.outer()); - if (p.outer().size()<4) - continue; + if (p.outer().size()<4) { + // A collapsed outer ring means the whole feature disappears from the + // tile, at any size - so before giving up, retry without the + // backtracking that can truncate a ring to two points. Every ring that + // already survives is left untouched. + scaleRingNoBacktrack(p.outer(), poly.outer()); + if (p.outer().size()<4) + continue; + } // Copy the inner rings if (p.inners().size() < poly.inners().size()) diff --git a/src/geom.cpp b/src/geom.cpp index 43d7d079..a3221c2b 100644 --- a/src/geom.cpp +++ b/src/geom.cpp @@ -178,13 +178,17 @@ struct PrecheckBox { }; struct PrecheckScratch { std::vector segs; - std::vector boxes, holeBoxes; + std::vector boxes; std::vector> pairs; + std::vector bandStart, bandFill, bandSegs, crossed; void trim() { if (segs.capacity() > kPrecheckRetained) std::vector().swap(segs); if (boxes.capacity() > kPrecheckRetained) std::vector().swap(boxes); - if (holeBoxes.capacity() > kPrecheckRetained) std::vector().swap(holeBoxes); if (pairs.capacity() > kPrecheckRetained) std::vector>().swap(pairs); + if (bandStart.capacity() > kPrecheckRetained) std::vector().swap(bandStart); + if (bandFill.capacity() > kPrecheckRetained) std::vector().swap(bandFill); + if (bandSegs.capacity() > kPrecheckRetained) std::vector().swap(bandSegs); + if (crossed.capacity() > kPrecheckRetained) std::vector().swap(crossed); } }; thread_local PrecheckScratch precheckScratch; @@ -331,20 +335,27 @@ bool segmentsClear(std::vector &segs, double tol) { return true; } +// Whether the ray from h to +x crosses segment a->b: 1 = yes, -1 = no, 0 = unsure. +// Vertices are classed as above (y > hy) or not with exact comparisons (the +// half-open rule), so a ray through a vertex is counted consistently and the +// crossing parity is exact; only the side of h is decided with a tolerance. +inline int rayClearlyCrosses(const Point &h, const Point &a, const Point &b, double tol) { + const double hx = h.x(), hy = h.y(); + if ((a.y() > hy) == (b.y() > hy)) return -1; + if (std::max(a.x(), b.x()) < hx) return -1; + if (std::min(a.x(), b.x()) > hx) return 1; + const int o = clearOrient(a, b, h, tol); + if (o == 0) return 0; + return (b.y() > a.y()) == (o > 0) ? 1 : -1; +} + // Crossing-number test: 1 = h clearly inside r, -1 = clearly outside, 0 = unsure. int pointClearlyInRing(const Point &h, const Ring &r, double tol) { bool inside = false; - const double hx = h.x(), hy = h.y(); for (std::size_t i = 0; i + 1 < r.size(); i++) { - const Point &a = r[i], &b = r[i + 1]; - if ((a.y() > hy + tol && b.y() > hy + tol) || (a.y() < hy - tol && b.y() < hy - tol)) continue; - if (std::fabs(a.y() - hy) <= tol || std::fabs(b.y() - hy) <= tol) return 0; - // a and b are clearly on opposite sides of the horizontal line through h - if (std::max(a.x(), b.x()) < hx - tol) continue; - if (std::min(a.x(), b.x()) > hx + tol) { inside = !inside; continue; } - const int o = clearOrient(a, b, h, tol); - if (o == 0) return 0; - if ((b.y() > a.y()) == (o > 0)) inside = !inside; // the ray to +x crosses a->b + const int c = rayClearlyCrosses(h, r[i], r[i + 1], tol); + if (c == 0) return 0; + if (c > 0) inside = !inside; } return inside ? 1 : -1; } @@ -360,12 +371,91 @@ int ringClearlyInRing(const Ring &inner, const Ring &r, double tol) { return 0; } +// Holes lie inside the outer ring (ring 0) and outside each other. `segs` holds the +// segments of all rings of p, known not to touch. They are bucketed into horizontal +// bands, so that a ray from a vertex of each hole is tested against the segments of +// that vertex's band only, counting crossings per ring (odd = inside that ring). +bool holesClearlyPlaced(const Polygon &p, PrecheckScratch &scratch, double tol) { + const auto &inners = p.inners(); + const std::vector &segs = scratch.segs; + if (inners.empty()) return true; + if (segs.size() >= std::numeric_limits::max()) return false; + + double minY = std::numeric_limits::infinity(), maxY = -minY; + for (const PrecheckSeg &s : segs) { minY = std::min(minY, s.minY); maxY = std::max(maxY, s.maxY); } + minY -= tol; maxY += tol; + // About 4 segments per band; fewer bands while tall segments would make the index much bigger than segs + std::size_t bands = std::max(1, segs.size() / 4); + double scale = 0; + auto bandOf = [&](double y) { + const double b = (y - minY) * scale; + return b <= 0 ? std::size_t(0) : std::min(bands - 1, static_cast(b)); + }; + std::size_t entries; + for (;;) { + scale = bands / (maxY - minY); + entries = 0; + for (const PrecheckSeg &s : segs) entries += bandOf(s.maxY + tol) - bandOf(s.minY - tol) + 1; + if (entries <= 4 * segs.size() || bands == 1) break; + bands = (bands + 1) / 2; + } + if (entries >= std::numeric_limits::max()) return false; + + // Segments of band b: bandSegs[bandStart[b] .. bandStart[b+1]) + std::vector &start = scratch.bandStart, &fill = scratch.bandFill, &items = scratch.bandSegs; + start.assign(bands + 1, 0); + for (const PrecheckSeg &s : segs) + for (std::size_t b = bandOf(s.minY - tol), hi = bandOf(s.maxY + tol); b <= hi; b++) start[b + 1]++; + for (std::size_t b = 0; b < bands; b++) start[b + 1] += start[b]; + fill.assign(start.begin(), start.end() - 1); + items.resize(entries); + for (std::size_t k = 0; k < segs.size(); k++) + for (std::size_t b = bandOf(segs[k].minY - tol), hi = bandOf(segs[k].maxY + tol); b <= hi; b++) + items[fill[b]++] = static_cast(k); + + // 1 = h inside the outer ring and no other hole, -1 = not, 0 = unsure + std::vector &crossed = scratch.crossed; + auto locate = [&](const Point &h, uint32_t ownRing) { + crossed.clear(); + const std::size_t b = bandOf(h.y()); + for (uint32_t k = start[b]; k < start[b + 1]; k++) { + const PrecheckSeg &s = segs[items[k]]; + if (s.ring == ownRing) continue; + const int c = rayClearlyCrosses(h, s.a[0], s.a[1], tol); + if (c == 0) return 0; + if (c > 0) crossed.push_back(s.ring); + } + std::sort(crossed.begin(), crossed.end()); + bool inOuter = false; + for (std::size_t i = 0; i < crossed.size(); ) { + std::size_t j = i; + while (j < crossed.size() && crossed[j] == crossed[i]) j++; + if ((j - i) % 2 == 1) { + if (crossed[i] != 0) return -1; // inside another hole + inOuter = true; + } + i = j; + } + return inOuter ? 1 : -1; + }; + + // A ring known not to touch the others lies wherever any of its vertices does + for (std::size_t i = 0; i < inners.size(); i++) { + const Ring &hole = inners[i]; + const std::size_t m = hole.size() - 1; + const std::size_t tries = std::min(m, 8); + int loc = 0; + for (std::size_t k = 0; k < tries && loc == 0; k++) + loc = locate(hole[k * m / tries], static_cast(i + 1)); + if (loc != 1) return false; + } + return true; +} + bool polygonClear(const Polygon &p, PrecheckScratch &scratch, PrecheckBox &outerBox) { const Ring &outer = p.outer(); const auto &inners = p.inners(); if (inners.size() >= std::numeric_limits::max()) return false; - // Every hole is located by walking the outer ring - if (inners.size() * outer.size() > kPrecheckMaxWork) return false; double maxAbs = 0; if (!ringFinite(outer, maxAbs)) return false; for (const auto &inner : inners) @@ -374,29 +464,16 @@ bool polygonClear(const Polygon &p, PrecheckScratch &scratch, PrecheckBox &outer std::vector &segs = scratch.segs; segs.clear(); - std::vector &holeBoxes = scratch.holeBoxes; - holeBoxes.resize(inners.size()); if (!ringClear(outer, false, tol, 0, segs, outerBox)) return false; - for (std::size_t i = 0; i < inners.size(); i++) { - if (!ringClear(inners[i], true, tol, static_cast(i + 1), segs, holeBoxes[i])) return false; - holeBoxes[i].id = static_cast(i); - } + PrecheckBox holeBox; + for (std::size_t i = 0; i < inners.size(); i++) + if (!ringClear(inners[i], true, tol, static_cast(i + 1), segs, holeBox)) return false; // No two rings touch, and no ring touches itself. if (!segmentsClear(segs, tol)) return false; // Holes lie inside the outer ring and are not nested in each other. - for (std::size_t i = 0; i < inners.size(); i++) { - if (boxesApart(holeBoxes[i], outerBox, tol)) return false; - if (ringClearlyInRing(inners[i], outer, tol) != 1) return false; - } - std::size_t work = 0; - return forOverlappingBoxes(holeBoxes, tol, [&](uint32_t i, uint32_t j) { - work += inners[i].size() + inners[j].size(); - return work <= kPrecheckMaxWork && - ringClearlyInRing(inners[i], inners[j], tol) == -1 && - ringClearlyInRing(inners[j], inners[i], tol) == -1; - }); + return holesClearlyPlaced(p, scratch, tol); } // a's outer ring, known not to touch any ring of b, lies outside b's interior: @@ -839,6 +916,11 @@ struct RectClipper { bool inside(const Point &p) const { return p.x() >= x0 && p.x() <= x1 && p.y() >= y0 && p.y() <= y1; } + // Both end points beyond the same edge: the segment can't reach the box + bool beyondOneEdge(const Point &a, const Point &b) const { + return (a.x() < x0 && b.x() < x0) || (a.x() > x1 && b.x() > x1) || + (a.y() < y0 && b.y() < y0) || (a.y() > y1 && b.y() > y1); + } bool strictlyInside(const Point &p) const { return p.x() > x0 && p.x() < x1 && p.y() > y0 && p.y() < y1; } @@ -965,7 +1047,9 @@ RingClip clipRing(const RectClipper &rc, const Ring &ring, std::vector t0; + // Most segments of a large polygon lie far from a small tile's box: reject + // those with four comparisons before the clip computation + const bool hit = !rc.beyondOneEdge(a, b) && rc.clipSegment(a, b, t0, t1, pa, pb) && t1 > t0; if (!hit || (building && t0 > 0)) { if (building && !finish()) return RingClip::Fail; if (!hit) continue; diff --git a/src/mbtiles.cpp b/src/mbtiles.cpp index c056d3a6..1d2e653f 100644 --- a/src/mbtiles.cpp +++ b/src/mbtiles.cpp @@ -2,15 +2,30 @@ #include "helpers.h" #include #include +#include using namespace sqlite; using namespace std; MBTiles::MBTiles(): pendingStatements1(std::make_shared>()), - pendingStatements2(std::make_shared>()) + pendingStatements2(std::make_shared>()), + pendingBytes(0), + batches([this](PendingStatement& stmt) { + const std::lock_guard lock(m); + insertOrReplace(stmt.zoom, stmt.x, stmt.y, stmt.data, stmt.isMerge); + }, + [](const PendingStatement& stmt) { return sizeof(PendingStatement) + stmt.data.size(); }) {} +// At most this much is buffered while SQLite is busy; beyond it, workers wait +// for the writer instead (a planet run otherwise buffered ~15M tiles, over 3 GB). +static const size_t MAX_PENDING_BYTES = 256 << 20; + +static size_t pendingCost(const std::string& data) { + return sizeof(PendingStatement) + data.size(); +} + MBTiles::~MBTiles() { if (db && inTransaction) db << "COMMIT;"; // commit all the changes if open } @@ -71,6 +86,7 @@ void MBTiles::flushPendingStatements() { while(!pendingStatements2->empty()) { const PendingStatement& stmt = pendingStatements2->back(); insertOrReplace(stmt.zoom, stmt.x, stmt.y, stmt.data, stmt.isMerge); + pendingBytes -= pendingCost(stmt.data); pendingStatements2->pop_back(); } @@ -80,19 +96,37 @@ void MBTiles::flushPendingStatements() { } void MBTiles::saveTile(int zoom, int x, int y, string *data, bool isMerge) { + if (batches.inBatch()) { + batches.add({zoom, x, y, *data, isMerge}); + return; + } + // If the lock is available, write directly to SQLite. if (m.try_lock()) { insertOrReplace(zoom, x, y, *data, isMerge); flushPendingStatements(); m.unlock(); - } else { + return; + } + + { // Else buffer the write for later, copying its binary blob. const std::lock_guard lock(pendingStatementsMutex); - pendingStatements1->push_back({zoom, x, y, *data, isMerge}); + if (pendingBytes < MAX_PENDING_BYTES) { + pendingBytes += pendingCost(*data); + pendingStatements1->push_back({zoom, x, y, *data, isMerge}); + return; + } } + + // The buffer is full: wait for SQLite, then help drain the buffer. + const std::lock_guard lock(m); + insertOrReplace(zoom, x, y, *data, isMerge); + flushPendingStatements(); } void MBTiles::closeForWriting() { + if (!batches.allWritten()) throw std::runtime_error("MBTiles: a batch of tiles was never written - please file an issue"); flushPendingStatements(); preparedStatements[0].used(true); preparedStatements[1].used(true); diff --git a/src/mmap_allocator.cpp b/src/mmap_allocator.cpp index 65818873..4daf0c00 100644 --- a/src/mmap_allocator.cpp +++ b/src/mmap_allocator.cpp @@ -1,9 +1,13 @@ #include "mmap_allocator.h" +#include +#include #include #include #include #include #include +#include +#include #include #include @@ -191,6 +195,25 @@ bool mmap_dir_t::is_open() { if(!mmap_dir_filename.empty()) { try { + // Unmapping and deleting a big store (planet: ~140 GB in ~300 files) + // takes half a minute on one thread, so release the files from several. + const size_t threads = std::min(files.size(), std::min(16u, std::max(1u, std::thread::hardware_concurrency()))); + if (threads > 1) { + std::atomic next(0); + std::vector workers; + for (size_t t = 0; t < threads; t++) { + try { + workers.emplace_back([this, &next]() { + for (size_t i; (i = next++) < files.size(); ) + files[i].reset(); + }); + } catch (const std::system_error &) { + break; // fewer threads; files.clear() below releases the rest + } + } + for (auto &worker : workers) + worker.join(); + } files.clear(); if(mmap_dir_created) { diff --git a/src/osm_store.cpp b/src/osm_store.cpp index 5ec7a81c..3bb086f2 100644 --- a/src/osm_store.cpp +++ b/src/osm_store.cpp @@ -6,6 +6,7 @@ #include #include +#include #include "node_store.h" #include "way_store.h" @@ -16,6 +17,42 @@ static inline bool isClosed(const std::vector& way) { return !way.empty() && way.front() == way.back(); } +// geom::within(inner, outer) for many inner rings against one big outer ring. +// Boost analyses the outer ring afresh on every call (a reservoir with 2500 +// islands and a 600,000-point shore takes 47 s); here its segments are indexed +// once. An inner ring that shares no point with the outer ring lies wholly +// inside or wholly outside it, so one of its vertices decides; when the rings +// touch or cross, Boost decides as before. +class RingWithinIndex { + using Segment = bg::model::segment; + using Value = std::pair; + const Ring &outer; + bg::index::rtree> segments; + + static std::vector segmentBoxes(const Ring &ring) { + std::vector values; + for (std::size_t i = 0; i + 1 < ring.size(); i++) + values.emplace_back(bg::return_envelope(Segment(ring[i], ring[i + 1])), i); + return values; + } + +public: + RingWithinIndex(const Ring &outer) : outer(outer), segments(segmentBoxes(outer)) { } + + bool within(const Ring &inner) const { + if (inner.size() < 4) return geom::within(inner, outer); + for (std::size_t j = 0; j + 1 < inner.size(); j++) { + const Segment s(inner[j], inner[j + 1]); + for (auto it = segments.qbegin(bg::index::intersects(bg::return_envelope(s))); it != segments.qend(); ++it) { + const std::size_t i = it->second; + if (bg::intersects(s, Segment(outer[i], outer[i + 1]))) + return geom::within(inner, outer); + } + } + return geom::within(inner.front(), outer); + } +}; + UsedObjects::UsedObjects(Status status): status(status), mutex(256), ids(256 * 1024) { } @@ -76,17 +113,31 @@ MultiPolygon OSMStore::wayListMultiPolygon(WayVec::const_iterator outerBegin, Wa // add all inners and outers to the multipolygon std::vector filledInners; + std::vector innerBoxes; for (auto it = inners.begin(); it != inners.end(); ++it) { Ring inner; fillPoints(inner, it->begin(), it->end()); + innerBoxes.emplace_back(geom::return_envelope(inner)); filledInners.emplace_back(inner); } bool onlyOneOuter = outers.size()==1; for (auto ot = outers.begin(); ot != outers.end(); ot++) { Polygon poly; fillPoints(poly.outer(), ot->begin(), ot->end()); - for (auto it = filledInners.begin(); it != filledInners.end(); ++it) { - if (onlyOneOuter || geom::within(*it, poly.outer())) { poly.inners().emplace_back(*it); } + const Box outerBox = geom::return_envelope(poly.outer()); + // Index big outer rings that many inner rings must be tested against + std::unique_ptr index; + if (!onlyOneOuter && filledInners.size() >= 16 && poly.outer().size() >= 4096) + index.reset(new RingWithinIndex(poly.outer())); + for (std::size_t i = 0; i < filledInners.size(); i++) { + const Ring &inner = filledInners[i]; + // A ring within the outer has its bounding box within the outer's, + // so the (costly) within test is only needed when it does + if (onlyOneOuter || + ((inner.empty() || geom::covered_by(innerBoxes[i], outerBox)) && + (index ? index->within(inner) : geom::within(inner, poly.outer())))) { + poly.inners().emplace_back(inner); + } } mp.emplace_back(move(poly)); } diff --git a/src/output_object.cpp b/src/output_object.cpp index 3d5703ca..bc4c1855 100644 --- a/src/output_object.cpp +++ b/src/output_object.cpp @@ -42,8 +42,8 @@ static void writeAttributesTo( FeatureBuilder& fbuilder, char zoom ) { - attributeStore.forEachPairUnsafe(attributes, [&](const AttributePair& pair) { - const AttributePair* it = &pair; + attributeStore.forEachPairIndexCanonicalUnsafe(attributes, [&](uint32_t pairIndex) { + const AttributePair* it = &attributeStore.pairStore.getPairUnsafe(pairIndex); if (it->minzoom > zoom) return; // TODO: consider taking a data view that is stable @@ -179,7 +179,7 @@ const uint32_t* MvtAttributeMemo::resolve(const AttributeStore& attributeStore, if (pairs.used >= MEMO_MAX_PAIRS) pairs.clear(); const uint32_t begin = entries.size(); - attributeStore.forEachPairIndexUnsafe(index, [&](uint32_t pairIndex) { + attributeStore.forEachPairIndexCanonicalUnsafe(index, [&](uint32_t pairIndex) { uint32_t key, value; if (const Table::Slot* s = pairs.find(pairIndex)) { key = s->a; diff --git a/src/pmtiles.cpp b/src/pmtiles.cpp index c9561b59..fa520f34 100644 --- a/src/pmtiles.cpp +++ b/src/pmtiles.cpp @@ -10,7 +10,9 @@ #include "helpers.h" TileOffset::TileOffset() { } -PMTiles::PMTiles() { } +PMTiles::PMTiles() : + batches([this](PendingTile &tile) { writeTile(tile); }, + [](const PendingTile &tile) { return tile.data.size(); }) { } PMTiles::~PMTiles() { } void PMTiles::open(std::string &filename) { @@ -24,6 +26,7 @@ void PMTiles::open(std::string &filename) { // Finish writing the .pmtiles file void PMTiles::close(std::string &metadata) { std::cout << "\nClosing pmtiles file" << std::flush; + if (!batches.allWritten()) { throw std::runtime_error("PMTiles: a batch of tiles was never written - please file an issue"); } // add all tiles to directories, writing leaf directories as we go std::vector rootEntries; @@ -129,34 +132,42 @@ void PMTiles::flushEntries(std::vector &rootEntries, std::vect rootEntries.emplace_back(rootEntry); } -// Write a tile to file and store it in the index -// - if the tile is small and has already been written, reuse that instead -// - we're a bit fussy about mutexs because compress_string is expensive +// Save a tile: compress it here (compress_string is expensive), but leave tiny tiles +// raw until they're written, since most of them (e.g. sea) repeat an earlier one void PMTiles::saveTile(int zoom, int x, int y, std::string &data) { - TileOffset offset; - bool isNew = false; - uint64_t tileId = pmtiles::zxy_to_tileid(zoom,x,y); - - // if it's a tiny tile (e.g. sea), see if we've written it already - std::unique_lock indexLock1(indexMutex); - if (data.size() lock(batches.writeMutex()); + writeTile(tile); +} - // otherwise, compress it and write +// Write a tile to file and store it in the index +// - if the tile is tiny and has already been written, reuse that instead +void PMTiles::writeTile(PendingTile &tile) { + TileOffset offset; + auto cached = tile.isTiny ? tinyCache.find(tile.data) : tinyCache.end(); + if (cached != tinyCache.end()) { + offset = cached->second; } else { - indexLock1.unlock(); - std::string compressed = compress_string(data, Z_DEFAULT_COMPRESSION, true); - std::lock_guard lock(fileMutex); - // write to file + std::string compressed = tile.isTiny ? compress_string(tile.data, Z_DEFAULT_COMPRESSION, true) : std::move(tile.data); offset = TileOffset(static_cast(outputStream.tellp()) - HEADER_ROOT, compressed.size()); outputStream.write(compressed.c_str(), compressed.size()); numTilesWritten++; - isNew = true; + // add to cache if tiny + if (tile.isTiny) { + if (tinyCache.size()>TINY_MAX_SIZE) tinyCache.clear(); + tinyCache.insert({ tile.data, offset }); + } } - + // store in index - std::lock_guard indexLock2(indexMutex); + uint64_t tileId = tile.tileId; numTilesAddressed++; if (isSparse) { sparseIndex[tileId] = offset; @@ -164,10 +175,4 @@ void PMTiles::saveTile(int zoom, int x, int y, std::string &data) { if (tileId >= denseIndex.size()) denseIndex.resize(tileId+10000, { 0, 0xffffff }); denseIndex[tileId] = offset; } - - // add to cache if tiny - if (isNew && data.size()TINY_MAX_SIZE) tinyCache.clear(); - tinyCache.insert({ data, offset }); - } } diff --git a/src/shp_mem_tiles.cpp b/src/shp_mem_tiles.cpp index e5c69e52..fd673e65 100644 --- a/src/shp_mem_tiles.cpp +++ b/src/shp_mem_tiles.cpp @@ -78,14 +78,58 @@ bool ShpMemTiles::mayIntersect(const std::string& layerName, const Box& box) con uint32_t y1 = latp2tiley(latp1, spatialIndexZoom); uint32_t y2 = latp2tiley(latp2, spatialIndexZoom); - for (int x = std::min(x1, x2); x <= std::min((1u << spatialIndexZoom) - 1u, std::max(x1, x2)); x++) { - for (int y = std::min(y1, y2); y <= std::min((1u << spatialIndexZoom) - 1u, std::max(y1, y2)); y++) { - uint32_t z6x = x / (1u << (spatialIndexZoom - CLUSTER_ZOOM)); - uint32_t z6y = y / (1u << (spatialIndexZoom - CLUSTER_ZOOM)); + const uint32_t xMin = std::min(x1, x2), xMax = std::min((1u << spatialIndexZoom) - 1u, std::max(x1, x2)); + const uint32_t yMin = std::min(y1, y2), yMax = std::min((1u << spatialIndexZoom) - 1u, std::max(y1, y2)); + const uint32_t divisor = 1u << (spatialIndexZoom - CLUSTER_ZOOM); + + // First pass, without any geometry checks: is a tile in the box already known to + // intersect, and how many tiles still need the exact check below? + size_t unchecked = 0; + for (uint32_t x = xMin; x <= xMax; x++) { + for (uint32_t y = yMin; y <= yMax; y++) { + uint32_t z6x = x / divisor, z6y = y / divisor; + auto& bitvec = sparseLayerVector[z6x * CLUSTER_ZOOM_WIDTH + z6y]; + uint64_t index = 2u * ((x - z6x * divisor) * divisor + (y - z6y * divisor)); + if (!bitvec.empty() && bitvec[index]) { + if (bitvec[index + 1]) return true; + unchecked++; + } + } + } + if (unchecked == 0) return false; + + // Each tile's exact check intersects the tile with every candidate geometry, so its + // cost grows with the size of those geometries. A large query box that lies inside + // the bounding box of a huge geometry without touching it (say, an offshore marine + // reserve inside a country's bounding box) would check every one of thousands of + // tiles, and find nothing. Instead, check the query box once: if no geometry + // intersects it, none intersects any tile in it. Otherwise the caller's exact check + // decides. + const size_t maxTileChecks = 16; + if (unchecked > maxTileChecks) { + Box queryBox = box; + return !QueryMatchingGeometries( + layerName, + true, + queryBox, + [&](const RTree &rtree) { // indexQuery + vector results; + rtree.query(geom::index::intersects(queryBox), back_inserter(results)); + return results; + }, + [&](OutputObject const &oo) { // checkQuery + return geom::intersects(queryBox, retrieveMultiPolygon(oo.objectID)); + } + ).empty(); + } + + for (uint32_t x = xMin; x <= xMax; x++) { + for (uint32_t y = yMin; y <= yMax; y++) { + uint32_t z6x = x / divisor; + uint32_t z6y = y / divisor; auto& bitvec = sparseLayerVector[z6x * CLUSTER_ZOOM_WIDTH + z6y]; - uint32_t divisor = 1u << (spatialIndexZoom - CLUSTER_ZOOM); uint64_t index = 2u * ((x - z6x * divisor) * divisor + (y - z6y * divisor)); if (!bitvec.empty() && bitvec[index]) { @@ -137,7 +181,8 @@ void ShpMemTiles::StoreGeometry( const std::string& name, uint minzoom, int32_t score, - AttributeIndex attrIdx + AttributeIndex attrIdx, + bool knownValid ) { geom::model::box box; @@ -175,9 +220,9 @@ void ShpMemTiles::StoreGeometry( case POLYGON_: { - // Checked once here, so tiles can clip valid polygons on the fast path + // Checked once here (unless the caller has), so tiles can clip valid polygons on the fast path const MultiPolygon &mp = boost::get(geometry); - NodeID oid = storeMultiPolygon(mp, multiPolygonDefinitelyValid(mp) || geom::is_valid(mp)); + NodeID oid = storeMultiPolygon(mp, knownValid || multiPolygonDefinitelyValid(mp) || geom::is_valid(mp)); oo = std::make_shared(geomType, layerNum, oid, attrIdx, minzoom); std::vector oolist { *oo }; addGeometryToIndex(boost::get(geometry), oolist, 0); diff --git a/src/shp_processor.cpp b/src/shp_processor.cpp index f3b3c4f2..b32291b2 100644 --- a/src/shp_processor.cpp +++ b/src/shp_processor.cpp @@ -366,8 +366,8 @@ void ShpProcessor::processShapeGeometry(SHPObject* shape, AttributeIndex attrIdx // Make valid if needs be string reason; - const bool precheckValid = multiPolygonDefinitelyValid(multi); - if (!precheckValid && !geom::is_valid(multi, reason)) { + const bool valid = multiPolygonDefinitelyValid(multi) || geom::is_valid(multi, reason); + if (!valid) { if (verbose) cerr << "Shapefile entity " << shape->nShapeId << " type " << shapeType << " is invalid. Parts:" << shape->nParts << ". Reason:" << reason; make_valid(multi); if (verbose) { @@ -376,11 +376,17 @@ void ShpProcessor::processShapeGeometry(SHPObject* shape, AttributeIndex attrIdx cerr << endl; } } - // clip to bounding box - MultiPolygon out; - geom::intersection(multi, clippingBox, out); - if (boost::size(out)>0) { - shpMemTiles.StoreGeometry(layerNum, layer.name, POLYGON_, out, layer.indexed, hasName, name, minzoom, score, attrIdx); + // clip to bounding box (a valid polygon wholly inside it is stored as it is) + Box envelope; + geom::envelope(multi, envelope); + if (valid && geom::covered_by(envelope, clippingBox)) { + shpMemTiles.StoreGeometry(layerNum, layer.name, POLYGON_, multi, layer.indexed, hasName, name, minzoom, score, attrIdx, true); + } else { + MultiPolygon out; + geom::intersection(multi, clippingBox, out); + if (boost::size(out)>0) { + shpMemTiles.StoreGeometry(layerNum, layer.name, POLYGON_, out, layer.indexed, hasName, name, minzoom, score, attrIdx); + } } } else { diff --git a/src/tile_coordinates_set.cpp b/src/tile_coordinates_set.cpp index e6d6e78e..7a25a1ba 100644 --- a/src/tile_coordinates_set.cpp +++ b/src/tile_coordinates_set.cpp @@ -40,6 +40,12 @@ void TileCoordinatesSet::forEachSet(const std::function> 12] |= (uint64_t)1 << ((w >> 6) & 63); } +// Fills each row of the box word by word, rather than tile by tile. +void PreciseTileCoordinatesSet::setBox(TileCoordinate minX, TileCoordinate maxX, TileCoordinate minY, TileCoordinate maxY) { + const uint64_t dim = 1ull << zoom_; + if (minX > maxX || minY > maxY || minX >= dim || minY >= dim) + return; + maxX = std::min(maxX, dim - 1); + maxY = std::min(maxY, dim - 1); + for (uint64_t x = minX; x <= maxX; x++) { + const uint64_t first = x * dim + minY, last = x * dim + maxY; + const uint64_t w0 = first >> 6, w1 = last >> 6; + for (uint64_t w = w0; w <= w1; w++) { + uint64_t mask = ~0ull; + if (w == w0) mask &= ~0ull << (first & 63); + if (w == w1) mask &= ~0ull >> (63 - (last & 63)); + tiles[w] |= mask; + } + for (uint64_t block = w0 >> 6; block <= (w1 >> 6); block++) + summary[block >> 6] |= (uint64_t)1 << (block & 63); + } +} + +void PreciseTileCoordinatesSet::merge(const PreciseTileCoordinatesSet& other) { + if (other.zoom_ != zoom_) + throw std::invalid_argument("PreciseTileCoordinatesSet::merge: zooms differ"); + forEachMarkedWord(other.summary, numWords, [&](size_t w) { tiles[w] |= other.tiles[w]; }); + for (size_t s = 0; s < summary.size(); s++) + summary[s] |= other.summary[s]; +} + void PreciseTileCoordinatesSet::forEachSet(const std::function& f) const { // loc = x * 2^zoom + y, so ascending loc order is x-major, y-minor. // Only blocks marked in the summary can hold set bits; visiting them in @@ -134,3 +169,24 @@ void LossyTileCoordinatesSet::set(TileCoordinate x, TileCoordinate y) { throw std::logic_error("LossyTileCoordinatesSet::set() is not implemented; LossyTileCoordinatesSet is read-only"); } +void LossyTileCoordinatesSet::forEachSet(const std::function& f) const { + // Expand the underlying set instead of testing all 4^zoom tiles (2^34 at z17). + // Its tiles arrive x-major, y-minor; one column of them becomes `scale` + // columns here, which keeps this set's order x-major, y-minor too. + std::vector column; + TileCoordinate columnX = 0; + auto flush = [&]() { + for (TileCoordinate dx = 0; dx < scale; dx++) + for (TileCoordinate y : column) + for (TileCoordinate dy = 0; dy < scale; dy++) + f(columnX * scale + dx, y * scale + dy); + column.clear(); + }; + tiles.forEachSet([&](TileCoordinate x, TileCoordinate y) { + if (!column.empty() && x != columnX) flush(); + columnX = x; + column.push_back(y); + }); + if (!column.empty()) flush(); +} + diff --git a/src/tile_data.cpp b/src/tile_data.cpp index 89b367e0..c1fcc02e 100644 --- a/src/tile_data.cpp +++ b/src/tile_data.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -6,6 +7,7 @@ #include "tile_data.h" #include "coordinates_geom.h" #include "leased_store.h" +#include "stable_hash.h" #include using namespace std; @@ -166,6 +168,13 @@ void TileDataSource::collectTilesWithObjectsAtZoom(std::vector(indexZoom, objectsWithIds.begin(), objectsWithIds.size(), zooms); } +void TileDataSource::collectTilesWithObjectsInShard(size_t shard, std::vector>& zooms) { + if (shard < objects.size()) + collectTilesWithObjectsInShardTemplate(indexZoom, objects[shard], shard, zooms); + if (shard < objectsWithIds.size()) + collectTilesWithObjectsInShardTemplate(indexZoom, objectsWithIds[shard], shard, zooms); +} + void addCoveredTilesToOutput(const uint indexZoom, std::vector>& zooms, const Box& box) { size_t maxZoom = zooms.size() - 1; // std::cout << "addCoveredTilesToOutput maxZoom=" << maxZoom << ", indexZoom - maxZoom = " << (indexZoom - maxZoom) << std::endl; @@ -174,16 +183,11 @@ void addCoveredTilesToOutput(const uint indexZoom, std::vector= 0; zoom--) { - zooms[zoom]->set(zx, zy); - zx /= 2; - zy /= 2; - } - } + // The parents of the tiles in the box are the tiles in the box scaled down + for (int zoom = maxZoom; zoom >= 0; zoom--) { + zooms[zoom]->setBox(minx, maxx, miny, maxy); + minx /= 2; maxx /= 2; + miny /= 2; maxy /= 2; } } @@ -385,7 +389,13 @@ Geometry TileDataSource::buildWayGeometry(OutputGeometryType const geomType, } MultiPolygon mp; - if (cachedClip == nullptr) + // rect_clip below needs the input as it was before fast_clip. For a large valid + // polygon, keep it rather than load it from the store again for every tile. + std::size_t inputPoints = 0; + if (cachedClip == nullptr && inputKnownValid) + for (auto const &p : uncached) inputPoints += geom::num_points(p); + const bool keepUncached = inputPoints > 1000; + if (cachedClip == nullptr && !keepUncached) mp = std::move(uncached); else geom::assign(mp, input); @@ -405,9 +415,9 @@ Geometry TileDataSource::buildWayGeometry(OutputGeometryType const geomType, // instead, which is much cheaper than the is_valid/intersection/repair // path below; rect_clip bails out on anything it can't handle. MultiPolygon original; - if (cachedClip == nullptr) populateMultiPolygon(original, objectID); + if (cachedClip == nullptr && !keepUncached) populateMultiPolygon(original, objectID); MultiPolygon clipped; - if (rect_clip(cachedClip == nullptr ? original : input, box, clipped) && + if (rect_clip(cachedClip == nullptr && !keepUncached ? original : input, box, clipped) && (multiPolygonDefinitelyValid(clipped) || geom::is_valid(clipped))) { multiPolygonClipCache.add(bbox, objectID, clipped, true); return clipped; @@ -498,35 +508,172 @@ void TileDataSource::reportSize() const { void populateTilesAtZoom( const std::vector& sources, - std::vector>& zooms + std::vector>& zooms, + size_t threadNum ) { if (zooms.size() > 15) throw std::out_of_range("populateTilesAtZoom: expected at most z14 zooms (15), but found " + std::to_string(zooms.size()) + " vectors"); - for(size_t i=0; icollectTilesWithObjectsAtZoom(zooms); - sources[i]->collectTilesWithLargeObjectsAtZoom(zooms); + std::vector precise; + for (const auto& zoom : zooms) + precise.push_back(dynamic_cast(zoom.get())); + const bool allPrecise = std::find(precise.begin(), precise.end(), nullptr) == precise.end(); + + if (threadNum <= 1 || !allPrecise) { + for (size_t i = 0; i < sources.size(); i++) + sources[i]->collectTilesWithObjectsAtZoom(zooms); + } else { + // Objects are kept per z6 tile. Each worker takes whole columns of z6 + // tiles (so that it touches only its part of the bitmaps), sets bits in + // bitmaps of its own, and these are merged at the end. + std::atomic nextColumn(0); + std::vector>> local(threadNum); + boost::asio::thread_pool pool(threadNum); + for (size_t t = 0; t < threadNum; t++) { + boost::asio::post(pool, [&, t]() { + auto& mine = local[t]; + for (size_t zoom = 0; zoom < zooms.size(); zoom++) + mine.push_back(std::make_shared(zoom)); + for (size_t column; (column = nextColumn++) < CLUSTER_ZOOM_WIDTH; ) + for (size_t row = 0; row < CLUSTER_ZOOM_WIDTH; row++) + for (const auto source : sources) + source->collectTilesWithObjectsInShard(column * CLUSTER_ZOOM_WIDTH + row, mine); + }); + } + pool.join(); + for (const auto& mine : local) + for (size_t zoom = 0; zoom < mine.size(); zoom++) + precise[zoom]->merge(static_cast(*mine[zoom])); } + + for (size_t i = 0; i < sources.size(); i++) + sources[i]->collectTilesWithLargeObjectsAtZoom(zooms); } void sortOutputObjectIDs( const std::vector& sortOrders, + const AttributeStore& attributeStore, std::vector& data ); std::vector TileDataSource::getObjectsForTile( const std::vector& sortOrders, + const AttributeStore& attributeStore, unsigned int zoom, TileCoordinates coordinates ) { std::vector data; collectObjectsForTile(zoom, coordinates, data); collectLargeObjectsForTile(zoom, coordinates, data); - sortOutputObjectIDs(sortOrders, data); + sortOutputObjectIDs(sortOrders, attributeStore, data); data.erase(unique(data.begin(), data.end()), data.end()); + orderStoredGeometryTies(data); return data; } +namespace { +struct GeometryTieKey { + double x, y; // first point + uint64_t points; // number of points + uint64_t hash; // of all points, only computed when the above are equal + OutputObjectID o; +}; + +// Adds a ring/linestring to the key: first point and point count, or (withHash) the hash of all points +template +void addToTieKey(GeometryTieKey& key, const R& ring, bool withHash) { + if (withHash) { + key.hash = stableCombine(key.hash, static_cast(ring.size())); + for (const auto& p : ring) { + key.hash = stableCombine(key.hash, static_cast(p.x())); + key.hash = stableCombine(key.hash, static_cast(p.y())); + } + return; + } + if (key.points == 0 && !ring.empty()) { + key.x = ring.front().x(); + key.y = ring.front().y(); + } + key.points += ring.size(); +} + +void geometryTieKey(const TileDataSource& source, GeometryTieKey& key, bool withHash) { + const NodeID objectID = key.o.oo.objectID; + switch (key.o.oo.geomType) { + case POINT_: { + const Point& p = source.retrievePoint(objectID); + addToTieKey(key, std::array{p}, withHash); + break; + } + case LINESTRING_: + addToTieKey(key, source.retrieveLinestring(objectID), withHash); + break; + case MULTILINESTRING_: + for (const auto& ls : source.retrieveMultiLinestring(objectID)) + addToTieKey(key, ls, withHash); + break; + case POLYGON_: + for (const auto& poly : source.retrieveMultiPolygon(objectID)) { + addToTieKey(key, poly.outer(), withHash); + for (const auto& inner : poly.inners()) + addToTieKey(key, inner, withHash); + } + break; + } +} +} + +// Geometries stored at runtime (relations, shapefiles, ...) have an objectID that +// depends on which thread stored them, so the sort ties objects that share layer, +// z_order, type, attributes and id in an arbitrary order. Order such runs by +// geometry instead. OSM ways and nodes carry their OSM id in the objectID, and +// sort after stored geometries, so they never need this. +void TileDataSource::orderStoredGeometryTies(std::vector& data) const { + const NodeID storedLimit = 1ull << TILE_DATA_ID_SIZE; + auto tied = [](const OutputObjectID& a, const OutputObjectID& b) { + return a.oo.layer == b.oo.layer && a.oo.z_order == b.oo.z_order && a.oo.geomType == b.oo.geomType && + a.oo.attributes == b.oo.attributes && a.id == b.id; + }; + + std::vector keys; + size_t i = 0; + while (i < data.size()) { + size_t j = i + 1; + if (data[i].oo.objectID < storedLimit) + while (j < data.size() && data[j].oo.objectID < storedLimit && tied(data[i], data[j])) j++; + if (j - i < 2) { i = j; continue; } + + keys.clear(); + for (size_t k = i; k < j; k++) { + keys.push_back(GeometryTieKey{0, 0, 0, 0, data[k]}); + geometryTieKey(*this, keys.back(), false); + } + auto byPoint = [](const GeometryTieKey& a, const GeometryTieKey& b) { + if (a.x != b.x) return a.x < b.x; + if (a.y != b.y) return a.y < b.y; + return a.points < b.points; + }; + std::sort(keys.begin(), keys.end(), byPoint); + + // Still tied: hash all points (identical geometries may stay in any order) + for (size_t a = 0; a < keys.size(); ) { + size_t b = a + 1; + while (b < keys.size() && !byPoint(keys[a], keys[b])) b++; + if (b - a > 1) { + for (size_t k = a; k < b; k++) geometryTieKey(*this, keys[k], true); + std::sort(keys.begin() + a, keys.begin() + b, [](const GeometryTieKey& p, const GeometryTieKey& q) { + if (p.hash != q.hash) return p.hash < q.hash; + return p.o.oo.objectID < q.o.oo.objectID; + }); + } + a = b; + } + + for (size_t k = i; k < j; k++) data[k] = keys[k - i].o; + i = j; + } +} + // ------------------------------------ // Add geometries to tile/large indices @@ -660,7 +807,7 @@ NodeID TileDataSource::storePoint(const Point& input) { const auto& store = pointStore.get(this); NodeID offset = store.second->size(); - store.second->emplace_back(input); + store.second->push_back(input); NodeID rv = (store.first << (TILE_DATA_ID_SIZE - shardBits)) + offset; return rv; } diff --git a/src/tile_sorting.cpp b/src/tile_sorting.cpp index 64588163..cc79e20d 100644 --- a/src/tile_sorting.cpp +++ b/src/tile_sorting.cpp @@ -208,26 +208,75 @@ template void sortOutputObjects( typename AppendVectorNS::AppendVector::Iterator end ); +namespace { +struct AttributeKeyCacheEntry { + AttributeIndex index; + uint64_t key; +}; +// Direct-mapped, per thread: neighbouring tiles share most attribute sets +const size_t ATTRIBUTE_KEY_CACHE_SIZE = 1 << 12; +thread_local std::vector tlsAttributeKeys; +thread_local const AttributeStore* tlsAttributeKeysStore = nullptr; + +struct KeyedOutputObjectID { + uint64_t attributeKey; + OutputObjectID o; +}; + +uint64_t attributeKey(const AttributeStore& attributeStore, AttributeIndex index) { + if (tlsAttributeKeysStore != &attributeStore) { + tlsAttributeKeys.assign(ATTRIBUTE_KEY_CACHE_SIZE, AttributeKeyCacheEntry{~0u, 0}); + tlsAttributeKeysStore = &attributeStore; + } + uint32_t slot = index; + slot ^= slot >> 16; slot *= 0x7feb352dU; slot ^= slot >> 15; + AttributeKeyCacheEntry& entry = tlsAttributeKeys[slot & (ATTRIBUTE_KEY_CACHE_SIZE - 1)]; + if (entry.index != index) + entry = AttributeKeyCacheEntry{index, attributeStore.contentHashUnsafe(index)}; + return entry.key; +} +} + void sortOutputObjectIDs( const std::vector& sortOrders, + const AttributeStore& attributeStore, std::vector& data ) { - // Lexicographic comparison, with the order of: layer, geomType, attributes, and objectID. + // Lexicographic comparison, with the order of: layer, geomType, attributes, id and objectID. // Note that attributes is preferred to objectID. // It is to arrange objects with the identical attributes continuously. // Such objects will be merged into one object, to reduce the size of output. - boost::sort::pdqsort(data.begin(), data.end(), [&sortOrders](const OutputObjectID& x, const OutputObjectID& y) -> bool { - if (x.oo.layer < y.oo.layer) return true; - if (x.oo.layer > y.oo.layer) return false; - if (x.oo.z_order < y.oo.z_order) return sortOrders[x.oo.layer]; - if (x.oo.z_order > y.oo.z_order) return !sortOrders[x.oo.layer]; - if (x.oo.geomType < y.oo.geomType) return true; - if (x.oo.geomType > y.oo.geomType) return false; - if (x.oo.attributes < y.oo.attributes) return true; - if (x.oo.attributes > y.oo.attributes) return false; - if (x.oo.objectID < y.oo.objectID) return true; - return false; + // + // Attribute sets are compared by content, not by AttributeIndex: the index is + // handed out in insertion order, which depends on thread scheduling. The same + // goes for the objectID of stored geometries, so the OSM id goes first; + // TileDataSource::orderStoredGeometryTies handles the remaining ties. + std::vector keyed; + keyed.reserve(data.size()); + for (const OutputObjectID& o : data) + keyed.push_back(KeyedOutputObjectID{attributeKey(attributeStore, o.oo.attributes), o}); + + boost::sort::pdqsort(keyed.begin(), keyed.end(), [&sortOrders, &attributeStore](const KeyedOutputObjectID& a, const KeyedOutputObjectID& b) -> bool { + const OutputObject& x = a.o.oo; + const OutputObject& y = b.o.oo; + if (x.layer < y.layer) return true; + if (x.layer > y.layer) return false; + if (x.z_order < y.z_order) return sortOrders[x.layer]; + if (x.z_order > y.z_order) return !sortOrders[x.layer]; + if (x.geomType < y.geomType) return true; + if (x.geomType > y.geomType) return false; + if (x.attributes != y.attributes) { + if (a.attributeKey != b.attributeKey) return a.attributeKey < b.attributeKey; + const int c = attributeStore.compareContentUnsafe(x.attributes, y.attributes); + if (c != 0) return c < 0; + return x.attributes < y.attributes; + } + if (a.o.id != b.o.id) return a.o.id < b.o.id; + return x.objectID < y.objectID; }); + + for (size_t i = 0; i < keyed.size(); i++) + data[i] = keyed[i].o; } void sortTileCoordinates( diff --git a/src/tilemaker.cpp b/src/tilemaker.cpp index 7a7ae1e5..bdecc1d4 100644 --- a/src/tilemaker.cpp +++ b/src/tilemaker.cpp @@ -97,11 +97,19 @@ int main(const int argc, const char* argv[]) { #ifdef __GLIBC__ // glibc raises its mmap threshold dynamically (up to 32/64 MB) once a large // mmapped chunk is freed; multi-MB buffers then land in per-thread arenas - // that are never trimmed, inflating peak RSS. Setting the trim threshold - // explicitly also disables that dynamic adjustment (the mmap threshold - // stays at its 128 KB default), unless the user tunes malloc via the environment. - if (!getenv("MALLOC_TRIM_THRESHOLD_") && !getenv("MALLOC_MMAP_THRESHOLD_") && !getenv("GLIBC_TUNABLES")) { - mallopt(M_TRIM_THRESHOLD, 4 << 20); + // that are never trimmed, inflating peak RSS. Setting malloc parameters + // explicitly turns that dynamic adjustment off, so both are set here: + // - a 16 MB top pad makes the heap grow and shrink in large steps; with many + // threads allocating, the default 128 KB steps cost many brk() calls and + // page faults + // - a fixed 8 MB mmap threshold: otherwise it would stay at 128 KB, and every + // larger buffer would be a fresh mmap with zeroed pages. Boost's validity + // check on a huge polygon grows such buffers over and over (one relation + // with 1.9M points took 3x as long). + // Users who tune malloc via the environment keep their settings. + if (!getenv("MALLOC_TRIM_THRESHOLD_") && !getenv("MALLOC_MMAP_THRESHOLD_") && !getenv("MALLOC_TOP_PAD_") && !getenv("GLIBC_TUNABLES")) { + mallopt(M_TOP_PAD, 16 << 20); + mallopt(M_MMAP_THRESHOLD, 8 << 20); } #endif @@ -446,7 +454,7 @@ int main(const int argc, const char* argv[]) { clock_gettime(CLOCK_MONOTONIC, &start); #endif std::cout << "collecting tiles" << std::flush; - populateTilesAtZoom(sources, zoomResults); + populateTilesAtZoom(sources, zoomResults, options.threadNum); #ifdef CLOCK_MONOTONIC clock_gettime(CLOCK_MONOTONIC, &end); uint64_t tileNs = 1e9 * (end.tv_sec - start.tv_sec) + end.tv_nsec - start.tv_nsec; @@ -504,8 +512,15 @@ int main(const int argc, const char* argv[]) { std::atomic batchCounter(0); const uint cutZoom = std::max(12, config.startZoom); - std::size_t batchSize = 0; - for(std::size_t startIndex = 0; startIndex < tileCoordinates.size(); startIndex += batchSize) { + // Batches are handed out as threads become free rather than all posted up + // front: a planet has ~14M of them, and their queued handlers held GBs. + std::mutex batchMutex; + std::size_t nextBatchStart = 0; + uint64_t nextBatchSeq = 0; + auto nextBatch = [&](std::size_t &startIndex, std::size_t &batchSize, uint64_t &seq) { + const std::lock_guard lock(batchMutex); + startIndex = nextBatchStart; + seq = nextBatchSeq++; // Compute how many tiles should be assigned to this batch -- // higher-zoom tiles are cheaper to compute, lower-zoom tiles more expensive. batchSize = 0; @@ -529,65 +544,89 @@ int main(const int argc, const char* argv[]) { batchSize++; } + nextBatchStart += batchSize; + return batchSize > 0; + }; - boost::asio::post(pool, [=, &tileCoordinates, &pool, &sharedData, &sources, &attributeStore, &io_mutex, &tilesWritten, &lastTilesWritten, &batchCounter]() { - clipCacheBatchId = ++batchCounter; - // A single-tile batch has no descendants to reuse its clips. - clipCacheNoStore = batchSize == 1; - std::vector tileTimings; - std::size_t endIndex = std::min(tileCoordinates.size(), startIndex + batchSize); - for(std::size_t i = startIndex; i < endIndex; ++i) { - unsigned int zoom = tileCoordinates[i].first; - TileCoordinates coords = tileCoordinates[i].second; + // .pmtiles and .mbtiles are written in batch order, so the file doesn't depend on thread scheduling + const bool toPMTiles = options.outputMode == OptionsParser::OutputMode::PMTiles; + const bool toMBTiles = options.outputMode == OptionsParser::OutputMode::MBTiles; + // While an earlier batch is still being worked on, each thread may run ahead by about + // 64 MB of finished tiles. With less, threads often wait for a slow batch (mostly a + // dense z12 area), above all when two threads share a core and each runs at half speed. + const uint64_t maxHeldBytes = (64ull << 20) * options.threadNum; + if (toPMTiles) sharedData.pmtiles.batches.setMaxHeldBytes(maxHeldBytes); + if (toMBTiles) sharedData.mbtiles.batches.setMaxHeldBytes(maxHeldBytes); + for (unsigned int t = 0; t < options.threadNum; t++) { + boost::asio::post(pool, [&]() { + std::size_t startIndex, batchSize; + uint64_t seq; + while (true) { + if (toPMTiles) sharedData.pmtiles.batches.waitForRoom(); + if (toMBTiles) sharedData.mbtiles.batches.waitForRoom(); + if (!nextBatch(startIndex, batchSize, seq)) break; + if (toPMTiles) sharedData.pmtiles.batches.begin(seq); + if (toMBTiles) sharedData.mbtiles.batches.begin(seq); + clipCacheBatchId = ++batchCounter; + // A single-tile batch has no descendants to reuse its clips. + clipCacheNoStore = batchSize == 1; + std::vector tileTimings; + std::size_t endIndex = std::min(tileCoordinates.size(), startIndex + batchSize); + for(std::size_t i = startIndex; i < endIndex; ++i) { + unsigned int zoom = tileCoordinates[i].first; + TileCoordinates coords = tileCoordinates[i].second; #ifdef CLOCK_MONOTONIC - timespec start, end; - if (options.logTileTimings) - clock_gettime(CLOCK_MONOTONIC, &start); + timespec start, end; + if (options.logTileTimings) + clock_gettime(CLOCK_MONOTONIC, &start); #endif - std::vector> data; - for (auto source : sources) { - data.emplace_back(source->getObjectsForTile(sortOrders, zoom, coords)); - } - outputProc(sharedData, sources, attributeStore, data, coords, zoom); + std::vector> data; + for (auto source : sources) { + data.emplace_back(source->getObjectsForTile(sortOrders, attributeStore, zoom, coords)); + } + outputProc(sharedData, sources, attributeStore, data, coords, zoom); #ifdef CLOCK_MONOTONIC - if (options.logTileTimings) { - clock_gettime(CLOCK_MONOTONIC, &end); - uint64_t tileNs = 1e9 * (end.tv_sec - start.tv_sec) + end.tv_nsec - start.tv_nsec; - std::string output = "z" + std::to_string(zoom) + "/" + std::to_string(coords.x) + "/" + std::to_string(coords.y) + " took " + std::to_string(tileNs/1e6) + " ms"; - tileTimings.push_back(output); - } + if (options.logTileTimings) { + clock_gettime(CLOCK_MONOTONIC, &end); + uint64_t tileNs = 1e9 * (end.tv_sec - start.tv_sec) + end.tv_nsec - start.tv_nsec; + std::string output = "z" + std::to_string(zoom) + "/" + std::to_string(coords.x) + "/" + std::to_string(coords.y) + " took " + std::to_string(tileNs/1e6) + " ms"; + tileTimings.push_back(output); + } #endif - } + } - if (options.logTileTimings) { - const std::lock_guard lock(io_mutex); - std::cout << std::endl; - for (const auto& output : tileTimings) - std::cout << output << std::endl; - } + if (options.logTileTimings) { + const std::lock_guard lock(io_mutex); + std::cout << std::endl; + for (const auto& output : tileTimings) + std::cout << output << std::endl; + } - tilesWritten += (endIndex - startIndex); - - if (io_mutex.try_lock()) { - uint64_t written = tilesWritten.load(); - - if (written >= lastTilesWritten + tileCoordinates.size() / 100 || ISATTY) { - lastTilesWritten = written; - // Show progress grouped by z6 (or lower) - size_t z = tileCoordinates[startIndex].first; - size_t x = tileCoordinates[startIndex].second.x; - size_t y = tileCoordinates[startIndex].second.y; - if (z > CLUSTER_ZOOM) { - x = x / (1 << (z - CLUSTER_ZOOM)); - y = y / (1 << (z - CLUSTER_ZOOM)); - z = CLUSTER_ZOOM; + if (toPMTiles) sharedData.pmtiles.batches.end(); + if (toMBTiles) sharedData.mbtiles.batches.end(); + tilesWritten += (endIndex - startIndex); + + if (io_mutex.try_lock()) { + uint64_t written = tilesWritten.load(); + + if (written >= lastTilesWritten + tileCoordinates.size() / 100 || ISATTY) { + lastTilesWritten = written; + // Show progress grouped by z6 (or lower) + size_t z = tileCoordinates[startIndex].first; + size_t x = tileCoordinates[startIndex].second.x; + size_t y = tileCoordinates[startIndex].second.y; + if (z > CLUSTER_ZOOM) { + x = x / (1 << (z - CLUSTER_ZOOM)); + y = y / (1 << (z - CLUSTER_ZOOM)); + z = CLUSTER_ZOOM; + } + cout << "z" << z << "/" << x << "/" << y << ", writing tile " << written << " of " << tileCoordinates.size() << " \r" << std::flush; } - cout << "z" << z << "/" << x << "/" << y << ", writing tile " << written << " of " << tileCoordinates.size() << " \r" << std::flush; + io_mutex.unlock(); } - io_mutex.unlock(); } }); } diff --git a/test/append_vector.test.cpp b/test/append_vector.test.cpp index b3104428..f0d5d704 100644 --- a/test/append_vector.test.cpp +++ b/test/append_vector.test.cpp @@ -21,6 +21,12 @@ MU_TEST(test_append_vector) { mu_check(vec[25] == 25); + const AppendVector& constVec = vec; + mu_check(constVec[25] == 25); + mu_check(constVec[8191] == 8191); + mu_check(constVec[8192] == 8192); + mu_check(constVec[9999] == 9999); + const AppendVector::Iterator& it = vec.begin(); mu_check(*it == 0); mu_check(*(it + 1) == 1); diff --git a/test/correct.test.cpp b/test/correct.test.cpp new file mode 100644 index 00000000..7178d173 --- /dev/null +++ b/test/correct.test.cpp @@ -0,0 +1,72 @@ +#include +#include +#include +#include "external/minunit.h" +#include "geometry/correct.hpp" + +typedef boost::geometry::model::d2::point_xy point_t; +typedef boost::geometry::model::polygon polygon_t; +typedef boost::geometry::model::multi_polygon multi_polygon_t; + +static bool near(double a, double b) { return std::abs(a - b) < 1e-9; } + +MU_TEST(test_correct_merges_overlapping_holes) { + // Two overlapping holes are merged into one; a disjoint hole is kept as is + polygon_t input; + boost::geometry::read_wkt( + "POLYGON((0 0,0 10,10 10,10 0,0 0)," + "(1 1,3 1,3 3,1 3,1 1),(2 2,4 2,4 4,2 4,2 2),(6 6,7 6,7 7,6 7,6 6))", input); + multi_polygon_t output; + geometry::correct(input, output); + + mu_check(output.size() == 1); + mu_check(output[0].inners().size() == 2); + mu_check(near(boost::geometry::area(output), 100 - 7 - 1)); + mu_check(boost::geometry::is_valid(output)); +} + +MU_TEST(test_correct_merges_overlapping_outers) { + multi_polygon_t input; + boost::geometry::read_wkt( + "MULTIPOLYGON(((0 0,0 2,2 2,2 0,0 0)),((1 1,1 3,3 3,3 1,1 1)),((5 5,5 6,6 6,6 5,5 5)))", input); + multi_polygon_t output; + geometry::correct(input, output); + + mu_check(output.size() == 2); + mu_check(near(boost::geometry::area(output), 4 + 4 - 1 + 1)); + mu_check(boost::geometry::is_valid(output)); +} + +MU_TEST(test_correct_many_disjoint_holes) { + // 400 disjoint holes, e.g. a lake with many islands: all are kept + std::string wkt = "POLYGON((0 0,0 100,100 100,100 0,0 0)"; + for (int x = 0; x < 20; x++) + for (int y = 0; y < 20; y++) { + double x0 = 1 + x * 5, y0 = 1 + y * 5; + wkt += ",(" + std::to_string(x0) + " " + std::to_string(y0) + "," + std::to_string(x0 + 2) + " " + std::to_string(y0) + "," + + std::to_string(x0 + 2) + " " + std::to_string(y0 + 2) + "," + std::to_string(x0) + " " + std::to_string(y0 + 2) + "," + + std::to_string(x0) + " " + std::to_string(y0) + ")"; + } + wkt += ")"; + polygon_t input; + boost::geometry::read_wkt(wkt, input); + multi_polygon_t output; + geometry::correct(input, output); + + mu_check(output.size() == 1); + mu_check(output[0].inners().size() == 400); + mu_check(near(boost::geometry::area(output), 100 * 100 - 400 * 4)); + mu_check(boost::geometry::is_valid(output)); +} + +MU_TEST_SUITE(test_suite_correct) { + MU_RUN_TEST(test_correct_merges_overlapping_holes); + MU_RUN_TEST(test_correct_merges_overlapping_outers); + MU_RUN_TEST(test_correct_many_disjoint_holes); +} + +int main() { + MU_RUN_SUITE(test_suite_correct); + MU_REPORT(); + return MU_EXIT_CODE; +} diff --git a/test/geom.test.cpp b/test/geom.test.cpp index e33b5990..2a31d97d 100644 --- a/test/geom.test.cpp +++ b/test/geom.test.cpp @@ -116,6 +116,44 @@ MU_TEST(test_precheck_many_holes) { checkPrecheck(mp, false); } +static Ring squareRing(double x, double y, double size) { + Ring r; + geom::append(r, Point(x, y)); geom::append(r, Point(x + size, y)); + geom::append(r, Point(x + size, y + size)); geom::append(r, Point(x, y + size)); + geom::append(r, Point(x, y)); + return r; +} + +MU_TEST(test_precheck_large_polygon) { + // A 40000-vertex outer ring (a circle) with 3600 holes: far more than walking the + // outer ring for every hole would allow (the old 2^26 work cap) + Polygon p; + const int n = 40000; + for (int i = 0; i <= n; i++) { + const double a = 2 * M_PI * (i % n) / n; + geom::append(p.outer(), Point(100 + 100 * std::cos(a), 100 + 100 * std::sin(a))); + } + for (int i = 0; i < 60; i++) + for (int j = 0; j < 60; j++) + p.inners().push_back(squareRing(40 + i * 2, 40 + j * 2, 1)); + MultiPolygon mp; + mp.push_back(p); + geom::correct(mp); + checkPrecheck(mp, true); + + // A hole outside the outer ring + MultiPolygon outside = mp; + outside[0].inners().push_back(squareRing(300, 100, 1)); + geom::correct(outside); + checkPrecheck(outside, false); + + // A hole nested in a hole on the other side of the polygon + MultiPolygon nested = mp; + nested[0].inners().insert(nested[0].inners().begin(), squareRing(40.25, 40.25, 0.5)); + geom::correct(nested); + checkPrecheck(nested, false); +} + MU_TEST_SUITE(test_suite_geom) { MU_RUN_TEST(test_rect_clip_concave); MU_RUN_TEST(test_rect_clip_holes); @@ -124,6 +162,7 @@ MU_TEST_SUITE(test_suite_geom) { MU_RUN_TEST(test_rect_clip_touch_from_inside); MU_RUN_TEST(test_precheck_overlapping_bboxes); MU_RUN_TEST(test_precheck_many_holes); + MU_RUN_TEST(test_precheck_large_polygon); } int main() { diff --git a/test/ordered_batches.test.cpp b/test/ordered_batches.test.cpp new file mode 100644 index 00000000..5fe47787 --- /dev/null +++ b/test/ordered_batches.test.cpp @@ -0,0 +1,89 @@ +#include +#include +#include +#include +#include +#include +#include "external/minunit.h" +#include "ordered_batches.h" + +MU_TEST(test_ordered_batches_single_thread) { + std::vector written; + OrderedBatches batches([&](int& item) { written.push_back(item); }, [](const int&) { return 1; }); + + // batch 1 finishes first, so it's held until batch 0 is written + batches.begin(1); + mu_check(batches.inBatch()); + batches.add(10); + batches.add(11); + batches.end(); + mu_check(!batches.inBatch()); + mu_check(written.empty()); + mu_check(!batches.allWritten()); + + batches.begin(0); + batches.add(0); + batches.end(); + mu_check(written == std::vector({ 0, 10, 11 })); + mu_check(batches.allWritten()); + + // an empty batch still moves the order on + batches.begin(3); + batches.add(30); + batches.end(); + batches.begin(2); + batches.end(); + mu_check(written == std::vector({ 0, 10, 11, 30 })); + mu_check(batches.allWritten()); +} + +MU_TEST(test_ordered_batches_threads) { + // Batches are handed out in order and finish in any order; with a small limit on what + // may be held, threads also have to wait for room + const uint64_t numBatches = 2000, itemsPerBatch = 5; + std::vector written; + OrderedBatches batches([&](uint64_t& item) { written.push_back(item); }, [](const uint64_t&) { return 1; }); + batches.setMaxHeldBytes(20); + + std::mutex handOut; + uint64_t nextSeq = 0; + std::vector threads; + for (int t = 0; t < 8; t++) { + threads.emplace_back([&, t]() { + std::mt19937 rng(t); + while (true) { + batches.waitForRoom(); + uint64_t seq; + { + std::lock_guard lock(handOut); + if (nextSeq == numBatches) return; + seq = nextSeq++; + } + batches.begin(seq); + for (uint64_t i = 0; i < itemsPerBatch; i++) { + if (rng() % 4 == 0) std::this_thread::yield(); + batches.add(seq * itemsPerBatch + i); + } + batches.end(); + } + }); + } + for (auto& thread : threads) thread.join(); + + mu_check(batches.allWritten()); + mu_check(written.size() == numBatches * itemsPerBatch); + bool inOrder = true; + for (uint64_t i = 0; i < written.size(); i++) if (written[i] != i) inOrder = false; + mu_check(inOrder); +} + +MU_TEST_SUITE(test_suite_ordered_batches) { + MU_RUN_TEST(test_ordered_batches_single_thread); + MU_RUN_TEST(test_ordered_batches_threads); +} + +int main() { + MU_RUN_SUITE(test_suite_ordered_batches); + MU_REPORT(); + return MU_EXIT_CODE; +} diff --git a/test/shp_mem_tiles.test.cpp b/test/shp_mem_tiles.test.cpp new file mode 100644 index 00000000..a0cbcb2f --- /dev/null +++ b/test/shp_mem_tiles.test.cpp @@ -0,0 +1,66 @@ +#include +#include "external/minunit.h" +#include "shp_mem_tiles.h" + +bool verbose = false; + +namespace { + MultiPolygon square(double x, double y, double size) { + MultiPolygon mp; + Polygon p; + geom::append(p.outer(), Point(x, y)); + geom::append(p.outer(), Point(x, y + size)); + geom::append(p.outer(), Point(x + size, y + size)); + geom::append(p.outer(), Point(x + size, y)); + geom::append(p.outer(), Point(x, y)); + mp.push_back(p); + return mp; + } + + Box box(double x1, double y1, double x2, double y2) { + return Box(Point(x1, y1), Point(x2, y2)); + } +} + +MU_TEST(test_may_intersect) { + Declutter declutter; + ShpMemTiles shp(1, 14, declutter); + shp.CreateNamedLayerIndex("layer"); + + // Two small squares at opposite corners: the shape's bounding box, (0,0)-(10,10), + // is almost all empty, like that of a country with far-flung islands. + MultiPolygon mp = square(0, 0, 1); + mp.push_back(square(9, 9, 1)[0]); + shp.StoreGeometry(0, "layer", POLYGON_, mp, true, true, "corners", 0, 0, 0); + + // Outside the shape's bounding box + mu_check(!shp.mayIntersect("layer", box(20, 20, 21, 21))); + + // A small box (a few z15 tiles) in the empty middle: checked tile by tile + mu_check(!shp.mayIntersect("layer", box(5, 5, 5.001, 5.001))); + // A large box (tens of thousands of z15 tiles) in the empty middle: checked at once + mu_check(!shp.mayIntersect("layer", box(3, 3, 7, 7))); + + // Small and large boxes that touch a square + mu_check(shp.mayIntersect("layer", box(0.5, 0.5, 0.501, 0.501))); + mu_check(shp.mayIntersect("layer", box(0.5, 0.5, 5, 5))); + mu_check(shp.mayIntersect("layer", box(5, 5, 9.5, 9.5))); + + // A large box that surrounds a square without containing a vertex of it + mu_check(shp.mayIntersect("layer", box(-1, -1, 2, 2))); + + // Answers stay the same once tiles have been checked and cached + mu_check(!shp.mayIntersect("layer", box(5, 5, 5.001, 5.001))); + mu_check(!shp.mayIntersect("layer", box(3, 3, 7, 7))); + mu_check(shp.mayIntersect("layer", box(0.5, 0.5, 0.501, 0.501))); +} + +MU_TEST_SUITE(test_suite_shp_mem_tiles) { + MU_RUN_TEST(test_may_intersect); +} + +int main() { + MU_RUN_SUITE(test_suite_shp_mem_tiles); + MU_REPORT(); + return MU_EXIT_CODE; +} diff --git a/test/tile_coordinates_set.test.cpp b/test/tile_coordinates_set.test.cpp index 87a054b8..973cc13c 100644 --- a/test/tile_coordinates_set.test.cpp +++ b/test/tile_coordinates_set.test.cpp @@ -45,10 +45,71 @@ MU_TEST(test_tile_coordinates_set) { mu_check(z2.test(1, 1) == true); mu_check(z2.test(2, 2) == false); } + + // forEachSet on a wrapped set visits the same tiles as a full scan, in x-major, y-minor order + { + PreciseTileCoordinatesSet z3(3); + z3.set(0, 0); z3.set(0, 5); z3.set(2, 1); z3.set(2, 7); z3.set(7, 3); + LossyTileCoordinatesSet z5(5, z3); + + std::vector> expected, visited; + for (TileCoordinate x = 0; x < 32; x++) + for (TileCoordinate y = 0; y < 32; y++) + if (z5.test(x, y)) expected.push_back({x, y}); + z5.forEachSet([&](TileCoordinate x, TileCoordinate y) { visited.push_back({x, y}); }); + + mu_check(expected.size() == 5 * 16); + mu_check(visited == expected); + } +} + +static bool sameTiles(const TileCoordinatesSet& a, const TileCoordinatesSet& b) { + std::vector> ta, tb; + a.forEachSet([&](TileCoordinate x, TileCoordinate y) { ta.emplace_back(x, y); }); + b.forEachSet([&](TileCoordinate x, TileCoordinate y) { tb.emplace_back(x, y); }); + return ta == tb && a.size() == b.size(); +} + +MU_TEST(test_tile_coordinates_set_box) { + // setBox must set exactly the tiles that set() would, including boxes that + // start and end mid-word, span several words, or are clipped at the edge + const TileCoordinate boxes[][4] = { + { 0, 0, 0, 0 }, { 3, 9, 5, 70 }, { 100, 101, 0, 1023 }, { 7, 7, 60, 200 }, + { 1000, 1023, 990, 1023 }, { 1020, 1023, 1000, 1023 } + }; + for (const auto& b : boxes) { + PreciseTileCoordinatesSet bySet(10), byBox(10); + for (TileCoordinate x = b[0]; x <= b[1]; x++) + for (TileCoordinate y = b[2]; y <= b[3]; y++) + bySet.set(x, y); + byBox.setBox(b[0], b[1], b[2], b[3]); + mu_check(sameTiles(bySet, byBox)); + } + // Partly outside the 1024x1024 grid: the part inside is set + PreciseTileCoordinatesSet edge(10); + edge.setBox(1020, 1030, 1022, 1030); + mu_check(edge.size() == 4 * 2); + mu_check(edge.test(1023, 1023)); +} + +MU_TEST(test_tile_coordinates_set_merge) { + PreciseTileCoordinatesSet a(12), b(12), both(12); + for (TileCoordinate i = 0; i < 4096; i += 37) { + a.set(i, (i * 7) % 4096); + both.set(i, (i * 7) % 4096); + } + for (TileCoordinate i = 0; i < 4096; i += 53) { + b.set((i * 3) % 4096, i); + both.set((i * 3) % 4096, i); + } + a.merge(b); + mu_check(sameTiles(a, both)); } MU_TEST_SUITE(test_suite_tile_coordinates_set) { MU_RUN_TEST(test_tile_coordinates_set); + MU_RUN_TEST(test_tile_coordinates_set_box); + MU_RUN_TEST(test_tile_coordinates_set_merge); } int main() { From a5c518cc02609bb2413074f9a7e1440f9fcbed87 Mon Sep 17 00:00:00 2001 From: dktue <2878623+dktue@users.noreply.github.com> Date: Sat, 7 Nov 2026 18:18:30 +0100 Subject: [PATCH 02/10] test: add #986 (sort the tile list with at most 16 threads) Same change as #986. With it, the sort before the tile phase takes ~13 s instead of 2-5 minutes at 96 threads, so planet times no longer vary by minutes between runs (14:12 / 14:08 in memory at 96 threads). Co-Authored-By: Claude Opus 5.5 (1M context) --- src/tile_sorting.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/tile_sorting.cpp b/src/tile_sorting.cpp index cc79e20d..3bb9442f 100644 --- a/src/tile_sorting.cpp +++ b/src/tile_sorting.cpp @@ -335,5 +335,7 @@ void sortTileCoordinates( return false; }, - threadNum); + // block_indirect_sort slows down badly with many threads: on the planet (346M tiles) it took + // 200-300 s at 96 threads instead of about 7 s at 16, with the threads spinning on its internal lock. + std::min(threadNum, 16)); } \ No newline at end of file From 154852c167a75c0ac5f70c591583be19f547e207 Mon Sep 17 00:00:00 2001 From: dktue <2878623+dktue@users.noreply.github.com> Date: Sun, 8 Nov 2026 10:09:13 +0100 Subject: [PATCH 03/10] perf: wake threads waiting for room only once there is room end() woke every thread waiting in waitForRoom() after each batch it wrote, although most of the time the held tiles were still over the limit. At 96 threads that meant dozens of threads waking, finding no room and sleeping again for every batch: 125M context switches and 7,700 s of system time on the planet. Now they are only woken once the held tiles are back under the limit. Planet in memory, 96 threads, all open performance PRs: 14:05 -> 12:45 (-9.5%), system time 7,708 -> 2,868 s, context switches 125M -> 35M. Output unchanged (same batches, same order). Co-Authored-By: Claude Opus 5.5 (1M context) --- include/ordered_batches.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/include/ordered_batches.h b/include/ordered_batches.h index 2d2b3b90..d67dff4a 100644 --- a/include/ordered_batches.h +++ b/include/ordered_batches.h @@ -68,7 +68,10 @@ class OrderedBatches { lock.lock(); heldBytes -= batch.first; nextSeq++; - heldShrunk.notify_all(); + // Wake the waiting threads only once there is room. Woken for every batch + // written, they all found no room and went back to sleep, again and again: + // most of the system time of a 96-thread planet run. + if (heldBytes <= maxHeldBytes) heldShrunk.notify_all(); } draining = false; } From 89208f38bcc1d8a2206721e54895832cfb83aee6 Mon Sep 17 00:00:00 2001 From: dktue <2878623+dktue@users.noreply.github.com> Date: Sun, 8 Nov 2026 10:09:13 +0100 Subject: [PATCH 04/10] perf: clip large polygons once per z8 tile and share the piece Every tile batch that needs a large polygon copied it from the store and clipped it from scratch, at every zoom. Now polygons with at least 10,000 points are clipped once per z8 tile, before the tiles are written (all threads, largest polygon first), and tiles below clip from that piece. After that the pieces are only read, so there is no lock while tiles are written; a z6 tile's pieces are freed once no tile in it is still to come. A piece only depends on the polygon and the z8 tile, so the output stays the same for any thread count. A piece whose clip needed the repair path is not used: once one came out empty and a lake vanished from all tiles below. Planet in memory, 96 threads, all open performance PRs + #983's wake-up fix: 12:45 -> 10:55 (-14.4%), user CPU -27%, peak RSS +7.5 GB. Germany byte-identical at 32 and 8 threads; planet sample: same feature counts, area within 0.003%. Co-Authored-By: Claude Opus 5.5 (1M context) --- include/clip_cache.h | 7 ++- include/osm_mem_tiles.h | 2 + include/tile_data.h | 21 +++++++ src/tile_data.cpp | 126 ++++++++++++++++++++++++++++++++++++++++ src/tilemaker.cpp | 42 ++++++++++++++ 5 files changed, 197 insertions(+), 1 deletion(-) diff --git a/include/clip_cache.h b/include/clip_cache.h index 66a400ab..e650671a 100644 --- a/include/clip_cache.h +++ b/include/clip_cache.h @@ -16,6 +16,9 @@ class TileBbox; extern thread_local uint64_t clipCacheBatchId; // Set when the current batch can't reuse clips (a single tile), so add() is skipped. extern thread_local bool clipCacheNoStore; +// Set while a shared piece of a large polygon is computed (TileDataSource::precomputeSharedPieces): +// the batch cache is then neither read nor written. +extern thread_local bool clipCacheBypass; template class ClipCache { @@ -36,6 +39,8 @@ class ClipCache { } Entry getEntry(uint zoom, TileCoordinate x, TileCoordinate y, NodeID objectID) const { + if (clipCacheBypass) + return Entry(); // Look for a previously clipped version at z-1, z-2, ... (nearest ancestor first) LocalStore& s = local(zoom, TileCoordinates(x, y)); Entry rv; @@ -59,7 +64,7 @@ class ClipCache { void add(const TileBbox& bbox, const NodeID objectID, const T& output, bool valid = false) { // The point of caching is to reuse the clip, so caching at the terminal zoom is // pointless. - if (bbox.zoom == baseZoom || clipCacheNoStore) + if (bbox.zoom == baseZoom || clipCacheNoStore || clipCacheBypass) return; // Levels beyond MAX_LEVELS could never be looked up by a child tile anyway diff --git a/include/osm_mem_tiles.h b/include/osm_mem_tiles.h index 1966bc4b..5ab90fc1 100644 --- a/include/osm_mem_tiles.h +++ b/include/osm_mem_tiles.h @@ -57,6 +57,8 @@ class OsmMemTiles : public TileDataSource { void populateLinestring(Linestring& ls, NodeID objectID) const; Linestring& getOrBuildLinestring(NodeID objectID) const; void populateMultiPolygon(MultiPolygon& dst, NodeID objectID) override; + // Polygons from ways are built from the way store, only relations are in the multipolygon store + bool storesMultiPolygon(NodeID objectID) const override { return objectID < OSM_THRESHOLD; } // Way-store polygons are only emitted by OsmLuaProcessing::Layer when // CorrectGeometry found them Valid (or Reversed: valid after reversing the ring) // on the same lon/latp coordinates. diff --git a/include/tile_data.h b/include/tile_data.h index 729ef009..b7ceaf71 100644 --- a/include/tile_data.h +++ b/include/tile_data.h @@ -8,7 +8,10 @@ #include #include #include +#include +#include #include +#include #include #include #include "append_vector.h" @@ -419,6 +422,24 @@ class TileDataSource { void orderStoredGeometryTies(std::vector& data) const; virtual Geometry buildWayGeometry(OutputGeometryType const geomType, NodeID const objectID, const TileBbox &bbox); + + // Large polygons are clipped once per SHARED_PIECE_ZOOM tile before the tiles are written, + // and deeper tiles clip from that piece instead of the whole polygon (see tile_data.cpp). + static constexpr unsigned SHARED_PIECE_ZOOM = 8; + // present(x, y): whether any tile below SHARED_PIECE_ZOOM tile x/y will be written + void precomputeSharedPieces(const std::function &present, unsigned threadNum); + // Drop the shared pieces inside a z6 tile, once no tile in it is still to be written + void releaseSharedPieces(TileCoordinate z6x, TileCoordinate z6y); + // Whether objectID's polygon is kept in the multipolygon store (retrieveMultiPolygon) + virtual bool storesMultiPolygon(NodeID objectID) const { return true; } + struct SharedPiece { + std::shared_ptr geom; // null: no usable piece, clip the whole polygon + bool valid = false; + }; + // Filled by precomputeSharedPieces, then only read while the tiles are written (no lock). + // releaseSharedPieces resets the pieces of a z6 tile no thread reads any more. + std::unordered_set pieceObjects; + std::map, std::shared_ptr> sharedPieces; virtual LatpLon buildNodeGeometry(NodeID const objectID, const TileBbox &bbox) const; void open() { diff --git a/src/tile_data.cpp b/src/tile_data.cpp index c1fcc02e..cdee86a3 100644 --- a/src/tile_data.cpp +++ b/src/tile_data.cpp @@ -90,6 +90,118 @@ TileDataSource::TileDataSource(size_t threadNum, unsigned int indexZoom, bool in thread_local std::vector>* tlsPendingSmallIndexObjects = nullptr; thread_local uint64_t clipCacheBatchId = 0; thread_local bool clipCacheNoStore = false; +thread_local bool clipCacheBypass = false; +thread_local bool clipNeededRepair = false; // set when a POLYGON_ clip went down the repair path +thread_local bool clipResultValid = false; // whether the last POLYGON_ clip is known to be valid + +// Shared pieces. Every tile batch that needs a large polygon (often 100,000+ points) used to copy +// it from the store and clip it from scratch, at every zoom. Instead, polygons with at least +// SHARED_PIECE_MIN_POINTS points are clipped once per SHARED_PIECE_ZOOM tile before the tiles are +// written, and tiles below clip from that piece. A piece only depends on the polygon and the tile, +// never on the thread or the order, so the output stays the same for any thread count. +// +// The pieces are computed up front, all threads at once and largest polygon first: computed on +// demand, a slow piece (up to ~15 s on the planet) held up every thread whose tiles needed it, +// a third of the tile phase on a 96-thread planet run. After that they are only read, so the +// tile threads look them up without a lock. +#define SHARED_PIECE_MIN_POINTS 10000 + +void TileDataSource::precomputeSharedPieces(const std::function &present, unsigned threadNum) { + const unsigned zoom = SHARED_PIECE_ZOOM; + if (zoom > indexZoom) return; + const unsigned shift = indexZoom - zoom; + // Candidates: polygons in the large-object index (those spanning 16+ index tiles), with the + // index tiles they cover + std::map boxes; + auto add = [&](const Box &box, const OutputObject &oo) { + if (oo.geomType != POLYGON_ || !storesMultiPolygon(oo.objectID)) return; + auto it = boxes.find(oo.objectID); + if (it == boxes.end()) boxes.emplace(oo.objectID, box); + else boost::geometry::expand(it->second, box); + }; + for (auto const &r : boxRtree) add(r.first, r.second); + for (auto const &r : boxRtreeWithIds) add(r.first, r.second.oo); + if (boxes.empty()) return; + const std::vector> candidates(boxes.begin(), boxes.end()); + + // The pieces each candidate needs: tiles below which a tile is written, that its envelope reaches + struct Job { std::tuple key; uint64_t points; }; + std::vector jobs; + std::mutex jobsMutex; + std::atomic next(0); + auto findJobs = [&]() { + std::vector local; + for (size_t i = next++; i < candidates.size(); i = next++) { + const NodeID objectID = candidates[i].first; + uint64_t points = 0; + Box envelope; + boost::geometry::assign_inverse(envelope); + for (auto const &p : retrieveMultiPolygon(objectID)) { + points += p.outer().size(); + for (auto const &inner : p.inners()) points += inner.size(); + for (auto const &pt : p.outer()) boost::geometry::expand(envelope, pt); + } + if (points < SHARED_PIECE_MIN_POINTS) continue; + const Box &box = candidates[i].second; + const TileCoordinate x0 = (TileCoordinate)box.min_corner().x() >> shift, x1 = (TileCoordinate)box.max_corner().x() >> shift; + const TileCoordinate y0 = (TileCoordinate)box.min_corner().y() >> shift, y1 = (TileCoordinate)box.max_corner().y() >> shift; + for (TileCoordinate x = x0; x <= x1; x++) + for (TileCoordinate y = y0; y <= y1; y++) + if (present(x, y) && boost::geometry::intersects(envelope, TileBbox(TileCoordinates(x, y), zoom, false, false).clippingBox)) + local.push_back(Job { std::make_tuple(objectID, x, y), points }); + } + std::lock_guard lock(jobsMutex); + jobs.insert(jobs.end(), local.begin(), local.end()); + }; + { + boost::asio::thread_pool pool(threadNum); + for (unsigned t = 0; t < threadNum; t++) boost::asio::post(pool, findJobs); + pool.join(); + } + std::sort(jobs.begin(), jobs.end(), [](const Job &a, const Job &b) { return a.points != b.points ? a.points > b.points : a.key < b.key; }); + for (const Job &job : jobs) { + pieceObjects.insert(std::get<0>(job.key)); + sharedPieces.emplace(job.key, nullptr); + } + + // Compute them, largest polygon first. Each job writes its own map entry; the map itself + // doesn't change any more. + next = 0; + auto compute = [&]() { + for (size_t i = next++; i < jobs.size(); i = next++) { + const auto &key = jobs[i].key; + auto piece = std::make_shared(); + try { + clipCacheBypass = true; + clipNeededRepair = false; + Geometry g = buildWayGeometry(POLYGON_, std::get<0>(key), TileBbox(TileCoordinates(std::get<1>(key), std::get<2>(key)), zoom, false, false)); + // A piece that needed the repair path can be wrong (once a lake came out empty), and + // every tile below it would inherit that: leave those tiles to clip the whole polygon. + if (!clipNeededRepair) { + piece->geom = std::make_shared(std::move(boost::get(g))); + // buildWayGeometry has just checked it (for a huge piece, is_valid takes seconds) + piece->valid = clipResultValid; + } + } catch (std::exception &) { + // The tiles clip the whole polygon, and report the problem there + } + clipCacheBypass = false; + sharedPieces.find(key)->second = piece; + } + }; + { + boost::asio::thread_pool pool(threadNum); + for (unsigned t = 0; t < threadNum; t++) boost::asio::post(pool, compute); + pool.join(); + } +} + +void TileDataSource::releaseSharedPieces(TileCoordinate z6x, TileCoordinate z6y) { + const unsigned shift = SHARED_PIECE_ZOOM - CLUSTER_ZOOM; + for (auto &e : sharedPieces) + if ((std::get<1>(e.first) >> shift) == z6x && (std::get<2>(e.first) >> shift) == z6y) + e.second.reset(); +} void TileDataSource::finalize(size_t threadNum) { uint64_t finalized = 0; @@ -333,6 +445,15 @@ Geometry TileDataSource::buildWayGeometry(OutputGeometryType const geomType, auto cachedEntry = multiPolygonClipCache.getEntry(bbox.zoom, bbox.index.x, bbox.index.y, objectID); std::shared_ptr cachedClip = cachedEntry.geom; bool inputKnownValid = cachedClip == nullptr ? polygonKnownValid(objectID) : cachedEntry.valid; + if (cachedClip == nullptr && bbox.zoom > SHARED_PIECE_ZOOM && pieceObjects.count(objectID)) { + // A large polygon: clip from its shared piece + const unsigned shift = bbox.zoom - SHARED_PIECE_ZOOM; + auto it = sharedPieces.find(std::make_tuple(objectID, bbox.index.x >> shift, bbox.index.y >> shift)); + if (it != sharedPieces.end() && it->second && it->second->geom) { + cachedClip = it->second->geom; + inputKnownValid = it->second->valid; + } + } MultiPolygon uncached; @@ -404,6 +525,7 @@ Geometry TileDataSource::buildWayGeometry(OutputGeometryType const geomType, // fast_clip was the identity on a geometry that already passed // is_valid, so correct() is a no-op and it is still valid. multiPolygonClipCache.add(bbox, objectID, mp, true); + clipResultValid = true; return mp; } geom::correct(mp); @@ -420,12 +542,14 @@ Geometry TileDataSource::buildWayGeometry(OutputGeometryType const geomType, if (rect_clip(cachedClip == nullptr && !keepUncached ? original : input, box, clipped) && (multiPolygonDefinitelyValid(clipped) || geom::is_valid(clipped))) { multiPolygonClipCache.add(bbox, objectID, clipped, true); + clipResultValid = true; return clipped; } } if (!valid) valid = geom::is_valid(mp,failure); if (!valid) { + clipNeededRepair = true; if (verbose) { // Build the whole line first and emit it with a single stream write: // tilemaker runs multi-threaded and chained operator<< calls are not @@ -461,6 +585,7 @@ Geometry TileDataSource::buildWayGeometry(OutputGeometryType const geomType, repairMultiPolygon(output, objectID, &outputValid); // Children of this tile can then take the fast paths for valid input multiPolygonClipCache.add(bbox, objectID, output, outputValid); + clipResultValid = outputValid; return output; } else if (!valid) { // occasionally also wrong_topological_dimension, disconnected_interior: @@ -471,6 +596,7 @@ Geometry TileDataSource::buildWayGeometry(OutputGeometryType const geomType, } multiPolygonClipCache.add(bbox, objectID, mp, valid); + clipResultValid = valid; return mp; } diff --git a/src/tilemaker.cpp b/src/tilemaker.cpp index bdecc1d4..a2643cac 100644 --- a/src/tilemaker.cpp +++ b/src/tilemaker.cpp @@ -510,6 +510,19 @@ int main(const int argc, const char* argv[]) { // Cluster tiles: breadth-first for z0..z5, depth-first for z6 sortTileCoordinates(config.baseZoom, options.threadNum, tileCoordinates); + // Shared pieces of large polygons, for the SHARED_PIECE_ZOOM tiles below which a tile is written + { + const unsigned pieceZoom = TileDataSource::SHARED_PIECE_ZOOM; + std::vector piecePresent((size_t)1 << (2 * pieceZoom), false); + for (auto const &t : tileCoordinates) + if (t.first > pieceZoom) + piecePresent[((size_t)(t.second.x >> (t.first - pieceZoom)) << pieceZoom) + (t.second.y >> (t.first - pieceZoom))] = true; + auto present = [&](TileCoordinate x, TileCoordinate y) { + return x < (1u << pieceZoom) && y < (1u << pieceZoom) && piecePresent[((size_t)x << pieceZoom) + y]; + }; + for (auto source : sources) source->precomputeSharedPieces(present, options.threadNum); + } + std::atomic batchCounter(0); const uint cutZoom = std::max(12, config.startZoom); // Batches are handed out as threads become free rather than all posted up @@ -517,6 +530,20 @@ int main(const int argc, const char* argv[]) { std::mutex batchMutex; std::size_t nextBatchStart = 0; uint64_t nextBatchSeq = 0; + // Shared pieces (TileDataSource::precomputeSharedPieces) lie inside one z6 tile. Tiles from z6 on + // are written z6 tile by z6 tile, so once the hand-out has moved past a z6 tile and none of its + // batches is still running, no tile needs its pieces any more. + std::vector z6Running(CLUSTER_ZOOM_WIDTH * CLUSTER_ZOOM_WIDTH, 0); + int64_t z6HandedOut = -1; + auto z6Of = [&](std::size_t i) -> int64_t { + const unsigned z = tileCoordinates[i].first; + if (z < CLUSTER_ZOOM) return -1; + return (int64_t)(tileCoordinates[i].second.x >> (z - CLUSTER_ZOOM)) * CLUSTER_ZOOM_WIDTH + (tileCoordinates[i].second.y >> (z - CLUSTER_ZOOM)); + }; + auto releaseZ6 = [&](int64_t b) { + for (auto source : sources) source->releaseSharedPieces(b / CLUSTER_ZOOM_WIDTH, b % CLUSTER_ZOOM_WIDTH); + }; + static thread_local std::vector batchZ6; // z6 tiles of the batch this thread is working on auto nextBatch = [&](std::size_t &startIndex, std::size_t &batchSize, uint64_t &seq) { const std::lock_guard lock(batchMutex); startIndex = nextBatchStart; @@ -545,6 +572,16 @@ int main(const int argc, const char* argv[]) { batchSize++; } nextBatchStart += batchSize; + batchZ6.clear(); + for (std::size_t i = startIndex; i < startIndex + batchSize; i++) { + const int64_t b = z6Of(i); + if (b >= 0 && (batchZ6.empty() || batchZ6.back() != b)) batchZ6.push_back(b); + } + for (int64_t b : batchZ6) z6Running[b]++; + if (!batchZ6.empty() && batchZ6.back() != z6HandedOut) { + if (z6HandedOut >= 0 && z6Running[z6HandedOut] == 0) releaseZ6(z6HandedOut); + z6HandedOut = batchZ6.back(); + } return batchSize > 0; }; @@ -605,6 +642,11 @@ int main(const int argc, const char* argv[]) { std::cout << output << std::endl; } + { + const std::lock_guard lock(batchMutex); + for (int64_t b : batchZ6) + if (--z6Running[b] == 0 && b != z6HandedOut) releaseZ6(b); + } if (toPMTiles) sharedData.pmtiles.batches.end(); if (toMBTiles) sharedData.mbtiles.batches.end(); tilesWritten += (endIndex - startIndex); From 528bb9992faf8e399f30fe3f8e7de687a4787607 Mon Sep 17 00:00:00 2001 From: dktue <2878623+dktue@users.noreply.github.com> Date: Sun, 8 Nov 2026 14:11:34 +0100 Subject: [PATCH 05/10] perf(pmtiles): place and write tiles in two ordered steps At 96 threads the one thread that writes batches in order became the bottleneck of the tile phase: busy 260 s of a 323 s tile phase, with the others waiting for room for 5,100 thread-seconds. Writing the bytes was more than half of its work (ofstream::write, and a tellp() per tile). - The writer keeps its own tile data offset instead of calling tellp() for every tile, and the stream gets a 4 MB buffer. - OrderedBatches can take a second step, flushItem: also in batch order and one thread at a time, but run by whichever thread is free, so it doesn't hold up the first step. PMTiles places tiles (offset, repeated tiny tiles, index) in the first step and writes their bytes in the second. The file is byte-identical. Planet in memory, 96 threads, all open performance PRs: 10:41 -> 9:54 (-7.3%); tile phase 323 -> 245 s, threads busy 76 -> 94 of 96 on average. New test: the second step runs in order, never twice at once, and always after the first (also clean under TSan). Co-Authored-By: Claude Opus 5.5 (1M context) --- include/ordered_batches.h | 53 +++++++++++++++++++++++++++++---- include/pmtiles.h | 7 +++-- src/pmtiles.cpp | 20 +++++++++++-- test/ordered_batches.test.cpp | 56 +++++++++++++++++++++++++++++++++++ 4 files changed, 125 insertions(+), 11 deletions(-) diff --git a/include/ordered_batches.h b/include/ordered_batches.h index d67dff4a..9531565f 100644 --- a/include/ordered_batches.h +++ b/include/ordered_batches.h @@ -3,6 +3,7 @@ #define _ORDERED_BATCHES_H #include +#include #include #include #include @@ -21,11 +22,15 @@ // being worked on by a thread that isn't waiting, so waitForRoom() always returns. // // writeItem is called by one thread at a time, with writeMutex() held, in batch order. +// If flushItem is given, a batch then goes through flushItem as well: also one thread at a time +// and in batch order, but without holding up writeItem. Whichever thread is free does it, so the +// two steps run side by side. A batch counts as held until it has been flushed. template class OrderedBatches { public: - OrderedBatches(std::function writeItem, std::function itemBytes) - : writeItem(writeItem), itemBytes(itemBytes) { } + OrderedBatches(std::function writeItem, std::function itemBytes, + std::function flushItem = nullptr) + : writeItem(writeItem), itemBytes(itemBytes), flushItem(flushItem) { } void setMaxHeldBytes(uint64_t bytes) { std::lock_guard lock(m); @@ -55,7 +60,10 @@ class OrderedBatches { heldBytes += bytes; held.emplace(c.seq, std::make_pair(bytes, std::move(c.items))); c.items = std::vector(); - if (draining) return; + if (draining) { + flushQueued(lock); + return; + } draining = true; while (!held.empty() && held.begin()->first == nextSeq) { auto batch = std::move(held.begin()->second); @@ -66,24 +74,33 @@ class OrderedBatches { for (auto& item : batch.second) writeItem(item); } lock.lock(); - heldBytes -= batch.first; nextSeq++; + if (flushItem) { + toFlush.emplace_back(std::move(batch)); + heldShrunk.notify_one(); // a waiting thread can flush it + continue; + } + heldBytes -= batch.first; // Wake the waiting threads only once there is room. Woken for every batch // written, they all found no room and went back to sleep, again and again: // most of the system time of a 96-thread planet run. if (heldBytes <= maxHeldBytes) heldShrunk.notify_all(); } draining = false; + flushQueued(lock); } void waitForRoom() { std::unique_lock lock(m); - heldShrunk.wait(lock, [&]() { return heldBytes <= maxHeldBytes; }); + while (heldBytes > maxHeldBytes) { + if (!toFlush.empty() && !flushing) flushQueued(lock); + else heldShrunk.wait(lock); + } } bool allWritten() { std::lock_guard lock(m); - return held.empty(); + return held.empty() && toFlush.empty() && !flushing; } // Held while items are written; lock it to write anything outside a batch @@ -95,6 +112,27 @@ class OrderedBatches { uint64_t seq = 0; std::vector items; }; + // Flush the batches waiting for flushItem, unless another thread already does; called and + // returns with m held + void flushQueued(std::unique_lock &lock) { + if (!flushItem || flushing) return; + flushing = true; + while (!toFlush.empty()) { + auto batch = std::move(toFlush.front()); + toFlush.pop_front(); + lock.unlock(); + for (auto& item : batch.second) flushItem(item); + batch.second = std::vector(); + lock.lock(); + heldBytes -= batch.first; + if (heldBytes <= maxHeldBytes) heldShrunk.notify_all(); + } + flushing = false; + // batches queued while finishing the last one are taken by the next thread that comes by; + // wake one in case all others wait for room + heldShrunk.notify_one(); + } + // The batch this thread is working on (one per thread and item type) static Current& current() { static thread_local Current c; @@ -103,6 +141,7 @@ class OrderedBatches { std::function writeItem; std::function itemBytes; + std::function flushItem; uint64_t maxHeldBytes = 1ull << 30; std::mutex m; // guards the fields below std::mutex w; // held while writing @@ -111,6 +150,8 @@ class OrderedBatches { uint64_t nextSeq = 0; uint64_t heldBytes = 0; std::map>> held; // seq -> (bytes, items) + std::deque>> toFlush; // written in order, waiting for flushItem + bool flushing = false; // a thread is flushing }; #endif //_ORDERED_BATCHES_H diff --git a/include/pmtiles.h b/include/pmtiles.h index bcc701f1..fb2e6682 100644 --- a/include/pmtiles.h +++ b/include/pmtiles.h @@ -38,7 +38,7 @@ struct TileOffset { struct PendingTile { uint64_t tileId; bool isTiny; - std::string data; + std::string data; // after writeTile: the bytes flushTile appends to the file (empty if a repeat) }; class PMTiles { @@ -60,6 +60,8 @@ class PMTiles { private: std::ofstream outputStream; + std::vector outputBuffer; // a large stream buffer: one thread writes all tiles + uint64_t tileDataEnd = 0; // bytes of tile data written so far (tellp() costs a flush and a seek per tile) std::mutex fileMutex; // guards leaf directory writes in close() uint64_t leafStart = 0; uint64_t numTilesWritten = 0; @@ -69,7 +71,8 @@ class PMTiles { std::vector denseIndex; std::unordered_map tinyCache; - void writeTile(PendingTile &tile); // with batches.writeMutex() held + void writeTile(PendingTile &tile); // with batches.writeMutex() held: places the tile + void flushTile(PendingTile &tile); // in the same order, by one thread at a time: writes its bytes void appendWithRLE(std::vector &entries, pmtiles::entryv3 &entry); void appendTileEntry(uint64_t tileId, TileOffset offset, std::vector &rootEntries, std::vector &entries); void flushEntries(std::vector &rootEntries, std::vector &entries); diff --git a/src/pmtiles.cpp b/src/pmtiles.cpp index fa520f34..dceb5af4 100644 --- a/src/pmtiles.cpp +++ b/src/pmtiles.cpp @@ -10,13 +10,19 @@ #include "helpers.h" TileOffset::TileOffset() { } +// Placing tiles (offset, repeats, index) and writing their bytes are two steps, each in batch order +// and one thread at a time, but run by different threads side by side: one thread doing both was +// the bottleneck of the tile phase at 96 threads. PMTiles::PMTiles() : batches([this](PendingTile &tile) { writeTile(tile); }, - [](const PendingTile &tile) { return tile.data.size(); }) { } + [](const PendingTile &tile) { return tile.data.size(); }, + [this](PendingTile &tile) { flushTile(tile); }) { } PMTiles::~PMTiles() { } void PMTiles::open(std::string &filename) { std::cout << "Creating pmtiles at " << filename << std::endl; + outputBuffer.resize(4 << 20); + outputStream.rdbuf()->pubsetbuf(outputBuffer.data(), outputBuffer.size()); outputStream.open(filename, std::ios::out | std::ios::trunc | std::ios::binary); // dummy header/root directory for now - we'll write it all later char header[HEADER_ROOT] = "PMTiles"; @@ -145,6 +151,12 @@ void PMTiles::saveTile(int zoom, int x, int y, std::string &data) { } std::lock_guard lock(batches.writeMutex()); writeTile(tile); + flushTile(tile); +} + +void PMTiles::flushTile(PendingTile &tile) { + if (!tile.data.empty()) outputStream.write(tile.data.c_str(), tile.data.size()); + tile.data = std::string(); } // Write a tile to file and store it in the index @@ -154,16 +166,18 @@ void PMTiles::writeTile(PendingTile &tile) { auto cached = tile.isTiny ? tinyCache.find(tile.data) : tinyCache.end(); if (cached != tinyCache.end()) { offset = cached->second; + tile.data = std::string(); // already in the file } else { std::string compressed = tile.isTiny ? compress_string(tile.data, Z_DEFAULT_COMPRESSION, true) : std::move(tile.data); - offset = TileOffset(static_cast(outputStream.tellp()) - HEADER_ROOT, compressed.size()); - outputStream.write(compressed.c_str(), compressed.size()); + offset = TileOffset(tileDataEnd, compressed.size()); + tileDataEnd += compressed.size(); numTilesWritten++; // add to cache if tiny if (tile.isTiny) { if (tinyCache.size()>TINY_MAX_SIZE) tinyCache.clear(); tinyCache.insert({ tile.data, offset }); } + tile.data = std::move(compressed); // flushTile writes it } // store in index diff --git a/test/ordered_batches.test.cpp b/test/ordered_batches.test.cpp index 5fe47787..b9ff9599 100644 --- a/test/ordered_batches.test.cpp +++ b/test/ordered_batches.test.cpp @@ -77,9 +77,65 @@ MU_TEST(test_ordered_batches_threads) { mu_check(inOrder); } +MU_TEST(test_ordered_batches_flush) { + // With flushItem: every item is written and then flushed, both in order, flushes never + // overlap, and a flush can be done by a thread other than the one that wrote the batch + const uint64_t numBatches = 2000, itemsPerBatch = 5; + std::vector written, flushed; + std::atomic flushingNow(0); + std::atomic overlap(false), flushedBeforeWritten(false); + std::atomic writtenCount(0); + OrderedBatches batches( + [&](uint64_t& item) { written.push_back(item); writtenCount++; }, + [](const uint64_t&) { return 1; }, + [&](uint64_t& item) { + if (flushingNow++ != 0) overlap = true; + if (item >= writtenCount) flushedBeforeWritten = true; + flushed.push_back(item); + if (item % 7 == 0) std::this_thread::yield(); + flushingNow--; + }); + batches.setMaxHeldBytes(20); + + std::mutex handOut; + uint64_t nextSeq = 0; + std::vector threads; + for (int t = 0; t < 8; t++) { + threads.emplace_back([&, t]() { + std::mt19937 rng(t); + while (true) { + batches.waitForRoom(); + uint64_t seq; + { + std::lock_guard lock(handOut); + if (nextSeq == numBatches) return; + seq = nextSeq++; + } + batches.begin(seq); + for (uint64_t i = 0; i < itemsPerBatch; i++) { + if (rng() % 4 == 0) std::this_thread::yield(); + batches.add(seq * itemsPerBatch + i); + } + batches.end(); + } + }); + } + for (auto& thread : threads) thread.join(); + + mu_check(batches.allWritten()); + mu_check(!overlap); + mu_check(!flushedBeforeWritten); + mu_check(written.size() == numBatches * itemsPerBatch); + mu_check(flushed.size() == numBatches * itemsPerBatch); + bool inOrder = true; + for (uint64_t i = 0; i < flushed.size(); i++) if (flushed[i] != i || written[i] != i) inOrder = false; + mu_check(inOrder); +} + MU_TEST_SUITE(test_suite_ordered_batches) { MU_RUN_TEST(test_ordered_batches_single_thread); MU_RUN_TEST(test_ordered_batches_threads); + MU_RUN_TEST(test_ordered_batches_flush); } int main() { From d019a904e60a626b802ff7bd3e2f08db9802671a Mon Sep 17 00:00:00 2001 From: dktue <2878623+dktue@users.noreply.github.com> Date: Sun, 8 Nov 2026 15:29:20 +0100 Subject: [PATCH 06/10] perf: let the tile threads compute the shared pieces The pieces used to be computed in a pass of their own before the tile phase, which ended with one thread on the largest piece (~6 s on the planet) while all others waited. Now the tile threads compute them before their first tile, largest polygon first, and go on to tiles once no piece is left to start; a tile that needs a piece still being computed waits for just that one (shared_future). A piece is known before the tiles start, so the map doesn't change and tiles look pieces up without a lock as before. Output byte-identical (Germany at 32 and 8 threads). Co-Authored-By: Claude Opus 5.5 (1M context) --- include/tile_data.h | 13 +++++++- src/tile_data.cpp | 79 +++++++++++++++++++++++---------------------- src/tilemaker.cpp | 2 ++ 3 files changed, 54 insertions(+), 40 deletions(-) diff --git a/include/tile_data.h b/include/tile_data.h index b7ceaf71..8d3f3d41 100644 --- a/include/tile_data.h +++ b/include/tile_data.h @@ -12,6 +12,8 @@ #include #include #include +#include +#include #include #include #include "append_vector.h" @@ -428,6 +430,8 @@ class TileDataSource { static constexpr unsigned SHARED_PIECE_ZOOM = 8; // present(x, y): whether any tile below SHARED_PIECE_ZOOM tile x/y will be written void precomputeSharedPieces(const std::function &present, unsigned threadNum); + // Compute pieces until none is left to start; the tile threads call this before their first tile + void computeSharedPieces(); // Drop the shared pieces inside a z6 tile, once no tile in it is still to be written void releaseSharedPieces(TileCoordinate z6x, TileCoordinate z6y); // Whether objectID's polygon is kept in the multipolygon store (retrieveMultiPolygon) @@ -439,7 +443,14 @@ class TileDataSource { // Filled by precomputeSharedPieces, then only read while the tiles are written (no lock). // releaseSharedPieces resets the pieces of a z6 tile no thread reads any more. std::unordered_set pieceObjects; - std::map, std::shared_ptr> sharedPieces; + std::map, std::shared_future>> sharedPieces; + struct PieceJob { + std::tuple key; + uint64_t points; + std::shared_ptr>> promise; + }; + std::vector pieceJobs; // largest polygon first + std::atomic nextPieceJob { 0 }; virtual LatpLon buildNodeGeometry(NodeID const objectID, const TileBbox &bbox) const; void open() { diff --git a/src/tile_data.cpp b/src/tile_data.cpp index cdee86a3..1f100c54 100644 --- a/src/tile_data.cpp +++ b/src/tile_data.cpp @@ -100,10 +100,11 @@ thread_local bool clipResultValid = false; // whether the last POLYGON_ clip is // written, and tiles below clip from that piece. A piece only depends on the polygon and the tile, // never on the thread or the order, so the output stays the same for any thread count. // -// The pieces are computed up front, all threads at once and largest polygon first: computed on -// demand, a slow piece (up to ~15 s on the planet) held up every thread whose tiles needed it, -// a third of the tile phase on a 96-thread planet run. After that they are only read, so the -// tile threads look them up without a lock. +// The pieces are known up front and computed by all threads, largest polygon first, before they +// start on tiles: computed on demand, a slow piece (up to ~15 s on the planet) held up every +// thread whose tiles needed it, a third of the tile phase on a 96-thread planet run. A thread +// that runs out of pieces to start goes on to tiles; only a tile that needs a piece still being +// computed waits for it. The map itself doesn't change, so the tiles look pieces up without a lock. #define SHARED_PIECE_MIN_POINTS 10000 void TileDataSource::precomputeSharedPieces(const std::function &present, unsigned threadNum) { @@ -125,7 +126,7 @@ void TileDataSource::precomputeSharedPieces(const std::function> candidates(boxes.begin(), boxes.end()); // The pieces each candidate needs: tiles below which a tile is written, that its envelope reaches - struct Job { std::tuple key; uint64_t points; }; + using Job = PieceJob; std::vector jobs; std::mutex jobsMutex; std::atomic next(0); @@ -148,7 +149,7 @@ void TileDataSource::precomputeSharedPieces(const std::function lock(jobsMutex); jobs.insert(jobs.end(), local.begin(), local.end()); @@ -159,40 +160,37 @@ void TileDataSource::precomputeSharedPieces(const std::function b.points : a.key < b.key; }); - for (const Job &job : jobs) { + for (Job &job : jobs) { pieceObjects.insert(std::get<0>(job.key)); - sharedPieces.emplace(job.key, nullptr); + job.promise = std::make_shared>>(); + sharedPieces.emplace(job.key, job.promise->get_future().share()); } + pieceJobs = std::move(jobs); + nextPieceJob = 0; +} - // Compute them, largest polygon first. Each job writes its own map entry; the map itself - // doesn't change any more. - next = 0; - auto compute = [&]() { - for (size_t i = next++; i < jobs.size(); i = next++) { - const auto &key = jobs[i].key; - auto piece = std::make_shared(); - try { - clipCacheBypass = true; - clipNeededRepair = false; - Geometry g = buildWayGeometry(POLYGON_, std::get<0>(key), TileBbox(TileCoordinates(std::get<1>(key), std::get<2>(key)), zoom, false, false)); - // A piece that needed the repair path can be wrong (once a lake came out empty), and - // every tile below it would inherit that: leave those tiles to clip the whole polygon. - if (!clipNeededRepair) { - piece->geom = std::make_shared(std::move(boost::get(g))); - // buildWayGeometry has just checked it (for a huge piece, is_valid takes seconds) - piece->valid = clipResultValid; - } - } catch (std::exception &) { - // The tiles clip the whole polygon, and report the problem there +void TileDataSource::computeSharedPieces() { + const unsigned zoom = SHARED_PIECE_ZOOM; + for (size_t i = nextPieceJob++; i < pieceJobs.size(); i = nextPieceJob++) { + const auto &key = pieceJobs[i].key; + auto piece = std::make_shared(); + try { + clipCacheBypass = true; + clipNeededRepair = false; + Geometry g = buildWayGeometry(POLYGON_, std::get<0>(key), TileBbox(TileCoordinates(std::get<1>(key), std::get<2>(key)), zoom, false, false)); + // A piece that needed the repair path can be wrong (once a lake came out empty), and + // every tile below it would inherit that: leave those tiles to clip the whole polygon. + if (!clipNeededRepair) { + piece->geom = std::make_shared(std::move(boost::get(g))); + // buildWayGeometry has just checked it (for a huge piece, is_valid takes seconds) + piece->valid = clipResultValid; } - clipCacheBypass = false; - sharedPieces.find(key)->second = piece; + } catch (std::exception &) { + // The tiles clip the whole polygon, and report the problem there } - }; - { - boost::asio::thread_pool pool(threadNum); - for (unsigned t = 0; t < threadNum; t++) boost::asio::post(pool, compute); - pool.join(); + clipCacheBypass = false; + pieceJobs[i].promise->set_value(piece); + pieceJobs[i].promise.reset(); // the map's future keeps the piece, until its z6 tile is released } } @@ -200,7 +198,7 @@ void TileDataSource::releaseSharedPieces(TileCoordinate z6x, TileCoordinate z6y) const unsigned shift = SHARED_PIECE_ZOOM - CLUSTER_ZOOM; for (auto &e : sharedPieces) if ((std::get<1>(e.first) >> shift) == z6x && (std::get<2>(e.first) >> shift) == z6y) - e.second.reset(); + e.second = std::shared_future>(); } void TileDataSource::finalize(size_t threadNum) { @@ -449,9 +447,12 @@ Geometry TileDataSource::buildWayGeometry(OutputGeometryType const geomType, // A large polygon: clip from its shared piece const unsigned shift = bbox.zoom - SHARED_PIECE_ZOOM; auto it = sharedPieces.find(std::make_tuple(objectID, bbox.index.x >> shift, bbox.index.y >> shift)); - if (it != sharedPieces.end() && it->second && it->second->geom) { - cachedClip = it->second->geom; - inputKnownValid = it->second->valid; + if (it != sharedPieces.end() && it->second.valid()) { + const std::shared_ptr &piece = it->second.get(); // waits if still being computed + if (piece && piece->geom) { + cachedClip = piece->geom; + inputKnownValid = piece->valid; + } } } diff --git a/src/tilemaker.cpp b/src/tilemaker.cpp index a2643cac..f118840d 100644 --- a/src/tilemaker.cpp +++ b/src/tilemaker.cpp @@ -598,6 +598,8 @@ int main(const int argc, const char* argv[]) { boost::asio::post(pool, [&]() { std::size_t startIndex, batchSize; uint64_t seq; + // Shared pieces first (see TileDataSource::precomputeSharedPieces) + for (auto source : sources) source->computeSharedPieces(); while (true) { if (toPMTiles) sharedData.pmtiles.batches.waitForRoom(); if (toMBTiles) sharedData.mbtiles.batches.waitForRoom(); From 697a6ff30c4ff97c91fefa3593361f35112b880c Mon Sep 17 00:00:00 2001 From: dktue <2878623+dktue@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:38:03 +0100 Subject: [PATCH 07/10] build: fix Windows and macOS builds of the combined base - polylabel.h: rename edgeDistBound's parameter `near` to `nearCell`. tile_data.h includes boost/asio since #972, which pulls in windows.h on Windows, and windows.h defines `near` as an empty macro. - test/shp_mem_tiles.test.cpp (#979): include shp_mem_tiles.h before minunit.h. minunit.h sets _POSIX_C_SOURCE, which on macOS hides ip_mreq, and boost/asio (through tile_data.h) needs it. Renames and include order only; output unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) --- include/polylabel.h | 10 +++++----- test/shp_mem_tiles.test.cpp | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/polylabel.h b/include/polylabel.h index 6ce44f16..30de7a6d 100644 --- a/include/polylabel.h +++ b/include/polylabel.h @@ -329,13 +329,13 @@ Cell getCentroidCell(const Polygon& polygon) { return Cell(area == 0 ? ring.at(0) : Point { ox + cx / area, oy + cy / area }, 0, polygon); } -// Upper bound on pointToPolygonDist(p, polygon), from the edge nearest to `near.c`. The full scan +// Upper bound on pointToPolygonDist(p, polygon), from the edge nearest to `nearCell.c`. The full scan // computes min over all edges of getSegDistSq(p, a, b) with the same arguments, so its result is // <= this edge's value; the margin covers FP contraction differences between call sites. -inline double edgeDistBound(const Point& p, const Cell& near, const Polygon& polygon) { - const auto& ring = near.kr < 0 ? polygon.outer() : polygon.inners()[near.kr]; - const Point& a = ring[near.ki]; - const Point& b = ring[near.ki == 0 ? ring.size() - 1 : near.ki - 1]; +inline double edgeDistBound(const Point& p, const Cell& nearCell, const Polygon& polygon) { + const auto& ring = nearCell.kr < 0 ? polygon.outer() : polygon.inners()[nearCell.kr]; + const Point& a = ring[nearCell.ki]; + const Point& b = ring[nearCell.ki == 0 ? ring.size() - 1 : nearCell.ki - 1]; return std::sqrt(getSegDistSq(p, a, b)) * (1 + 1e-12) + 1e-11; } diff --git a/test/shp_mem_tiles.test.cpp b/test/shp_mem_tiles.test.cpp index a0cbcb2f..b4629162 100644 --- a/test/shp_mem_tiles.test.cpp +++ b/test/shp_mem_tiles.test.cpp @@ -1,6 +1,6 @@ #include -#include "external/minunit.h" #include "shp_mem_tiles.h" +#include "external/minunit.h" bool verbose = false; From 4c0023f510cdd7da666bc9b8d2b19601ea5edb4e Mon Sep 17 00:00:00 2001 From: dktue <2878623+dktue@users.noreply.github.com> Date: Sun, 15 Nov 2026 16:21:20 +0100 Subject: [PATCH 08/10] fix: keep the allocator's region lists alive until thread-local destructors have run (macOS) On macOS the main thread's thread-local destructors run after the static destructors. Two of them touched objects that were already gone, and the unit tests aborted at exit with "mutex lock failed: Invalid argument": - mmap_allocator.cpp: thread_regions_t's destructor locks mmap_allocator_mutex and appends to free_mmap_files / free_mmap_shm_regions. These three are now allocated once and never destroyed. ~mmap_dir_t clears free_mmap_files first, so --store files are still released and deleted as before (that list used to be destroyed just before mmap_dir). - LeasedStore (tile_data.cpp): a thread gives its store leases back to the TileDataSource when it ends; the main thread does that at exit, after the sources (locals of main(), or of a unit test such as test_shp_mem_tiles) are destroyed, locking a destroyed storeMutex. The sources are now registered while alive (an immortal registry), and a lease goes back only to a live source. Sources are also numbered: a new source at the address of a destroyed one no longer gets that one's lease. No change in output. Co-Authored-By: Claude Opus 5.5 (1M context) --- include/leased_store.h | 16 ++++++++++------ include/tile_data.h | 10 ++++++++++ src/mmap_allocator.cpp | 14 +++++++++++--- src/tile_data.cpp | 29 +++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 9 deletions(-) diff --git a/include/leased_store.h b/include/leased_store.h index 0349d64d..019b110d 100644 --- a/include/leased_store.h +++ b/include/leased_store.h @@ -2,6 +2,7 @@ #define _LEASED_STORE_H #include "tile_data.h" +#include // LeasedStore implements "leases" for generated geometries. // @@ -45,23 +46,26 @@ inline std::vector>& g template class LeasedStore { - std::vector>> leases; + std::vector>> leases; // source, its leaseNumber, lease public: ~LeasedStore() { + // Only to the sources still alive (see registerLeaseSource) + std::lock_guard live(leaseSourcesMutex()); for (const auto& lease : leases) { - auto source = lease.first; + auto source = std::get<0>(lease); + if (!leaseSourceAlive(source, std::get<1>(lease))) continue; std::lock_guard lock(source->storeMutex); std::vector>& availableLeases = getAvailableLeases(source); - availableLeases.push_back(lease.second); + availableLeases.push_back(std::get<2>(lease)); } } std::pair get(TileDataSource* source) { for (const auto& lease : leases) { - if (lease.first == source) - return lease.second; + if (std::get<0>(lease) == source && std::get<1>(lease) == source->leaseNumber) + return std::get<2>(lease); } std::lock_guard lock(source->storeMutex); @@ -74,7 +78,7 @@ class LeasedStore { std::pair entry = availableLeases.back(); availableLeases.pop_back(); - leases.push_back(std::make_pair(source, entry)); + leases.push_back(std::make_tuple(source, source->leaseNumber, entry)); return entry; } diff --git a/include/tile_data.h b/include/tile_data.h index 8d3f3d41..0adf6d95 100644 --- a/include/tile_data.h +++ b/include/tile_data.h @@ -24,6 +24,14 @@ typedef std::vector SourceList; +// The TileDataSources alive, for the store leases a thread keeps until it ends (leased_store.h): the +// main thread gives its leases back at exit, after the sources are gone. A source is also told apart +// by its number, as a new source can be at the address of a destroyed one. +uint64_t registerLeaseSource(const TileDataSource *source); +void unregisterLeaseSource(const TileDataSource *source); +std::mutex &leaseSourcesMutex(); +bool leaseSourceAlive(const TileDataSource *source, uint64_t number); // with leaseSourcesMutex() held + class TileBbox; template void sortOutputObjects( @@ -375,6 +383,8 @@ class TileDataSource { public: TileDataSource(size_t threadNum, unsigned int indexZoom, bool includeID); + ~TileDataSource(); + const uint64_t leaseNumber = registerLeaseSource(this); // see registerLeaseSource void collectTilesWithObjectsAtZoom(std::vector>& zooms); // The same for the objects of one z6 tile (0 <= shard < CLUSTER_ZOOM_AREA) diff --git a/src/mmap_allocator.cpp b/src/mmap_allocator.cpp index 4daf0c00..33ced307 100644 --- a/src/mmap_allocator.cpp +++ b/src/mmap_allocator.cpp @@ -98,13 +98,15 @@ static mmap_dir_t mmap_dir; std::vector< mmap_shm_ptr > mmap_shm_regions; size_t mmap_shm::mmap_file_size = 0; -std::mutex mmap_allocator_mutex; +// Never destroyed (nor the two lists below): thread_regions_t's destructor uses them, and the +// main thread's thread-locals are destroyed after the statics on some platforms (macOS) +std::mutex &mmap_allocator_mutex = *new std::mutex; // Regions of threads that have exited, for reuse by new threads. Every thread // pool (one per .shp/GeoJSON layer, per .pbf read phase) starts new threads, and // a new region (or 1GB store file) for each of them would never be released. -std::vector free_mmap_files; -std::vector free_mmap_shm_regions; +std::vector &free_mmap_files = *new std::vector(); +std::vector &free_mmap_shm_regions = *new std::vector(); // The regions the current thread allocates from struct thread_regions_t @@ -195,6 +197,12 @@ bool mmap_dir_t::is_open() { if(!mmap_dir_filename.empty()) { try { + // The files of exited threads, kept for reuse: let them go first, so that they are + // deleted below (free_mmap_files is never destroyed) + { + std::lock_guard lock(mmap_allocator_mutex); + free_mmap_files.clear(); + } // Unmapping and deleting a big store (planet: ~140 GB in ~300 files) // takes half a minute on one thread, so release the files from several. const size_t threads = std::min(files.size(), std::min(16u, std::max(1u, std::thread::hardware_concurrency()))); diff --git a/src/tile_data.cpp b/src/tile_data.cpp index 1f100c54..866b853a 100644 --- a/src/tile_data.cpp +++ b/src/tile_data.cpp @@ -7,6 +7,7 @@ #include "tile_data.h" #include "coordinates_geom.h" #include "leased_store.h" +#include #include "stable_hash.h" #include @@ -48,6 +49,34 @@ thread_local LeasedStore linestringStore; thread_local LeasedStore multilinestringStore; thread_local LeasedStore multipolygonStore; +// Never destroyed: the thread-locals that use them may be destroyed after the statics (macOS) +static std::unordered_map &leaseSources() { + static std::unordered_map &sources = *new std::unordered_map(); + return sources; +} +std::mutex &leaseSourcesMutex() { + static std::mutex &m = *new std::mutex; + return m; +} +uint64_t registerLeaseSource(const TileDataSource *source) { + static uint64_t last = 0; + std::lock_guard lock(leaseSourcesMutex()); + const uint64_t number = ++last; + leaseSources()[source] = number; + return number; +} +void unregisterLeaseSource(const TileDataSource *source) { + std::lock_guard lock(leaseSourcesMutex()); + leaseSources().erase(source); +} +bool leaseSourceAlive(const TileDataSource *source, uint64_t number) { + const auto it = leaseSources().find(source); + return it != leaseSources().end() && it->second == number; +} +TileDataSource::~TileDataSource() { + unregisterLeaseSource(this); +} + TileDataSource::TileDataSource(size_t threadNum, unsigned int indexZoom, bool includeID) : includeID(includeID), From 3f55964ea3d05ea52067f41469335c1914ee5d1d Mon Sep 17 00:00:00 2001 From: dktue <2878623+dktue@users.noreply.github.com> Date: Sun, 15 Nov 2026 16:21:21 +0100 Subject: [PATCH 09/10] perf: finalize the stores and close PMTiles on all cores (#989 + #990) #989 and #990 adapted to perf/shared-pieces, as the base of this series. - SortedNodeStore/SortedWayStore::finalize publish the leftover groups (groups split between worker batches) from all threads instead of one after the other. On the planet that was ~10 s after the ways pass with every other thread idle. Same store content. (#989, unchanged) - PMTiles::close collects the directory entries and cuts them into leaves exactly as before, then serializes and compresses the leaves on all cores and writes them in order: ~9 s -> 3 s on the planet, byte-identical file. (#990; on perf/shared-pieces the two-step writer already holds the file, so the leaf writes need no lock and fileMutex goes.) Planet (in memory, 96 threads): -9 s and -6 s. Output byte-identical to perf/shared-pieces. Co-Authored-By: Claude Opus 5.5 (1M context) --- include/pmtiles.h | 3 -- src/pmtiles.cpp | 79 +++++++++++++++++++-------------------- src/sorted_node_store.cpp | 42 ++++++++++++++------- src/sorted_way_store.cpp | 42 ++++++++++++++------- 4 files changed, 94 insertions(+), 72 deletions(-) diff --git a/include/pmtiles.h b/include/pmtiles.h index fb2e6682..38095cdc 100644 --- a/include/pmtiles.h +++ b/include/pmtiles.h @@ -62,7 +62,6 @@ class PMTiles { std::ofstream outputStream; std::vector outputBuffer; // a large stream buffer: one thread writes all tiles uint64_t tileDataEnd = 0; // bytes of tile data written so far (tellp() costs a flush and a seek per tile) - std::mutex fileMutex; // guards leaf directory writes in close() uint64_t leafStart = 0; uint64_t numTilesWritten = 0; uint64_t numTilesAddressed = 0; @@ -74,8 +73,6 @@ class PMTiles { void writeTile(PendingTile &tile); // with batches.writeMutex() held: places the tile void flushTile(PendingTile &tile); // in the same order, by one thread at a time: writes its bytes void appendWithRLE(std::vector &entries, pmtiles::entryv3 &entry); - void appendTileEntry(uint64_t tileId, TileOffset offset, std::vector &rootEntries, std::vector &entries); - void flushEntries(std::vector &rootEntries, std::vector &entries); }; #endif //_PMTILES_H diff --git a/src/pmtiles.cpp b/src/pmtiles.cpp index dceb5af4..11b3e500 100644 --- a/src/pmtiles.cpp +++ b/src/pmtiles.cpp @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include "pmtiles.h" #include "helpers.h" @@ -34,24 +36,49 @@ void PMTiles::close(std::string &metadata) { std::cout << "\nClosing pmtiles file" << std::flush; if (!batches.allWritten()) { throw std::runtime_error("PMTiles: a batch of tiles was never written - please file an issue"); } - // add all tiles to directories, writing leaf directories as we go + // Add all tiles to directories: low zooms to the root, the rest to leaf directories of up to + // LEAF_DIRECTORY_SIZE entries. The leaves are serialized and compressed on all cores (one after + // the other that took ~9 s for the planet), then written in order, so the file is the same. std::vector rootEntries; - std::vector entries; - leafStart = outputStream.tellp(); - if (isSparse) { - for (auto it : sparseIndex) { - appendTileEntry(it.first, it.second, rootEntries, entries); + std::vector> leaves(1); + auto append = [&](uint64_t tileId, TileOffset offset) { + pmtiles::entryv3 entry = pmtiles::entryv3(tileId, offset.offset, offset.length, 1); // 1=RLE + if (tileId=LEAF_DIRECTORY_SIZE) leaves.emplace_back(); } + }; + if (isSparse) { + for (auto it : sparseIndex) append(it.first, it.second); } else { for (size_t tileId=0; tileId compressedLeaves(leaves.size()); + std::atomic next(0); + auto compressLeaves = [&]() { + for (size_t i = next++; i < leaves.size(); i = next++) + compressedLeaves[i] = compress_string(pmtiles::serialize_directory(leaves[i]), Z_DEFAULT_COMPRESSION, true); + }; + std::vector threads; + const size_t numThreads = std::min(leaves.size(), std::max(1u, std::thread::hardware_concurrency())); + for (size_t t = 0; t < numThreads; t++) threads.emplace_back(compressLeaves); + for (auto &t : threads) t.join(); + for (size_t i = 0; i < leaves.size(); i++) { + // write the leaf directory, and add a reference to it to the root directory + const uint64_t location = outputStream.tellp(); + outputStream.write(compressedLeaves[i].c_str(), compressedLeaves[i].size()); + rootEntries.emplace_back(pmtiles::entryv3(leaves[i][0].tile_id, location-leafStart, compressedLeaves[i].size(), 0)); + } } uint64_t leafLength = static_cast(outputStream.tellp()) - leafStart; @@ -96,18 +123,7 @@ void PMTiles::close(std::string &metadata) { } // Add an entry either to the current leaf directory, or (for lowzoom) the root directory -void PMTiles::appendTileEntry(uint64_t tileId, TileOffset offset, std::vector &rootEntries, std::vector &entries) { - pmtiles::entryv3 entry = pmtiles::entryv3(tileId, offset.offset, offset.length, 1); // 1=RLE - if (tileId=LEAF_DIRECTORY_SIZE) flushEntries(rootEntries, entries); - } -} - -// Handle run-length encoding for appendTileEntry +// Handle run-length encoding for the directory entries void PMTiles::appendWithRLE(std::vector &entries, pmtiles::entryv3 &entry) { if (entries.empty() || entries.back().offset != entry.offset || @@ -119,25 +135,6 @@ void PMTiles::appendWithRLE(std::vector &entries, pmtiles::ent entries.back().run_length++; } -// Write a leaf directory to file, and add a reference to it to the root directory -void PMTiles::flushEntries(std::vector &rootEntries, std::vector &entries) { - if (entries.size()==0) return; - uint64_t startId = entries[0].tile_id; - std::string directory = pmtiles::serialize_directory(entries); - std::string compressed = compress_string(directory, Z_DEFAULT_COMPRESSION, true); - entries.clear(); - - // write the leaf directory to disk - std::lock_guard lock(fileMutex); - uint64_t location = outputStream.tellp(); - uint64_t length = compressed.size(); - outputStream.write(compressed.c_str(), compressed.size()); - - // append reference to the root directory - pmtiles::entryv3 rootEntry = pmtiles::entryv3(startId, location-leafStart, length, 0); - rootEntries.emplace_back(rootEntry); -} - // Save a tile: compress it here (compress_string is expensive), but leave tiny tiles // raw until they're written, since most of them (e.g. sea) repeat an earlier one void PMTiles::saveTile(int zoom, int x, int y, std::string &data) { diff --git a/src/sorted_node_store.cpp b/src/sorted_node_store.cpp index c6d282e8..e9663a02 100644 --- a/src/sorted_node_store.cpp +++ b/src/sorted_node_store.cpp @@ -1,5 +1,8 @@ #include #include +#include +#include +#include #include #include #include @@ -530,20 +533,31 @@ void SortedNodeStore::finalize(size_t threadNum) { } workerBuffers.clear(); - // Empty the orphanage into the index. - std::vector copy; - for (const auto& entry: orphanage) { - for (const auto& orphan: entry.second) - copy.push_back(orphan); - - // Orphans may come from different workers, and thus be unsorted. - std::sort( - copy.begin(), - copy.end(), - [](auto const &a, auto const &b) { return a.first < b.first; } - ); - publishGroup(copy); - copy.clear(); + // Empty the orphanage into the index. The groups are independent and publishGroup is what the + // workers call during the pass anyway, so publish them from all threads: one after the other, + // the planet's leftover groups took ~10 s with every other thread idle. + std::vector*> orphanGroups; + for (const auto& entry: orphanage) + orphanGroups.push_back(&entry.second); + std::atomic nextGroup(0); + auto publishOrphans = [&]() { + std::vector copy; + for (size_t i = nextGroup++; i < orphanGroups.size(); i = nextGroup++) { + copy.assign(orphanGroups[i]->begin(), orphanGroups[i]->end()); + // Orphans may come from different workers, and thus be unsorted. + std::sort( + copy.begin(), + copy.end(), + [](auto const &a, auto const &b) { return a.first < b.first; } + ); + publishGroup(copy); + } + }; + { + boost::asio::thread_pool pool(std::max(1, threadNum)); + for (unsigned int t = 0; t < std::max(1, threadNum); t++) + boost::asio::post(pool, publishOrphans); + pool.join(); } orphanage.clear(); diff --git a/src/sorted_way_store.cpp b/src/sorted_way_store.cpp index 02f66147..81cd9132 100644 --- a/src/sorted_way_store.cpp +++ b/src/sorted_way_store.cpp @@ -1,4 +1,7 @@ #include +#include +#include +#include #include #include #include @@ -278,20 +281,31 @@ void SortedWayStore::finalize(unsigned int threadNum) { } workerBuffers.clear(); - // Empty the orphanage into the index. - std::vector>> copy; - for (const auto& entry: orphanage) { - for (const auto& orphan: entry.second) - copy.push_back(orphan); - - // Orphans may come from different workers, and thus be unsorted. - std::sort( - copy.begin(), - copy.end(), - [](auto const &a, auto const &b) { return a.first < b.first; } - ); - publishGroup(copy); - copy.clear(); + // Empty the orphanage into the index. The groups are independent and publishGroup is what the + // workers call during the pass anyway, so publish them from all threads: one after the other, + // the planet's leftover groups took ~10 s with every other thread idle. + std::vector>>*> orphanGroups; + for (const auto& entry: orphanage) + orphanGroups.push_back(&entry.second); + std::atomic nextGroup(0); + auto publishOrphans = [&]() { + std::vector>> copy; + for (size_t i = nextGroup++; i < orphanGroups.size(); i = nextGroup++) { + copy.assign(orphanGroups[i]->begin(), orphanGroups[i]->end()); + // Orphans may come from different workers, and thus be unsorted. + std::sort( + copy.begin(), + copy.end(), + [](auto const &a, auto const &b) { return a.first < b.first; } + ); + publishGroup(copy); + } + }; + { + boost::asio::thread_pool pool(std::max(1, threadNum)); + for (unsigned int t = 0; t < std::max(1, threadNum); t++) + boost::asio::post(pool, publishOrphans); + pool.join(); } orphanage.clear(); From 463c11b26263342903c4c4c0b7f5420cc341b9d6 Mon Sep 17 00:00:00 2001 From: dktue <2878623+dktue@users.noreply.github.com> Date: Sun, 15 Nov 2026 16:21:22 +0100 Subject: [PATCH 10/10] perf(lua): replay way_function/node_function from a memo; native MercatorArea Most ways run exactly the same Lua as many ways before them: the same tag reads give the same results and the same calls. With `way_memo = true` in the profile, tilemaker records, per thread, a tree of what way_function read (Find, Holds, IsClosed, NextRelation, ...) and which tilemaker functions it then called (Layer, Attribute..., MinZoom), and replays the calls for a way whose reads give the same results, without entering Lua. Ways with anything else (geometry queries, Id, relation tags, globals) run through Lua as before. - Opt-in declarations, documented in CONFIGURATION.md: way_memo_values (tags only tested for being empty and copied, e.g. name; replayed with the way's own value, optionally its length), way_memo_area (MercatorArea only compared with given numbers), way_memo_relations (NextRelation only tested, relation tags only via FindInRelation), and node_memo* for node_function. Multipolygon relations get a record of their own. - The walk reads the way's tags once into small key slots (memo_keys.h); TagMap builds its hash only when Lua asks for a tag. - Bounded: 2^18 nodes per thread, at most 256 children per node, full trees are compacted; a value-heavy branch (a name, a house number) falls back to Lua. - New Lua functions MercatorArea() (planar Web Mercator area) and MinZoomByArea(limit, thresholds): the minimum-zoom-by-area rule worked out in C++, so that way_function stays replayable. - The bundled OpenMapTiles profile uses all of this, and keeps tilemaker's functions in locals. - The Makefile compiles as C++17, like CMake since #953 (memo_keys.h uses std::string_view). Works with Lua 5.1/LuaJIT too (lua_objlen where luaL_len is missing). Output: the only change in this series. The old profile computed MercatorArea in Lua as Area() / cos^2(latitude of the centroid); the native function is the planar area on the Web Mercator plane. Areas close to a minimum-zoom threshold can land on the other side of it, so some landuse, water and building polygons start one zoom earlier or later. Everything else is identical; the memo itself does not change the output. Planet (96 threads): way_function replayed for ~1.17 billion of 1.22 billion ways, node_function for ~211 million nodes; together with the native area about -21% wall. Co-Authored-By: Claude Opus 5.5 (1M context) --- Makefile | 2 +- docs/CONFIGURATION.md | 15 + include/attribute_store.h | 8 + include/memo_keys.h | 273 ++++++++++ include/osm_lua_processing.h | 243 +++++++++ include/tag_map.h | 32 +- resources/process-openmaptiles.lua | 74 +-- src/attribute_store.cpp | 34 +- src/osm_lua_processing.cpp | 808 ++++++++++++++++++++++++++--- src/tag_map.cpp | 25 +- src/tilemaker.cpp | 2 + 11 files changed, 1403 insertions(+), 113 deletions(-) create mode 100644 include/memo_keys.h diff --git a/Makefile b/Makefile index b03576c2..64bcaa0c 100644 --- a/Makefile +++ b/Makefile @@ -84,7 +84,7 @@ else ifeq ($(TM_GIT_TAG)$(TM_GIT_DIRTY),$(TM_BASE_VERSION)) else TM_VERSION ?= $(TM_BASE_VERSION)+g$(TM_GIT_SHA)$(TM_GIT_DIRTY) endif -CXXFLAGS ?= -O3 -Wall -Wno-unknown-pragmas -Wno-sign-compare -std=c++14 -pthread -fPIE -DTM_VERSION=$(TM_VERSION) $(CONFIG) +CXXFLAGS ?= -O3 -Wall -Wno-unknown-pragmas -Wno-sign-compare -std=c++17 -pthread -fPIE -DTM_VERSION=$(TM_VERSION) $(CONFIG) CFLAGS ?= -O3 -Wall -Wno-unknown-pragmas -Wno-sign-compare -std=c99 -fPIE -DTM_VERSION=$(TM_VERSION) $(CONFIG) DEPFLAGS := -MD -MP DEPS := $(wildcard src/*.d src/external/*.d src/external/libdeflate/lib/*.d src/external/libdeflate/lib/*/*.d server/*.d test/*.d) diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index f7d94183..11230ded 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -210,6 +210,8 @@ To do that, you use these methods: * `MinZoom(zoom)`: set the minimum zoom level (0-15) at which this object will be written. Note that the JSON layer configuration minimum still applies (so `:MinZoom(5)` will have no effect if your layer only starts at z6). * `Score(number)`: set how important this point feature is (a whole number; anything else is rounded down), so that tilemaker can decide which zoom level to show it from. Only meaningful for points (i.e. features written with `Layer` for a node, or `LayerAsCentroid`) in a layer with `declutter_below` set - see 'Decluttering point features' below. * `Length()` and `Area()`: return the length (metres)/area (square metres) of the current object. Requires Boost 1.67+. +* `MercatorArea()`: return the area of the current object on the Web Mercator plane, in square metres at the scale of the equator (so it grows towards the poles, as the map does). It is cheap to compute and tells how much of a tile the object covers, which makes it the right measure for minimum zoom rules. +* `MinZoomByArea(limit, thresholds)`: set the minimum zoom of the current object from its `MercatorArea()`: the first zoom z from `limit` on whose threshold the area exceeds, where `thresholds` lists the thresholds for zoom 1, 2, 3...; if there is none, one more than the number of thresholds. Unlike the same rule written in Lua with `MercatorArea()`, this keeps `way_function` replayable (see `way_memo` above). * `Centroid()`: return the lat/lon of the centre of the current object as a two-element Lua table (element 1 is lat, 2 is lon). The simplest possible function, to include roads/paths and nothing else, might look like this: @@ -238,6 +240,19 @@ If your Lua file causes an error due to mistaken syntax, you can test it at the `way_keys` is similar, but for ways. For ways, you may also wish to express the filter in terms of the tag value, or as an inversion. For example, to exclude buildings: `way_keys = {"~building"}`. To build a map only of major roads: `way_keys = {"highway=motorway", "highway=trunk", "highway=primary", "highway=secondary"}` +`way_memo = true` lets tilemaker replay `way_function` instead of running it, for ways that read the same as an earlier way. This is only allowed if `way_function` is a function of what it reads with `Find`, `Holds`, `IsClosed` and `NextRelation` (no other state, such as global counters), and what it does is call tilemaker's functions (`Layer`, `Attribute`... `MinZoom`): tilemaker records which reads lead to which calls, and replays the calls for a way whose reads give the same results. Anything else (geometry queries such as `Area`, relation tags, `Id`...) makes ways with those reads run through Lua. A tag whose value is only tested for being empty and copied unchanged into some attributes can be declared in `way_memo_values`, e.g. `way_memo_values = { name = { "name:latin" } }`: ways are then replayed with their own value of the tag, instead of being told apart by it. An attribute that is set to the length of the value (in bytes, as Lua's `#value` or `value:len()`) is declared as `attribute = "length"`, e.g. `way_memo_values = { ref = { "ref", ref_length = "length" } }`. + +Two more declarations let the memo replay ways that use some of the other functions: + +* `way_memo_area = { 1000, 5000 }`: `way_function` uses `MercatorArea()` only to compare it (`<`, `<=`, `>`, `>=`, `==`, `~=`) with these numbers. The memo then tells ways apart by the results of these comparisons, not by the area (e.g. `if MercatorArea() < 1000 then MinZoom(8) else MinZoomByArea(0, thresholds) end`; `MinZoomByArea` needs no declaration, it is worked out for each way). +* `way_memo_relations = true`: `way_function` uses the relation id `NextRelation()` returns only to test whether there is one (`if not rel then break end`), and reads the relations' tags only with `FindInRelation` (no `RestartRelations`). The memo then tells ways apart by the roles and the relation tags read. + +As with `way_memo_values`, tilemaker can't check these promises: if `way_function` does more with the area or the relations, ways are replayed with the wrong calls. + +Multipolygon relations, which also run `way_function`, are replayed the same way (with a record of their own, apart from that of the ways): the promises above are about `way_function`, so they hold for them too. For a multipolygon, `NextRelation` and `FindInRelation` are about the relations it is a member of. + +`node_memo = true` and `node_memo_values`, `node_memo_area`, `node_memo_relations` do the same for `node_function` (with its own record). + `init_function(name, is_first)` and `exit_function` are called at the start and end of processing (once per thread). You can use this to output statistics or even to read a small amount of external data. `is_first` will be true only the first time `init_function` is called. Other functions are described below and in RELATIONS.md. diff --git a/include/attribute_store.h b/include/attribute_store.h index 849b3889..80215422 100644 --- a/include/attribute_store.h +++ b/include/attribute_store.h @@ -478,6 +478,14 @@ struct AttributeStore { void addAttributeWithKeyIndex(AttributeSet& attributeSet, uint16_t keyIndex, double v, char minzoom); void addAttributeWithKeyIndex(AttributeSet& attributeSet, uint16_t keyIndex, int v, char minzoom); void addAttributeWithKeyIndex(AttributeSet& attributeSet, uint16_t keyIndex, bool v, char minzoom); + // The pair index that addAttributeWithKeyIndex adds to the set. It is the same for the same + // arguments for as long as the store lives, so callers may keep it. + uint32_t pairIndexWithKeyIndex(uint16_t keyIndex, std::string const &key, const protozero::data_view v, char minzoom); + uint32_t pairIndexWithKeyIndex(uint16_t keyIndex, double v, char minzoom); + uint32_t pairIndexWithKeyIndex(uint16_t keyIndex, int v, char minzoom); + uint32_t pairIndexWithKeyIndex(uint16_t keyIndex, bool v, char minzoom); + // Counts an object whose AttributeIndex the caller knew without add() (for reportSize) + void noteLookup(); AttributeStore(): finalized(false), diff --git a/include/memo_keys.h b/include/memo_keys.h new file mode 100644 index 00000000..0506d785 --- /dev/null +++ b/include/memo_keys.h @@ -0,0 +1,273 @@ +/*! \file */ +#ifndef _MEMO_KEYS_H +#define _MEMO_KEYS_H + +// The way_function memo's walk (see OsmLuaProcessing::runWayFunction) without TagMap lookups. +// +// The memo tree reads few distinct keys (tens), but each way is tested against ~50 of them, and +// most are absent. So the keys the tree reads get small ids (MemoKeyTable), the way's tags are +// looked up once each (MemoTagSlots: which ids are present, with their values), and the walk runs +// on a compact copy of the tree (MemoHotNode: kind, key id and the children of the common results). +// The results are those of TagMap::getKey / getValueFromKey: a key matches by its exact bytes and a +// key added twice has its last value. + +#include +#include +#include +#include +#include + +class MemoKeyTable { +public: + static constexpr uint32_t NONE = UINT32_MAX; + + uint32_t find(const char* key, size_t size) const { + if (slots.empty()) return NONE; + const uint64_t p = prefixOf(key, size); + const size_t mask = slots.size() - 1; + for (size_t i = hashOf(p, size) & mask; ; i = (i + 1) & mask) { + const Slot& s = slots[i]; + if (s.id == NONE) return NONE; + if (s.prefix == p && s.size == size && (size <= 8 || memcmp(names[s.id].data() + 8, key + 8, size - 8) == 0)) + return s.id; + } + } + + // The id of a key, given a new one if it hasn't one yet + uint32_t add(std::string_view key) { + uint32_t id = find(key.data(), key.size()); + if (id != NONE) return id; + id = names.size(); + names.emplace_back(key); + if ((names.size() + 1) * 4 > slots.size()) rehash(slots.empty() ? 64 : slots.size() * 2); + else place(id); + return id; + } + + size_t size() const { return names.size(); } + // Per id (kept up to date by the owner, see OsmLuaProcessing::runWayFunction): a key whose + // value the tree only tests for being empty (way_memo_values) + std::vector valueOnly; + const std::string& name(uint32_t id) const { return names[id]; } + +private: + struct Slot { uint64_t prefix; uint32_t size; uint32_t id; }; + std::vector names; + std::vector slots; // open addressing, at most a quarter full + + // The first (up to) 8 bytes of a key as an integer + static uint64_t prefixOf(const char* key, size_t size) { + uint64_t p = 0; + if (size >= 8) memcpy(&p, key, 8); + else if (size >= 4) { + // bytes 0..3 and size-4..size-1 (the overlap has the same bytes in both) + uint32_t lo, hi; + memcpy(&lo, key, 4); memcpy(&hi, key + size - 4, 4); + p = uint64_t(lo) | uint64_t(hi) << (8 * (size - 4)); + } else if (size > 0) { + const auto b = [&](size_t i) { return uint64_t(static_cast(key[i])) << (8 * i); }; + p = b(0) | b(size / 2) | b(size - 1); + } + return p; + } + static size_t hashOf(uint64_t prefix, size_t size) { + const uint64_t h = (prefix ^ (uint64_t(size) << 56)) * 0x9e3779b97f4a7c15ull; + return h >> 40; + } + void place(uint32_t id) { + const std::string& k = names[id]; + const uint64_t p = prefixOf(k.data(), k.size()); + const size_t mask = slots.size() - 1; + size_t i = hashOf(p, k.size()) & mask; + while (slots[i].id != NONE) i = (i + 1) & mask; + slots[i] = Slot{ p, uint32_t(k.size()), id }; + } + void rehash(size_t n) { + slots.assign(n, Slot{ 0, 0, NONE }); + for (uint32_t id = 0; id < names.size(); id++) place(id); + } +}; + +// The current object's tags with a key of the table: present or not, and the value +class MemoTagSlots { +public: + // Look up the tags (as added: a later one replaces an earlier one of the same key), and make + // the object's signature for MemoSigCache: `flags`, then per tag with a key of the table (as + // added) its id and value (valueOnly keys: whether it is empty). Equal signatures mean equal + // tags as the walk sees them. Returns its length, 0 if too long for the cache. + template + size_t fill(const MemoKeyTable& table, const Tags& tags, uint8_t flags, char* sig, size_t sigMax) { + if (stamp.size() < table.size()) { stamp.resize(table.size() + 16, 0); value.resize(table.size() + 16); } + if (++gen == 0) { std::fill(stamp.begin(), stamp.end(), 0); gen = 1; } + present = 1ull << 63; + size_t len = 0; + sig[len++] = flags; + for (const auto& tag : tags) { + const uint32_t id = table.find(tag.key.data(), tag.key.size()); + if (id == MemoKeyTable::NONE) continue; + stamp[id] = gen; + value[id] = std::string_view(tag.value.data(), tag.value.size()); + present |= keyBit(id); + if (len == 0) continue; // too long already + const size_t vlen = tag.value.size(); + const bool valueOnly = table.valueOnly[id]; + // id (1 or 2 bytes), then 1 byte (valueOnly: empty or not), else the length and the bytes + const size_t need = (id < 128 ? 1 : 2) + 1 + (valueOnly ? 0 : vlen); + if (id >= 16384 || vlen >= 256 || len + need > sigMax) { len = 0; continue; } + if (id < 128) sig[len++] = char(id); + else { sig[len++] = char(0x80 | (id >> 8)); sig[len++] = char(id & 0xff); } + if (valueOnly) sig[len++] = char(vlen != 0); + else { sig[len++] = char(vlen); memcpy(sig + len, tag.value.data(), vlen); len += vlen; } + } + return len; + } + // A bit per key id (ids share bits: a clear bit means all its keys are absent); bit 63 always set + static uint64_t keyBit(uint32_t id) { return 1ull << (id % 63); } + uint64_t present = 0; + // (ids the table got after fill(): absent) + bool has(uint32_t id) const { return id < stamp.size() && stamp[id] == gen; } + std::string_view get(uint32_t id) const { return has(id) ? value[id] : std::string_view(); } + +private: + uint32_t gen = 0; + std::vector stamp; // == gen: present + std::vector value; +}; + +// A direct-mapped cache of recent signatures (short byte strings) -> 32-bit values. Used for +// - the walk: the node it reached (a Leaf or Lua node) per object signature (MemoTagSlots::fill). +// The walk is a function of the tags it can read, IsClosed and the relations, and objects in a +// row are often alike (buildings, streets of a town). An edge of the tree never changes once +// added, so a path once complete keeps ending at the same node while nodes are added; +// memoCompact renumbers the nodes (clear()). +// - the AttributeIndex of a replayed segment with way_memo_values attributes, per segment and +// values (see OsmLuaProcessing::memoSegmentSig) +class MemoSigCache { +public: + static constexpr size_t SIG_MAX = 52; + explicit MemoSigCache(size_t size) : entries(size) {} // size: a power of 2 + static uint64_t hashOf(const char* sig, size_t len) { + uint64_t h = len * 0x9e3779b97f4a7c15ull; + size_t i = 0; + for (; i + 8 <= len; i += 8) { uint64_t w; memcpy(&w, sig + i, 8); h = (h ^ w) * 0xff51afd7ed558ccdull; h ^= h >> 29; } + uint64_t w = 0; + for (size_t k = 0; i < len; i++, k++) w |= uint64_t(static_cast(sig[i])) << (8 * k); + h = (h ^ w) * 0xc4ceb9fe1a85ec53ull; h ^= h >> 32; + return h; + } + uint32_t find(uint64_t hash, const char* sig, size_t len) const { + const Entry& e = entries[hash & (entries.size() - 1)]; + return e.len == len && e.hash == hash && memcmp(e.sig, sig, len) == 0 ? e.node : UINT32_MAX; + } + void put(uint64_t hash, const char* sig, size_t len, uint32_t node) { + Entry& e = entries[hash & (entries.size() - 1)]; + e.hash = hash; e.node = node; e.len = len; memcpy(e.sig, sig, len); + } + void clear() { for (Entry& e : entries) e.len = 0; } +private: + struct Entry { uint64_t hash = 0; uint32_t node = 0; uint16_t len = 0; char sig[SIG_MAX]; }; // len 0: empty + std::vector entries; +}; + +#ifndef MEMO_LEVELS +#define MEMO_LEVELS 2 +#endif +#ifndef MEMO_FAN +#define MEMO_FAN 4 +#endif + +// A memo tree node in compact form: what the walk needs, for the results "", "0", "1" and "\x01" +// (the other results: the full node's child()) +// Jumps: from a read node, the walk of a way that has none of the keys along the next (up to) 4 +// resp. 16 reads takes their "absent" children ("" for Find, "0" otherwise); jump[l] is where it +// ends up, mask[l] the bits of those keys (IsClosed/NextRelation: CLOSED_BIT/RELATION_BIT). +// A node without a jump has mask bit 63, which `present` always has. +struct MemoHotNode { + uint8_t kind = 0; // OsmLuaProcessing::MemoNode::Kind + bool valueOnly = false; // a tag of way_memo_values: branches on empty / not empty only + bool overflow = false; // too many results: the others run through Lua + uint16_t children = 0; // number of children (all results) + uint32_t key = MemoKeyTable::NONE; // Find, Holds: the key's id + uint32_t quick[4] = { UINT32_MAX, UINT32_MAX, UINT32_MAX, UINT32_MAX }; // children for "", "0", "1", "\x01" + uint32_t more = UINT32_MAX; // the node's other children / its ops (Leaf): index in the owner's side table + uint32_t jump[MEMO_LEVELS]; + uint64_t mask[MEMO_LEVELS]; + MemoHotNode() { for (int l = 0; l < MEMO_LEVELS; l++) { jump[l] = 0; mask[l] = 1ull << 63; } } +}; +static constexpr uint64_t MEMO_CLOSED_BIT = 1ull << 61, MEMO_RELATION_BIT = 1ull << 62; + +// (Re)compute the jumps of nodes [from, size). A child has a higher index than its parent, so going +// backwards, the jumps of the children are known. Jumps of a growing tree stay right (an edge +// once added never changes), they only get shorter than they could be until recomputed. +inline void memoComputeJumps(MemoHotNode* hot, size_t size, size_t from = 0) { + for (size_t i = size; i-- > from; ) { + MemoHotNode& node = hot[i]; + // level 0: up to MEMO_FAN single steps + uint32_t m = i; uint64_t mask = 0; int steps = 0; + for (; steps < MEMO_FAN; steps++) { + const MemoHotNode& x = hot[m]; + if (x.kind < 1 || x.kind > 4) break; + const uint32_t next = x.quick[x.kind == 1 ? 0 : 1]; + if (next == UINT32_MAX || next >= size) break; + mask |= x.kind == 1 || x.kind == 2 ? MemoTagSlots::keyBit(x.key) : x.kind == 3 ? MEMO_CLOSED_BIT : MEMO_RELATION_BIT; + m = next; + } + node.jump[0] = m; node.mask[0] = steps ? mask : 1ull << 63; + // level l: up to MEMO_FAN jumps of level l-1 (of the children: computed already) + for (int l = 1; l < MEMO_LEVELS; l++) { + m = i; mask = 0; steps = 0; + for (int k = 0; k < MEMO_FAN; k++) { + const MemoHotNode& x = hot[m]; + if (x.mask[l - 1] >> 63) break; + mask |= x.mask[l - 1]; m = x.jump[l - 1]; steps++; + } + node.jump[l] = m; node.mask[l] = steps > 1 ? mask : 1ull << 63; + } + } +} + +// Walk the tree from the root as runWayFunction does. Kinds: 0 Unset, 1 Find, 2 Holds, 3 IsClosed, +// 4 NextRelation, 5 Leaf, 6 Lua, 7 MercatorArea, 8 FindInRelation. Returns the node reached that isn't +// a read (Unset, Leaf, Lua), or UINT32_MAX when the result read at node `last` has no child yet. +// childOf(n, result): the child of node n for a result other than the four quick ones. +// extra: the results of the reads that aren't tags: nextRelation() (only called with relations), +// area(), findInRelation(key id) +template +inline uint32_t memoWalk(const MemoHotNode* hot, const MemoTagSlots& slots, bool isClosed, bool hasRelations, const ChildOf& childOf, Extra& extra, uint32_t& last) { + auto byResult = [&](uint32_t n, std::string_view r) { + if (r.empty()) return hot[n].quick[0]; + if (r.size() == 1 && (r[0] == '0' || r[0] == '1' || r[0] == '\x01')) return hot[n].quick[r[0] == '0' ? 1 : r[0] == '1' ? 2 : 3]; + return childOf(n, r); + }; + const uint64_t present = slots.present | (isClosed ? MEMO_CLOSED_BIT : 0) | (hasRelations ? MEMO_RELATION_BIT : 0); + uint32_t n = 0; + while (true) { + const MemoHotNode& node = hot[n]; + bool jumped = false; + for (int l = MEMO_LEVELS; l-- > 0; ) + if (!(present & node.mask[l])) { n = node.jump[l]; jumped = true; break; } + if (jumped) continue; + uint32_t next; + switch (node.kind) { + case 1: { // Find + if (!slots.has(node.key)) { next = node.quick[0]; break; } + const std::string_view v = slots.get(node.key); + if (v.empty()) next = node.quick[0]; + else if (node.valueOnly) next = node.quick[3]; + else if (v.size() == 1 && (v[0] == '0' || v[0] == '1' || v[0] == '\x01')) next = node.quick[v[0] == '0' ? 1 : v[0] == '1' ? 2 : 3]; + else next = childOf(n, v); + break; + } + case 2: next = node.quick[slots.has(node.key) ? 2 : 1]; break; // Holds + case 3: next = node.quick[isClosed ? 2 : 1]; break; + case 4: next = hasRelations ? byResult(n, extra.nextRelation()) : node.quick[1]; break; + case 7: next = byResult(n, extra.area()); break; + case 8: next = byResult(n, extra.findInRelation(node.key)); break; + default: return n; + } + if (next == UINT32_MAX) { last = n; return UINT32_MAX; } + n = next; + } +} + +#endif //_MEMO_KEYS_H diff --git a/include/osm_lua_processing.h b/include/osm_lua_processing.h index ca22a5f0..f55c2c57 100644 --- a/include/osm_lua_processing.h +++ b/include/osm_lua_processing.h @@ -3,6 +3,7 @@ #define _OSM_LUA_PROCESSING_H #include +#include #include #include #include @@ -32,6 +33,7 @@ extern "C" { #include "external/kaguya.hpp" #include "lua_pool.h" +#include "memo_keys.h" // FIXME: why is this global ? extern bool verbose; @@ -155,6 +157,10 @@ class OsmLuaProcessing { // Returns area double Area(); + // Area in Web Mercator square metres, the unit the zoom levels' pixel sizes are given in + double MercatorArea(); + // MinZoom(z) for the first z = 1, 2... with z >= limit and the area above thresholds[z-1]; else one beyond + void MinZoomByArea(double limit, const std::vector &thresholds); double multiPolygonArea(const MultiPolygon &mp) const; // Returns length @@ -307,10 +313,241 @@ class OsmLuaProcessing { static std::unordered_map dataStore; static std::mutex dataStoreMutex; + // ---- Memo of way_function (opt-in: way_memo = true in the profile) + // way_function is a function of what it reads (Find, Holds, IsClosed, NextRelation) and what it + // writes is a sequence of calls (Layer, Attribute..., MinZoom...). The first time, a way runs + // through Lua and both are recorded into a decision tree: each node is a read, branched by its + // result, and a leaf holds the calls. A later way that reads the same results reaches the same + // leaf, and the calls are replayed without Lua. Anything else the profile asks for (geometry, + // relations, Id, SetData...) makes that path run through Lua from then on. + struct MemoOp { + enum Kind : uint8_t { Layer, Centroid, Attr, AttrNumeric, AttrBoolean, AttrInteger, FastStr, FastNumeric, FastBoolean, FastInteger, MinZoom, ZOrder, Score, ModifyId, MinZoomByArea } kind; + bool flag = false; + char minzoom = 0; + double number = 0; + std::string key, str; + std::vector numbers; + std::string valueKey; // not empty: str is the current object's value of this tag (see way_memo_values) + bool valueLength = false; // valueKey, numeric ops: the attribute is the value's length in bytes + uint32_t valueKeyId = MemoKeyTable::NONE; // its id in memoKeys + // Filled by the replays (on the tree's thread), see memoReplay + enum Segment : uint8_t { SegmentKnown = 1, SegmentAttributes = 2, SegmentConstant = 4 }; + uint8_t segment = 0; // Layer, Centroid: about the attribute ops up to the next Layer/Centroid op + int16_t layerIndex = -1; // Layer, Centroid: layers.layerMap[key] + uint32_t segmentSet = UINT32_MAX; // Layer, Centroid (constant segment): AttributeIndex of an output with this segment's attributes + uint32_t segmentOps = 0; // Layer, Centroid: the number of ops of its segment (after it) + uint32_t segmentId = 0; // Layer, Centroid: the segment's id in memoSegmentSets + int16_t keyIndex = -1; // attribute ops: attributeStore.keyStore.key2index(key) + int16_t seenLayer = -1; // attribute ops: setVectorLayerMetadata has run for this layer and key + uint32_t pairIndex = 0; // attribute ops without valueKey: the pair added (0: not yet known) + }; + // The tree's nodes are MemoHotNodes (memo_keys.h: kind, key id, the children for the results + // "", "0", "1" and "\x01"); a node with children for other results, or a Leaf, has a MemoMore. + struct MemoNode { + // MercatorArea (way_memo_area): the area compared with each threshold (memoAreaResult); + // FindInRelation (way_memo_relations): the current relation's value of the key + enum Kind : uint8_t { Unset, Find, Holds, IsClosed, NextRelation, Leaf, Lua, MercatorArea, FindInRelation }; + static int quickOf(std::string_view r) { + if (r.empty()) return 0; + if (r.size() != 1) return -1; + return r[0] == '0' ? 1 : r[0] == '1' ? 2 : r[0] == '\x01' ? 3 : -1; + } + }; + struct MemoMore { + std::vector> children; // result -> node, results other than the quick ones + std::vector index; // with many children: open-addressing hash of results -> children position + 1 + std::vector ops; // Leaf + uint32_t uses = 0; // Leaf, Lua: objects that ended here since the last memoCompact + static uint64_t hashOf(std::string_view r) { + uint64_t h = 1469598103934665603ull; + for (unsigned char c : r) { h ^= c; h *= 1099511628211ull; } + return h; + } + uint32_t child(std::string_view r) const { + if (index.empty()) { + for (const auto &c : children) if (c.first.size() == r.size() && c.first == r) return c.second; + return UINT32_MAX; + } + const size_t mask = index.size() - 1; + for (size_t i = hashOf(r) & mask; index[i] != 0; i = (i + 1) & mask) { + const auto &c = children[index[i] - 1]; + if (c.first.size() == r.size() && c.first == r) return c.second; + } + return UINT32_MAX; + } + void addChild(std::string_view r, uint32_t node) { + children.emplace_back(std::string(r), node); + if (children.size() < 8) return; + if (children.size() * 4 > index.size()) { + // (re)build, at most a quarter full + size_t size = 256; + while (size < children.size() * 4) size *= 2; + index.assign(size, 0); + for (size_t k = 0; k + 1 < children.size(); k++) placeChild(k); + } + placeChild(children.size() - 1); + } + void placeChild(size_t k) { + const size_t mask = index.size() - 1; + size_t i = hashOf(children[k].first) & mask; + while (index[i] != 0) i = (i + 1) & mask; + index[i] = k + 1; + } + }; + uint32_t memoChild(uint32_t n, std::string_view r) const { + const MemoHotNode &node = memoNodes[n]; + const int q = MemoNode::quickOf(r); + if (q >= 0) return node.quick[q]; + return node.more == UINT32_MAX ? UINT32_MAX : memoMore[node.more].child(r); + } + MemoMore &memoMoreOf(uint32_t n) { + if (memoNodes[n].more == UINT32_MAX) { memoNodes[n].more = memoMore.size(); memoMore.emplace_back(); } + return memoMore[memoNodes[n].more]; + } + void memoAddChild(uint32_t n, std::string_view r, uint32_t child) { + memoNodes[n].children++; + const int q = MemoNode::quickOf(r); + if (q >= 0) memoNodes[n].quick[q] = child; + else memoMoreOf(n).addChild(r, child); + } + struct MemoRead { MemoNode::Kind kind; std::string key, result; bool valueOnly = false; }; + struct MemoTrace { + bool active = false; + bool luaOnly = false; // asked for something that isn't recorded + // A string attribute had the value of a memo_values tag it isn't declared for: a copy, or the + // same string by chance? Other objects with the same reads can't tell either, so this trace + // isn't added to the tree (they may replay a leaf recorded from an object without the doubt) + bool ambiguous = false; + std::vector reads; + std::vector ops; + std::vector> values; // tags of way_memo_values read, with their values + std::vector emptyValues; // tags of way_memo_values read empty + }; + // way_memo_values: tags whose values way_function only tests for being empty and copies unchanged + // into the listed attributes; the memo replays such ways with the current values + std::unordered_map> memoValueTags; + bool memoValueTag(const char *key, size_t keyLen) const { + return !memoValueTags.empty() && memoValueTags.count(std::string(key, keyLen)) > 0; + } + // way_memo_values entries `attribute = "length"`: attribute -> tag (the attribute is the length of the tag's value) + std::unordered_map memoValueLengths; + // way_memo_area: way_function uses MercatorArea() only to compare it with these numbers, so the + // memo reads it as the results of those comparisons (memoAreaResult) + std::vector memoArea; + // way_memo_relations: way_function uses the id NextRelation() returns only to test whether there + // is one, and the relations' tags only through FindInRelation: the memo reads the roles and tags + bool memoRelations = false; + std::string memoAreaResult(double area) const; + // NextRelation() has returned a relation, and FindInRelation reads its tags + bool relationCurrent() const { return relationSubscript >= 0 && size_t(relationSubscript) < relationList.size(); } + // Record a string attribute: a copy of a way_memo_values tag becomes a placeholder + void memoStringAttribute(MemoOp &&op); + // Record a numeric attribute: the length of a way_memo_values tag, declared so, becomes a placeholder + void memoNumberAttribute(MemoOp &&op); + bool memoEnabled = false; + MemoTrace memoTrace; + std::vector memoNodes; + std::vector memoMore; + MemoKeyTable memoKeys; // the keys the tree reads, and those of way_memo_values + MemoTagSlots memoSlots; // the current way's tags with those keys + MemoSigCache memoWalkCache { 4096 }; // recent walks (of the tree in memoNodes) + // AttributeIndex of outputs of segments with way_memo_values attributes, by segment and values + MemoSigCache memoSegmentSets { 1024 }; + uint32_t memoSegmentIds = 0; // segment ids given out + size_t memoSegmentSig(const MemoOp &layerOp, char *sig) const; + protozero::data_view memoOpValue(const MemoOp &op) const; + size_t memoJumpsAt = 0; // memoNodes.size() when the jumps were last computed + uint64_t memoHits = 0, memoRecorded = 0, memoLua = 0; + // The memo of node_function (opt-in: node_memo = true, node_memo_values as way_memo_values) is + // the same, with its own tree; so is that of way_function for multipolygon relations (with + // way_memo, a tree of its own: LayerAsCentroid, Layer and the geometry act differently for + // them). The members above hold the memo of the objects being processed (memoKind), + // memoSaved those of the other kinds; memoSelect swaps them. + enum MemoKind : uint8_t { MemoWays, MemoNodes, MemoRelations, MEMO_KINDS }; + struct MemoState { + bool enabled = false; + std::unordered_map> valueTags; + std::unordered_map valueLengths; + std::vector area; + bool relations = false; + std::vector nodes; + std::vector more; + MemoKeyTable keys; + size_t jumpsAt = 0, fullMisses = 0; + uint64_t hits = 0, recorded = 0, lua = 0; + }; + MemoState memoSaved[MEMO_KINDS]; // [memoKind] is unused + MemoKind memoKind = MemoWays; + bool memoForNodes = false; // memoKind == MemoNodes + bool nodeMemoEnabled = false; // node_memo = true in the profile + void memoSwap(MemoState &o) { + std::swap(memoEnabled, o.enabled); + std::swap(memoValueTags, o.valueTags); + std::swap(memoValueLengths, o.valueLengths); + std::swap(memoArea, o.area); + std::swap(memoRelations, o.relations); + std::swap(memoNodes, o.nodes); + std::swap(memoMore, o.more); + std::swap(memoKeys, o.keys); + std::swap(memoJumpsAt, o.jumpsAt); + std::swap(memoHits, o.hits); + std::swap(memoRecorded, o.recorded); + std::swap(memoLua, o.lua); + std::swap(memoFullMisses, o.fullMisses); + } + void memoSelect(MemoKind kind) { + if (kind == memoKind) return; + memoSwap(memoSaved[memoKind]); // the current memo to its place, the unused state here + memoSwap(memoSaved[kind]); // and that to the place of the one selected + memoWalkCache.clear(); + memoKind = kind; + memoForNodes = kind == MemoNodes; + } + void memoSelect(bool nodes) { memoSelect(nodes ? MemoNodes : MemoWays); } + // hits, recorded, through Lua, tree nodes of each memo (by MemoKind) + void memoCounts(uint64_t counts[MEMO_KINDS][4]) const { + for (int k = 0; k < MEMO_KINDS; k++) { + uint64_t *c = counts[k]; + if (k == memoKind) { c[0] += memoHits; c[1] += memoRecorded; c[2] += memoLua; c[3] += memoNodes.size(); } + else { const MemoState &o = memoSaved[k]; c[0] += o.hits; c[1] += o.recorded; c[2] += o.lua; c[3] += o.nodes.size(); } + } + } + void memoRead(MemoNode::Kind kind, const char *key, size_t keyLen, const std::string &result) { + if (!memoTrace.luaOnly) memoTrace.reads.push_back({ kind, std::string(key, keyLen), result }); + } + // Find() of a way_memo_values tag: the branch is only whether it is empty + void memoReadValue(const char *key, size_t keyLen, const std::string &value) { + if (memoTrace.luaOnly) return; + memoTrace.reads.push_back({ MemoNode::Find, std::string(key, keyLen), value.empty() ? std::string() : std::string(1, '\x01'), true }); + if (value.empty()) { + for (const auto &e : memoTrace.emptyValues) if (e.size() == keyLen && e.compare(0, keyLen, key, keyLen) == 0) return; + memoTrace.emptyValues.emplace_back(key, keyLen); + return; + } + for (const auto &v : memoTrace.values) if (v.first.size() == keyLen && v.first.compare(0, keyLen, key, keyLen) == 0) return; // read before + memoTrace.values.emplace_back(std::string(key, keyLen), value); + } + void memoOp(MemoOp &&op) { memoTrace.ops.push_back(std::move(op)); } + void memoLuaOnly() { memoTrace.luaOnly = true; } + void runWayFunction(); // way_function (memoForNodes: node_function) through the memo + void runWayFunctionInLua(bool record); + void memoInsert(); + size_t memoFullMisses = 0; // objects since the last memoCompact that weren't recorded as the tree was full + void memoCompact(); + void memoReplay(std::vector &ops); + void memoAttribute(MemoOp &op, const protozero::data_view &str); + void memoSegmentAttributes(std::vector &ops, size_t layerOp); + void layerAsCentroid(const std::string &layerName, CentroidAlgorithm algorithm, const std::vector &relationRoles); + void layerAsCentroidAt(unsigned layerIndex, CentroidAlgorithm algorithm, const std::vector &relationRoles); + void layerAt(unsigned layerIndex, bool area); + static void printMemoStats(); private: /// Internal: clear current cached state inline void reset() { outputs.clear(); + outputPreset.clear(); + outputLearn.clear(); + outputSegment.clear(); currentRelation = nullptr; stringTable = nullptr; llVecPtr = nullptr; @@ -408,6 +645,12 @@ class OsmLuaProcessing { class LayerDefinition &layers; std::vector> outputs; // All output objects that have been created + // Memo replay, per output (may be shorter than outputs): its AttributeIndex if known already + // (UINT32_MAX: not), the layer op to tell its AttributeIndex (segmentSet, or memoSegmentSets for + // a segment with way_memo_values attributes; or nullptr), the layer op's position + std::vector outputPreset; + std::vector outputLearn; + std::vector outputSegment; // Point features in decluttered layers: the exact location we'd have indexed them at, // plus the score from Score(). Sparse - most objects add nothing here. diff --git a/include/tag_map.h b/include/tag_map.h index 4f7df1d6..9684263e 100644 --- a/include/tag_map.h +++ b/include/tag_map.h @@ -24,6 +24,9 @@ // This is true since the strings are owned by the protobuf block reader // 3. Max number of tag values will fit in a short // OSM limit is 5,000 tags per object +// +// The hashed form is built lazily, on the first lookup: most ways are answered by the +// way_function memo from the tags as added (see added()) and never need it. struct Tag { protozero::data_view key; protozero::data_view value; @@ -34,8 +37,14 @@ class TagMap { TagMap(); void reset(); - bool empty() const; - void addTag(const protozero::data_view& key, const protozero::data_view& value); + bool empty() const { return tags.empty(); } + void addTag(const protozero::data_view& key, const protozero::data_view& value) { + tags.push_back(Tag{key, value}); + if (built) insert(key, value); + } + + // The tags in the order they were added (a key added twice: the last value counts) + const std::vector& added() const { return tags; } // Return -1 if key not found, else return its keyLoc. int64_t getKey(const char* key, size_t size) const; @@ -73,15 +82,22 @@ class TagMap { uint32_t valueLoc; }; - uint32_t ensureKey(const protozero::data_view& key); - uint32_t ensureValue(const protozero::data_view& value); + uint32_t ensureKey(const protozero::data_view& key) const; + uint32_t ensureValue(const protozero::data_view& value) const; + void insert(const protozero::data_view& key, const protozero::data_view& value) const; + void ensureBuilt() const { if (!built) build(); } + void build() const; + + std::vector tags; - std::array, SHARDS> keys; - std::array, SHARDS> values; + // The hashed form of `tags`, built on demand + mutable bool built = false; + mutable std::array, SHARDS> keys; + mutable std::array, SHARDS> values; // Bit i set => keys[i] (resp. values[i]) may be non-empty. - uint16_t usedKeyShards = 0; - uint16_t usedValueShards = 0; + mutable uint16_t usedKeyShards = 0; + mutable uint16_t usedValueShards = 0; }; #endif //_TAG_MAP_H diff --git a/resources/process-openmaptiles.lua b/resources/process-openmaptiles.lua index 7591e60f..c34403a6 100644 --- a/resources/process-openmaptiles.lua +++ b/resources/process-openmaptiles.lua @@ -3,6 +3,11 @@ -- Copyright (c) 2016, KlokanTech.com & OpenMapTiles contributors. -- Used under CC-BY 4.0 +-- Speed: tilemaker's functions and the standard library as locals (a global is a table lookup per use). +-- tilemaker registers its functions after loading this file, so they are picked up in init_function. +local tonumber, ipairs, pairs, math, string, table = tonumber, ipairs, pairs, math, string, table +local Find, Holds, Layer, LayerAsCentroid, Attribute, AttributeNumeric, AttributeInteger, AttributeBoolean, MinZoom, ZOrder, IsClosed, Area, Centroid, NextRelation, FindInRelation, Accept, MercatorArea, MinZoomByArea + -------- -- Alter these lines to control which languages are written for place/streetnames -- @@ -16,8 +21,22 @@ default_language_attribute = "name_int" additional_languages = { } -------- +-- way_function only reads tags (and IsClosed/NextRelation) and calls tilemaker's functions: tilemaker may replay it +way_memo = true +-- Tag values way_function only tests for being empty and copies unchanged into these attributes: +-- the memo replays such ways with each way's own value +-- (ref: also its length, into ref_length) +way_memo_values = { name = { "name:latin" }, ["addr:housenumber"] = { "housenumber" }, ref = { "ref", ref_length = "length" } } +-- way_function uses NextRelation() only to test for a relation, and the relations' tags only through FindInRelation +way_memo_relations = true +-- node_function, likewise (the same tags: node_function only tests them for being empty and copies them unchanged) +node_memo = true +node_memo_values = { name = { "name:latin" }, ["addr:housenumber"] = { "housenumber" } } + -- Enter/exit Tilemaker function init_function(name,is_first) + Find, Holds, Layer, LayerAsCentroid, Attribute, AttributeNumeric, AttributeInteger, AttributeBoolean, MinZoom, ZOrder, IsClosed, Area, Centroid, NextRelation, FindInRelation, Accept, MercatorArea, MinZoomByArea = + _G.Find, _G.Holds, _G.Layer, _G.LayerAsCentroid, _G.Attribute, _G.AttributeNumeric, _G.AttributeInteger, _G.AttributeBoolean, _G.MinZoom, _G.ZOrder, _G.IsClosed, _G.Area, _G.Centroid, _G.NextRelation, _G.FindInRelation, _G.Accept, _G.MercatorArea, _G.MinZoomByArea end function exit_function() end @@ -44,6 +63,25 @@ ZRES10 = 152.9 ZRES11 = 76.4 ZRES12 = 38.2 ZRES13 = 19.1 +-- Their squares, worked out once (same values as ZRESn^2 at each use) +local ZRES0SQ = ZRES0^2 +local ZRES1SQ = ZRES1^2 +local ZRES2SQ = ZRES2^2 +local ZRES3SQ = ZRES3^2 +local ZRES4SQ = ZRES4^2 +local ZRES5SQ = ZRES5^2 +local ZRES6SQ = ZRES6^2 +local ZRES7SQ = ZRES7^2 +local ZRES8SQ = ZRES8^2 +local ZRES9SQ = ZRES9^2 +local ZRES10SQ = ZRES10^2 +local ZRES11SQ = ZRES11^2 +local ZRES12SQ = ZRES12^2 +local ZRES13SQ = ZRES13^2 +-- Area thresholds for zoom 1, 2... 13 (see SetMinZoomByAreaWithLimit) +local ZRESSQ = { ZRES0SQ, ZRES1SQ, ZRES2SQ, ZRES3SQ, ZRES4SQ, ZRES5SQ, ZRES6SQ, ZRES7SQ, ZRES8SQ, ZRES9SQ, ZRES10SQ, ZRES11SQ, ZRES12SQ } +-- way_function compares MercatorArea() only with these (see way_memo) +way_memo_area = { ZRES8SQ } -- The height of one floor, in meters BUILDING_FLOOR_HEIGHT = 3.66 @@ -746,7 +784,7 @@ function way_function() Layer("landuse", true) Attribute("class", l) if l=="residential" then - if MercatorArea()ZRES0^2 then MinZoom(1) - elseif minzoom <= 2 and area>ZRES1^2 then MinZoom(2) - elseif minzoom <= 3 and area>ZRES2^2 then MinZoom(3) - elseif minzoom <= 4 and area>ZRES3^2 then MinZoom(4) - elseif minzoom <= 5 and area>ZRES4^2 then MinZoom(5) - elseif minzoom <= 6 and area>ZRES5^2 then MinZoom(6) - elseif minzoom <= 7 and area>ZRES6^2 then MinZoom(7) - elseif minzoom <= 8 and area>ZRES7^2 then MinZoom(8) - elseif minzoom <= 9 and area>ZRES8^2 then MinZoom(9) - elseif minzoom <= 10 and area>ZRES9^2 then MinZoom(10) - elseif minzoom <= 11 and area>ZRES10^2 then MinZoom(11) - elseif minzoom <= 12 and area>ZRES11^2 then MinZoom(12) - elseif minzoom <= 13 and area>ZRES12^2 then MinZoom(13) - else MinZoom(14) end + -- The first zoom z >= minzoom whose pixel area the feature exceeds (ZRESSQ[z]), else 14: + -- worked out by tilemaker, so that way_function stays replayable (see way_memo) + MinZoomByArea(minzoom, ZRESSQ) end -- Calculate POIs (typically rank 1-4 go to 'poi' z12-14, rank 5+ to 'poi_detail' z14) @@ -918,8 +934,8 @@ function GetPOIRank() -- Can we find the tag? for _,k in ipairs(poiTagOrder) do list = poiTags[k] - if list[Find(k)] then - v = Find(k) -- k/v are the OSM tag pair + v = Find(k) -- k/v are the OSM tag pair + if list[v] then class = poiClasses[v] or poiTagsAsClass[k] or v rank = poiClassRanks[class] or 25 subclassKey = poiSubClasses[v] diff --git a/src/attribute_store.cpp b/src/attribute_store.cpp index f2a3d77d..cca1a2cc 100644 --- a/src/attribute_store.cpp +++ b/src/attribute_store.cpp @@ -355,26 +355,38 @@ void AttributeStore::addAttribute(AttributeSet& attributeSet, std::string const bool isHot = v >= 0 && v <= 25; attributeSet.addPair(pairStore.addPair(kv, isHot)); } -void AttributeStore::addAttributeWithKeyIndex(AttributeSet& attributeSet, uint16_t keyIndex, std::string const &key, const protozero::data_view v, char minzoom) { +uint32_t AttributeStore::pairIndexWithKeyIndex(uint16_t keyIndex, std::string const &key, const protozero::data_view v, char minzoom) { PooledString ps(&v); AttributePair kv(keyIndex, ps, minzoom); bool isHot = AttributePair::isHot(key, v); - attributeSet.addPair(pairStore.addPair(kv, isHot)); + return pairStore.addPair(kv, isHot); } -void AttributeStore::addAttributeWithKeyIndex(AttributeSet& attributeSet, uint16_t keyIndex, bool v, char minzoom) { +uint32_t AttributeStore::pairIndexWithKeyIndex(uint16_t keyIndex, bool v, char minzoom) { AttributePair kv(keyIndex,v,minzoom); bool isHot = true; // All bools are eligible to be hot pairs - attributeSet.addPair(pairStore.addPair(kv, isHot)); + return pairStore.addPair(kv, isHot); } -void AttributeStore::addAttributeWithKeyIndex(AttributeSet& attributeSet, uint16_t keyIndex, double v, char minzoom) { +uint32_t AttributeStore::pairIndexWithKeyIndex(uint16_t keyIndex, double v, char minzoom) { AttributePair kv(keyIndex,v,minzoom); bool isHot = v >= 0 && v <= 25 && ceil(v) == v; // Whole numbers in 0..25 are eligible to be hot pairs - attributeSet.addPair(pairStore.addPair(kv, isHot)); + return pairStore.addPair(kv, isHot); } -void AttributeStore::addAttributeWithKeyIndex(AttributeSet& attributeSet, uint16_t keyIndex, int v, char minzoom) { +uint32_t AttributeStore::pairIndexWithKeyIndex(uint16_t keyIndex, int v, char minzoom) { AttributePair kv(keyIndex,v,minzoom); bool isHot = v >= 0 && v <= 25; - attributeSet.addPair(pairStore.addPair(kv, isHot)); + return pairStore.addPair(kv, isHot); +} +void AttributeStore::addAttributeWithKeyIndex(AttributeSet& attributeSet, uint16_t keyIndex, std::string const &key, const protozero::data_view v, char minzoom) { + attributeSet.addPair(pairIndexWithKeyIndex(keyIndex, key, v, minzoom)); +} +void AttributeStore::addAttributeWithKeyIndex(AttributeSet& attributeSet, uint16_t keyIndex, bool v, char minzoom) { + attributeSet.addPair(pairIndexWithKeyIndex(keyIndex, v, minzoom)); +} +void AttributeStore::addAttributeWithKeyIndex(AttributeSet& attributeSet, uint16_t keyIndex, double v, char minzoom) { + attributeSet.addPair(pairIndexWithKeyIndex(keyIndex, v, minzoom)); +} +void AttributeStore::addAttributeWithKeyIndex(AttributeSet& attributeSet, uint16_t keyIndex, int v, char minzoom) { + attributeSet.addPair(pairIndexWithKeyIndex(keyIndex, v, minzoom)); } void AttributeSet::finalize() { @@ -421,6 +433,12 @@ thread_local std::vector cachedAttributeSetIndexes(256); thread_local uint64_t tlsSetLookups = 0; thread_local uint64_t tlsSetLookupsUncached = 0; +void AttributeStore::noteLookup() { + tlsSetLookups++; + if (tlsSetLookups % 1024 == 0) { + lookups += 1024; + } +} AttributeIndex AttributeStore::add(AttributeSet &attributes) { // TODO: there's probably a way to use C++ types to distinguish a finalized // and non-finalized AttributeSet, which would make this safer. diff --git a/src/osm_lua_processing.cpp b/src/osm_lua_processing.cpp index ba34304e..db616479 100644 --- a/src/osm_lua_processing.cpp +++ b/src/osm_lua_processing.cpp @@ -142,9 +142,9 @@ kaguya::LuaTable getAllTags(kaguya::State& luaState, const boost::container::fla return tagsTable; } -std::string rawId() { return osmLuaProcessing->Id(); } -std::string rawOsmType() { return osmLuaProcessing->OsmType(); } -kaguya::LuaTable rawAllKeys() { +std::string rawId() { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); return osmLuaProcessing->Id(); } +std::string rawOsmType() { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); return osmLuaProcessing->OsmType(); } +kaguya::LuaTable rawAllKeys() { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); if (osmLuaProcessing->isPostScanRelation) { return osmLuaProcessing->AllKeys(*g_luaState); } @@ -152,7 +152,7 @@ kaguya::LuaTable rawAllKeys() { auto tags = osmLuaProcessing->currentTags->exportToBoostMap(); return getAllKeys(*g_luaState, &tags); -}kaguya::LuaTable rawAllTags() { +}kaguya::LuaTable rawAllTags() { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); if (osmLuaProcessing->isPostScanRelation) { return osmLuaProcessing->AllTags(*g_luaState); } @@ -161,8 +161,8 @@ kaguya::LuaTable rawAllKeys() { return getAllTags(*g_luaState, &tags); } -bool rawHasTags() { return osmLuaProcessing->HasTags(); } -void rawSetTag(const std::string &key, const std::string &value) { return osmLuaProcessing->SetTag(key, value); } +bool rawHasTags() { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); return osmLuaProcessing->HasTags(); } +void rawSetTag(const std::string &key, const std::string &value) { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); return osmLuaProcessing->SetTag(key, value); } // Native lua_CFunction versions of Holds/Find, bypassing kaguya's invoker. // Semantics match the former kaguya-bound versions (KnownTagKey): kaguya does // no type check for a single (non-overloaded) function, so the argument is @@ -185,7 +185,9 @@ static int luaHolds(lua_State* L) { return 1; } - lua_pushboolean(L, osmLuaProcessing->currentTags->getKey(buffer, size) >= 0); + const bool held = osmLuaProcessing->currentTags->getKey(buffer, size) >= 0; + if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoRead(OsmLuaProcessing::MemoNode::Holds, buffer, size, held ? "1" : "0"); + lua_pushboolean(L, held); return 1; } @@ -211,8 +213,16 @@ static int luaFind(lua_State* L) { int64_t tagLoc = tags->getKey(buffer, size); if (tagLoc >= 0) { const protozero::data_view* value = tags->getValueFromKey(tagLoc); + if (osmLuaProcessing->memoTrace.active) { + if (osmLuaProcessing->memoValueTag(buffer, size)) osmLuaProcessing->memoReadValue(buffer, size, std::string(value->data(), value->size())); + else osmLuaProcessing->memoRead(OsmLuaProcessing::MemoNode::Find, buffer, size, std::string(value->data(), value->size())); + } lua_pushlstring(L, value->data(), value->size()); } else { + if (osmLuaProcessing->memoTrace.active) { + if (osmLuaProcessing->memoValueTag(buffer, size)) osmLuaProcessing->memoReadValue(buffer, size, std::string()); + else osmLuaProcessing->memoRead(OsmLuaProcessing::MemoNode::Find, buffer, size, std::string()); + } lua_pushvalue(L, lua_upvalueindex(1)); } return 1; @@ -279,36 +289,70 @@ static void wrapKaguyaBinding(lua_State* L, const char* name, lua_CFunction fn) lua_pop(L, 1); } -std::vector rawFindIntersecting(const std::string &layerName) { return osmLuaProcessing->FindIntersecting(layerName); } -bool rawIntersects(const std::string& layerName) { return osmLuaProcessing->Intersects(layerName); } -std::vector rawFindCovering(const std::string& layerName) { return osmLuaProcessing->FindCovering(layerName); } -bool rawCoveredBy(const std::string& layerName) { return osmLuaProcessing->CoveredBy(layerName); } -bool rawIsClosed() { return osmLuaProcessing->IsClosed(); } -bool rawIsMultiPolygon() { return osmLuaProcessing->IsMultiPolygon(); } -double rawArea() { return osmLuaProcessing->Area(); } -double rawLength() { return osmLuaProcessing->Length(); } -kaguya::optional> rawCentroid(kaguya::VariadicArgType algorithm) { return osmLuaProcessing->Centroid(algorithm); } +std::vector rawFindIntersecting(const std::string &layerName) { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); return osmLuaProcessing->FindIntersecting(layerName); } +bool rawIntersects(const std::string& layerName) { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); return osmLuaProcessing->Intersects(layerName); } +std::vector rawFindCovering(const std::string& layerName) { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); return osmLuaProcessing->FindCovering(layerName); } +bool rawCoveredBy(const std::string& layerName) { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); return osmLuaProcessing->CoveredBy(layerName); } +bool rawIsClosed() { + const bool closed = osmLuaProcessing->IsClosed(); + if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoRead(OsmLuaProcessing::MemoNode::IsClosed, "", 0, closed ? "1" : "0"); + return closed; +} +bool rawIsMultiPolygon() { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); return osmLuaProcessing->IsMultiPolygon(); } +double rawMercatorArea() { + const double area = osmLuaProcessing->MercatorArea(); + if (osmLuaProcessing->memoTrace.active) { + if (osmLuaProcessing->memoArea.empty()) osmLuaProcessing->memoLuaOnly(); + else osmLuaProcessing->memoRead(OsmLuaProcessing::MemoNode::MercatorArea, "", 0, osmLuaProcessing->memoAreaResult(area)); + } + return area; +} +void rawMinZoomByArea(double limit, std::vector thresholds) { return osmLuaProcessing->MinZoomByArea(limit, thresholds); } +double rawArea() { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); return osmLuaProcessing->Area(); } +double rawLength() { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); return osmLuaProcessing->Length(); } +kaguya::optional> rawCentroid(kaguya::VariadicArgType algorithm) { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); return osmLuaProcessing->Centroid(algorithm); } void rawModifyId(const int newId) { return osmLuaProcessing->ModifyId(newId); } void rawLayer(const std::string& layerName, bool area) { return osmLuaProcessing->Layer(layerName, area); } void rawLayerAsCentroid(const std::string &layerName, kaguya::VariadicArgType nodeSources) { return osmLuaProcessing->LayerAsCentroid(layerName, nodeSources); } void rawMinZoom(const double z) { return osmLuaProcessing->MinZoom(z); } void rawZOrder(const double z) { return osmLuaProcessing->ZOrder(z); } void rawScore(const double score) { return osmLuaProcessing->Score(score); } -OsmLuaProcessing::OptionalRelation rawNextRelation() { return osmLuaProcessing->NextRelation(); } -void rawRestartRelations() { return osmLuaProcessing->RestartRelations(); } -std::string rawFindInRelation(const std::string& key) { return osmLuaProcessing->FindInRelation(key); } -void rawAccept() { return osmLuaProcessing->Accept(); } -double rawAreaIntersecting(const std::string& layerName) { return osmLuaProcessing->AreaIntersecting(layerName); } - -void rawSetData(const std::string &key, const std::string &value) { +OsmLuaProcessing::OptionalRelation rawNextRelation() { + OsmLuaProcessing::OptionalRelation r = osmLuaProcessing->NextRelation(); + if (osmLuaProcessing->memoTrace.active) { + // way_memo_relations: the read is whether there is a relation, and its role + osmLuaProcessing->memoRead(OsmLuaProcessing::MemoNode::NextRelation, "", 0, r.done ? "0" : osmLuaProcessing->memoRelations ? "1" + r.role : "1"); + if (!r.done && !osmLuaProcessing->memoRelations) osmLuaProcessing->memoLuaOnly(); + } + return r; +} +void rawRestartRelations() { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); return osmLuaProcessing->RestartRelations(); } +std::string rawFindInRelation(const std::string& key) { + if (osmLuaProcessing->memoTrace.active && !(osmLuaProcessing->memoRelations && osmLuaProcessing->relationCurrent())) osmLuaProcessing->memoLuaOnly(); + std::string value = osmLuaProcessing->FindInRelation(key); + if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoRead(OsmLuaProcessing::MemoNode::FindInRelation, key.data(), key.size(), value); + return value; +} +void rawAccept() { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); return osmLuaProcessing->Accept(); } +double rawAreaIntersecting(const std::string& layerName) { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); return osmLuaProcessing->AreaIntersecting(layerName); } + +void rawSetData(const std::string &key, const std::string &value) { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); std::lock_guard lock(osmLuaProcessing->dataStoreMutex); osmLuaProcessing->dataStore[key] = value; } -std::string rawGetData(const std::string &key) { +std::string rawGetData(const std::string &key) { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); auto r = osmLuaProcessing->dataStore.find(key); return r==osmLuaProcessing->dataStore.end() ? "" : r->second; } +static std::unordered_map> readMemoValueTags(lua_State *L, const char *global, std::unordered_map &lengths); +static std::vector readMemoArea(lua_State *L, const char *global); + +// Memo of way_function: the processors that use it (for printMemoStats) +static std::mutex memoInstancesMutex; +static std::vector memoInstances; +static uint64_t memoDone[OsmLuaProcessing::MEMO_KINDS][4] = {}; // of processors gone, by MemoKind: hits, recorded, through Lua, tree nodes + bool supportsRemappingShapefiles = false; int lua_error_handler(int errCode, const char *errMessage) @@ -453,6 +497,8 @@ OsmLuaProcessing::OsmLuaProcessing( luaState["IsClosed"] = &rawIsClosed; luaState["IsMultiPolygon"] = &rawIsMultiPolygon; luaState["Area"] = &rawArea; + luaState["MercatorArea"] = &rawMercatorArea; + luaState["MinZoomByArea"] = &rawMinZoomByArea; luaState["AreaIntersecting"] = &rawAreaIntersecting; luaState["Length"] = &rawLength; luaState["Centroid"] = &rawCentroid; @@ -497,6 +543,31 @@ OsmLuaProcessing::OsmLuaProcessing( supportsWritingNodes = !!luaState["node_function"]; supportsWritingWays = !!luaState["way_function"]; supportsWritingRelations = !!luaState["relation_function"]; + memoEnabled = !!luaState["way_memo"]; + if (memoEnabled) { + memoValueTags = readMemoValueTags(luaState.state(), "way_memo_values", memoValueLengths); + memoArea = readMemoArea(luaState.state(), "way_memo_area"); + memoRelations = !!luaState["way_memo_relations"]; + } + nodeMemoEnabled = supportsWritingNodes && !!luaState["node_memo"]; + MemoState &nodeMemo = memoSaved[MemoNodes]; + nodeMemo.enabled = nodeMemoEnabled; + if (nodeMemoEnabled) { + nodeMemo.valueTags = readMemoValueTags(luaState.state(), "node_memo_values", nodeMemo.valueLengths); + nodeMemo.area = readMemoArea(luaState.state(), "node_memo_area"); + nodeMemo.relations = !!luaState["node_memo_relations"]; + } + // Multipolygon relations run way_function: its memo, with a tree of their own + MemoState &relationMemo = memoSaved[MemoRelations]; + relationMemo.enabled = memoEnabled; + relationMemo.valueTags = memoValueTags; + relationMemo.valueLengths = memoValueLengths; + relationMemo.area = memoArea; + relationMemo.relations = memoRelations; + if (memoEnabled || nodeMemoEnabled) { + std::lock_guard lock(memoInstancesMutex); + memoInstances.push_back(this); + } // ---- Call init_function of Lua logic @@ -506,6 +577,14 @@ OsmLuaProcessing::OsmLuaProcessing( } OsmLuaProcessing::~OsmLuaProcessing() { + { + std::lock_guard lock(memoInstancesMutex); + auto it = std::find(memoInstances.begin(), memoInstances.end(), this); + if (it != memoInstances.end()) { + memoInstances.erase(it); + memoCounts(memoDone); + } + } // Call exit_function of Lua logic luaState("if exit_function~=nil then exit_function() end"); } @@ -731,6 +810,34 @@ double OsmLuaProcessing::projectedPolygonArea(const Polygon &p) { } #endif +// Geometries are kept in Web Mercator (lon, latp in degrees), so the planar area only needs scaling +double OsmLuaProcessing::MercatorArea() { + if (!IsClosed()) return 0; + const double metresPerDegree = 6378137.0 * M_PI / 180.0; // Web Mercator, at the equator + double area = 0; + if (isRelation) { + for (const Polygon &p : multiPolygonCached()) area += std::abs(geom::area(p)); + } else if (isWay) { + area = std::abs(geom::area(polygonCached())); + } + return area * metresPerDegree * metresPerDegree; +} + +void OsmLuaProcessing::MinZoomByArea(double limit, const std::vector &thresholds) { + if (memoTrace.active) { + MemoOp op{ MemoOp::MinZoomByArea, false, 0, limit, std::string(), std::string() }; + op.numbers = thresholds; + memoOp(std::move(op)); + } + if (outputs.size()==0) { ProcessingError("Can't set minimum zoom if no Layer set"); return; } + const double area = MercatorArea(); + double z = thresholds.size() + 1; + for (std::size_t i = 0; i < thresholds.size(); i++) { + if (limit <= i + 1 && area > thresholds[i]) { z = i + 1; break; } + } + outputs.back().first.setMinZoom(z); +} + // Returns area double OsmLuaProcessing::Area() { if (!IsClosed()) return 0; @@ -788,10 +895,12 @@ const Linestring &OsmLuaProcessing::linestringCached() { if (!linestringInited) { linestringInited = true; // Same as osmStore.llListLinestring(), refilling the cache in place - linestringCache.clear(); - linestringCache.reserve(llVecPtr->size()); - for (const LatpLon &ll : *llVecPtr) - linestringCache.push_back(geom::make(ll.lon/10000000.0, ll.latp/10000000.0)); + const size_t n = llVecPtr->size(); + linestringCache.resize(n); + const LatpLon *src = llVecPtr->data(); + Point *dst = linestringCache.data(); + for (size_t i = 0; i < n; i++) + dst[i] = geom::make(src[i].lon/10000000.0, src[i].latp/10000000.0); } return linestringCache; } @@ -807,7 +916,18 @@ const MultiLinestring &OsmLuaProcessing::multiLinestringCached() { const Polygon &OsmLuaProcessing::polygonCached() { if (!polygonInited) { polygonInited = true; - polygonCache = osmStore.llListPolygon(llVecPtr->cbegin(), llVecPtr->cend()); + // As osmStore.llListPolygon(), refilling the cache in place: the points of + // linestringCached(), then corrected + auto &outer = polygonCache.outer(); + polygonCache.inners().clear(); + if (linestringInited) outer.assign(linestringCache.begin(), linestringCache.end()); + else { + outer.clear(); + outer.reserve(llVecPtr->size()); + for (const LatpLon &ll : *llVecPtr) + outer.push_back(geom::make(ll.lon/10000000.0, ll.latp/10000000.0)); + } + geom::correct(polygonCache); } return polygonCache; } @@ -826,12 +946,19 @@ const MultiPolygon &OsmLuaProcessing::multiPolygonCached() { // Add object to specified layer from Lua void OsmLuaProcessing::Layer(const string &layerName, bool area) { + if (memoTrace.active) memoOp({ MemoOp::Layer, area, 0, 0, layerName, std::string() }); outputKeys.clear(); - if (layers.layerMap.count(layerName) == 0) { + const auto layer = layers.layerMap.find(layerName); + if (layer == layers.layerMap.end()) { throw out_of_range("ERROR: Layer(): a layer named as \"" + layerName + "\" doesn't exist."); } + layerAt(layer->second, area); +} - uint layerMinZoom = layers.layers[layers.layerMap[layerName]].minzoom; +// Layer(), with the layer's index +void OsmLuaProcessing::layerAt(unsigned layerIndex, bool area) { + outputKeys.clear(); + uint layerMinZoom = layers.layers[layerIndex].minzoom; AttributeSet attributes; OutputGeometryType geomType = isRelation ? (area ? POLYGON_ : MULTILINESTRING_ ) : isWay ? (area ? POLYGON_ : LINESTRING_) : POINT_; @@ -840,7 +967,7 @@ void OsmLuaProcessing::Layer(const string &layerName, bool area) { // a highway and its name. Avoid duplicating geometry processing and storage // when this occurs. if (lastStoredGeometryId != 0 && lastStoredGeometryType == geomType) { - OutputObject oo(geomType, layers.layerMap[layerName], lastStoredGeometryId, 0, layerMinZoom); + OutputObject oo(geomType, layerIndex, lastStoredGeometryId, 0, layerMinZoom); outputs.push_back(std::make_pair(std::move(oo), attributes)); return; } @@ -853,7 +980,7 @@ void OsmLuaProcessing::Layer(const string &layerName, bool area) { NodeID id = USE_NODE_STORE | originalOsmID; if (materializeGeometries) id = osmMemTiles.storePoint(p); - OutputObject oo(geomType, layers.layerMap[layerName], id, 0, layerMinZoom); + OutputObject oo(geomType, layerIndex, id, 0, layerMinZoom); outputs.push_back(std::make_pair(std::move(oo), attributes)); noteIfDecluttered(LatpLon { latp, lon }); return; @@ -871,7 +998,7 @@ void OsmLuaProcessing::Layer(const string &layerName, bool area) { const bool knownValid = correctionResult == CorrectGeometryResult::Valid || PrecheckValid(mp) || geom::is_valid(mp); NodeID id = osmMemTiles.storeMultiPolygon(mp, knownValid); - OutputObject oo(geomType, layers.layerMap[layerName], id, 0, layerMinZoom); + OutputObject oo(geomType, layerIndex, id, 0, layerMinZoom); outputs.push_back(std::make_pair(std::move(oo), attributes)); } catch(std::out_of_range &err) { cout << "In relation " << originalOsmID << ": " << err.what() << endl; @@ -899,7 +1026,7 @@ void OsmLuaProcessing::Layer(const string &layerName, bool area) { wayEmitted = true; } else id = osmMemTiles.storeMultiPolygon(mp, correctionResult != CorrectGeometryResult::Corrected); - OutputObject oo(geomType, layers.layerMap[layerName], id, 0, layerMinZoom); + OutputObject oo(geomType, layerIndex, id, 0, layerMinZoom); outputs.push_back(std::make_pair(std::move(oo), attributes)); } @@ -918,7 +1045,7 @@ void OsmLuaProcessing::Layer(const string &layerName, bool area) { NodeID id = osmMemTiles.storeMultiLinestring(mls); lastStoredGeometryId = id; lastStoredGeometryType = geomType; - OutputObject oo(geomType, layers.layerMap[layerName], id, 0, layerMinZoom); + OutputObject oo(geomType, layerIndex, id, 0, layerMinZoom); outputs.push_back(std::make_pair(std::move(oo), attributes)); } else if (geomType==LINESTRING_) { @@ -944,13 +1071,13 @@ void OsmLuaProcessing::Layer(const string &layerName, bool area) { id = osmMemTiles.storeLinestring(ls); lastStoredGeometryId = id; lastStoredGeometryType = geomType; - OutputObject oo(geomType, layers.layerMap[layerName], id, 0, layerMinZoom); + OutputObject oo(geomType, layerIndex, id, 0, layerMinZoom); outputs.push_back(std::make_pair(std::move(oo), attributes)); } else { NodeID id = osmMemTiles.storeLinestring(ls); lastStoredGeometryId = id; lastStoredGeometryType = geomType; - OutputObject oo(geomType, layers.layerMap[layerName], id, 0, layerMinZoom); + OutputObject oo(geomType, layerIndex, id, 0, layerMinZoom); outputs.push_back(std::make_pair(std::move(oo), attributes)); } } @@ -969,24 +1096,40 @@ void OsmLuaProcessing::Layer(const string &layerName, bool area) { // When called for a relation, you can pass a list of roles. The point of a node // with that role will be used if available. void OsmLuaProcessing::LayerAsCentroid(const string &layerName, kaguya::VariadicArgType varargs) { - outputKeys.clear(); if (layers.layerMap.count(layerName) == 0) { throw out_of_range("ERROR: LayerAsCentroid(): a layer named as \"" + layerName + "\" doesn't exist."); - } - + } + // The first argument is the algorithm, the others relation roles to look for CentroidAlgorithm algorithm = defaultCentroidAlgorithm(); - + std::vector relationRoles; + bool first = true; for (auto needleRef : varargs) { const std::string needle = needleRef.get(); - algorithm = parseCentroidAlgorithm(needle); - break; + if (first) algorithm = parseCentroidAlgorithm(needle); + else relationRoles.push_back(needle); + first = false; + } + if (memoTrace.active) { + if (isRelation && !relationRoles.empty()) memoLuaOnly(); + else memoOp({ MemoOp::Centroid, false, 0, static_cast(static_cast(algorithm)), layerName, std::string() }); } + layerAsCentroid(layerName, algorithm, relationRoles); +} + +void OsmLuaProcessing::layerAsCentroid(const std::string &layerName, CentroidAlgorithm algorithm, const std::vector &relationRoles) { + outputKeys.clear(); + layerAsCentroidAt(layers.layerMap[layerName], algorithm, relationRoles); +} + +// layerAsCentroid(), with the layer's index +void OsmLuaProcessing::layerAsCentroidAt(unsigned layerIndex, CentroidAlgorithm algorithm, const std::vector &relationRoles) { + outputKeys.clear(); // This will be non-zero if we ultimately used a node from a relation to // label the point. NodeID relationNode = 0; - uint layerMinZoom = layers.layers[layers.layerMap[layerName]].minzoom; + uint layerMinZoom = layers.layers[layerIndex].minzoom; AttributeSet attributes; Point geomp; bool centroidFound = false; @@ -994,12 +1137,7 @@ void OsmLuaProcessing::LayerAsCentroid(const string &layerName, kaguya::Variadic // If we're a relation, see if the user would prefer we use one of its members // to label the point. if (isRelation) { - int i = -1; - for (auto needleRef : varargs) { - i++; - // Skip the first vararg, it's the algorithm. - if (i == 0) continue; - const std::string needle = needleRef.get(); + for (const std::string &needle : relationRoles) { // We do a linear search of the relation's members. This is not very efficient // for relations like Tongass National Park (ID 6535292, 29,000 members), @@ -1071,7 +1209,7 @@ void OsmLuaProcessing::LayerAsCentroid(const string &layerName, kaguya::Variadic id = USE_WAY_STORE | originalOsmID; wayEmitted = true; } - OutputObject oo(POINT_, layers.layerMap[layerName], id, 0, layerMinZoom); + OutputObject oo(POINT_, layerIndex, id, 0, layerMinZoom); outputs.push_back(std::make_pair(std::move(oo), attributes)); noteIfDecluttered(LatpLon { (int32_t)geomp.y(), (int32_t)geomp.x() }); } @@ -1200,11 +1338,13 @@ void OsmLuaProcessing::removeAttributeIfNeeded(uint16_t keyIndex) { // Force a new ID void OsmLuaProcessing::ModifyId(const int newId) { + if (memoTrace.active) memoOp({ MemoOp::ModifyId, false, 0, static_cast(newId), std::string(), std::string() }); originalOsmID = static_cast(newId); } // Set attributes in a vector tile's Attributes table void OsmLuaProcessing::Attribute(const string &key, const protozero::data_view val, const char minzoom) { + if (memoTrace.active) memoStringAttribute({ MemoOp::Attr, false, minzoom, 0, key, std::string(val.data(), val.size()) }); if (outputs.size()==0) { ProcessingError("Can't add Attribute if no Layer set"); return; } const uint16_t keyIndex = attributeStore.keyStore.key2index(key); removeAttributeIfNeeded(keyIndex); @@ -1213,6 +1353,7 @@ void OsmLuaProcessing::Attribute(const string &key, const protozero::data_view v } void OsmLuaProcessing::AttributeNumeric(const string &key, const double val, const char minzoom) { + if (memoTrace.active) memoNumberAttribute({ MemoOp::AttrNumeric, false, minzoom, static_cast(val), key, std::string() }); if (outputs.size()==0) { ProcessingError("Can't add Attribute if no Layer set"); return; } const uint16_t keyIndex = attributeStore.keyStore.key2index(key); removeAttributeIfNeeded(keyIndex); @@ -1221,6 +1362,7 @@ void OsmLuaProcessing::AttributeNumeric(const string &key, const double val, con } void OsmLuaProcessing::AttributeBoolean(const string &key, const bool val, const char minzoom) { + if (memoTrace.active) memoOp({ MemoOp::AttrBoolean, val, minzoom, 0, key, std::string() }); if (outputs.size()==0) { ProcessingError("Can't add Attribute if no Layer set"); return; } const uint16_t keyIndex = attributeStore.keyStore.key2index(key); removeAttributeIfNeeded(keyIndex); @@ -1229,6 +1371,7 @@ void OsmLuaProcessing::AttributeBoolean(const string &key, const bool val, const } void OsmLuaProcessing::AttributeInteger(const string &key, const int val, const char minzoom) { + if (memoTrace.active) memoNumberAttribute({ MemoOp::AttrInteger, false, minzoom, static_cast(val), key, std::string() }); if (outputs.size()==0) { ProcessingError("Can't add Attribute if no Layer set"); return; } const uint16_t keyIndex = attributeStore.keyStore.key2index(key); removeAttributeIfNeeded(keyIndex); @@ -1238,12 +1381,14 @@ void OsmLuaProcessing::AttributeInteger(const string &key, const int val, const // Set minimum zoom void OsmLuaProcessing::MinZoom(const double z) { + if (memoTrace.active) memoOp({ MemoOp::MinZoom, false, 0, z, std::string(), std::string() }); if (outputs.size()==0) { ProcessingError("Can't set minimum zoom if no Layer set"); return; } outputs.back().first.setMinZoom(z); } // Set the score used to rank this feature against its neighbours when decluttering void OsmLuaProcessing::Score(const double score) { + if (memoTrace.active) memoOp({ MemoOp::Score, false, 0, score, std::string(), std::string() }); if (outputs.size()==0) { ProcessingError("Can't set score if no Layer set"); return; } if (declutterOutputs.empty() || declutterOutputs.back().outputIndex != outputs.size()-1) { ProcessingError("Score() only applies to point features in a layer with declutter_below set"); @@ -1254,6 +1399,7 @@ void OsmLuaProcessing::Score(const double score) { // Set z_order void OsmLuaProcessing::ZOrder(const double z) { + if (memoTrace.active) memoOp({ MemoOp::ZOrder, false, 0, z, std::string(), std::string() }); if (outputs.size()==0) { ProcessingError("Can't set z_order if no Layer set"); return; } outputs.back().first.setZOrder(z); } @@ -1369,6 +1515,15 @@ template inline void addAttributeByIndex(AttributeStore& store, Attr // the key index and vector_layers bookkeeping served from the key cache. template void OsmLuaProcessing::attributeFast(const char *key, size_t keyLen, T val, char minzoom, uint type) { + if (memoTrace.active) { + MemoOp op{ static_cast(MemoOp::FastStr + type), false, minzoom, 0, std::string(key, keyLen), std::string() }; + if constexpr (std::is_same::value) op.str.assign(val.data(), val.size()); + else if constexpr (std::is_same::value) op.flag = val; + else op.number = static_cast(val); + if constexpr (std::is_same::value) memoStringAttribute(std::move(op)); + else if constexpr (std::is_same::value) memoOp(std::move(op)); + else memoNumberAttribute(std::move(op)); + } if (outputs.size()==0) { ProcessingError("Can't add Attribute if no Layer set"); return; } AttrKeyCacheEntry& e = lookupAttrKey(attributeStore.keyStore, key, keyLen); removeAttributeIfNeeded(e.keyIndex); @@ -1460,7 +1615,8 @@ bool OsmLuaProcessing::setNode(NodeID id, LatpLon node, const TagMap& tags) { //Start Lua processing for node try { - if (!callLuaGlobal("node_function", true)) luaState["node_function"](); + if (nodeMemoEnabled) { memoSelect(true); runWayFunction(); } + else if (!callLuaGlobal("node_function", true)) luaState["node_function"](); } catch(luaProcessingException &e) { std::cerr << "Lua error on node " << originalOsmID << std::endl; exit(1); @@ -1510,11 +1666,8 @@ bool OsmLuaProcessing::setWay(WayID wayId, LatpLonVec const &llVec, const TagMap if (ok) { //Start Lua processing for way try { - if (!callLuaGlobal("way_function", false)) { - kaguya::LuaFunction way_function = luaState["way_function"]; - kaguya::LuaRef ret = way_function(); - assert(!ret); - } + memoSelect(false); + runWayFunction(); } catch(luaProcessingException &e) { std::cerr << "Lua error on way " << originalOsmID << std::endl; exit(1); @@ -1561,7 +1714,8 @@ void OsmLuaProcessing::setRelation( if (!isNativeMP && !supportsWritingRelations) return; try { if (isNativeMP && supportsWritingWays) { - if (!callLuaGlobal("way_function", true)) luaState["way_function"](); + memoSelect(MemoRelations); + runWayFunction(); } else if (!isNativeMP && supportsWritingRelations) { if (!callLuaGlobal("relation_function", true)) luaState["relation_function"](); } @@ -1610,7 +1764,22 @@ std::vector &OsmLuaProcessing::finalizeOutputs() { size_t nextDeclutter = 0; for (size_t i = 0; i < this->outputs.size(); i++) { auto& output = this->outputs[i]; - output.first.setAttributeSet(attributeStore.add(output.second)); + if (i < outputPreset.size() && outputPreset[i] != UINT32_MAX) { + output.first.setAttributeSet(outputPreset[i]); + attributeStore.noteLookup(); + } else { + const AttributeIndex index = attributeStore.add(output.second); + output.first.setAttributeSet(index); + if (i < outputLearn.size() && outputLearn[i] != nullptr) { + MemoOp &learn = *outputLearn[i]; + if (learn.segment & MemoOp::SegmentConstant) learn.segmentSet = index; + else { + char sig[MemoSigCache::SIG_MAX]; + const size_t len = memoSegmentSig(learn, sig); + memoSegmentSets.put(MemoSigCache::hashOf(sig, len), sig, len, index); + } + } + } if (nextDeclutter < declutterOutputs.size() && declutterOutputs[nextDeclutter].outputIndex == i) { // Park it: Declutter::apply() will set its minimum zoom and index it later const auto& pending = declutterOutputs[nextDeclutter++]; @@ -1621,3 +1790,522 @@ std::vector &OsmLuaProcessing::finalizeOutputs() { } return list; } + +// ---- Memo of way_function (see osm_lua_processing.h) + +// A node branching more often than this reads something that is different for most ways (a name, +// a house number): from there on the ways run through Lua. Values such as building heights or +// rare building types are many, but each comes often enough to pay for its path (planet: 256 +// replays ~60% of the ways 64 left to Lua; more gains little). At most 65535. +static const size_t MEMO_MAX_CHILDREN = 256; +// per thread +static const size_t MEMO_MAX_NODES = 1 << 18; +// A full tree is compacted (memoCompact) after this many objects it had no room for +static const size_t MEMO_COMPACT_MISSES = std::max(MEMO_MAX_NODES / 64, 16); + +void OsmLuaProcessing::printMemoStats() { + std::lock_guard lock(memoInstancesMutex); + uint64_t counts[MEMO_KINDS][4]; + for (int k = 0; k < MEMO_KINDS; k++) for (int i = 0; i < 4; i++) counts[k][i] = memoDone[k][i]; + for (auto *p : memoInstances) p->memoCounts(counts); + const uint64_t *way = counts[MemoWays], *node = counts[MemoNodes], *rel = counts[MemoRelations]; + if (way[0] + way[1] + way[2] != 0) + std::cout << "way_function memo: " << way[0] << " replayed, " << way[1] << " recorded, " << way[2] << " through Lua, " << way[3] << " nodes" << std::endl; + if (node[0] + node[1] + node[2] != 0) + std::cout << "node_function memo: " << node[0] << " replayed, " << node[1] << " recorded, " << node[2] << " through Lua, " << node[3] << " nodes" << std::endl; + if (rel[0] + rel[1] + rel[2] != 0) + std::cout << "way_function memo (multipolygons): " << rel[0] << " replayed, " << rel[1] << " recorded, " << rel[2] << " through Lua, " << rel[3] << " nodes" << std::endl; +} + +void OsmLuaProcessing::runWayFunctionInLua(bool record) { + if (record) { + memoTrace.luaOnly = false; + memoTrace.ambiguous = false; + memoTrace.reads.clear(); + memoTrace.ops.clear(); + memoTrace.emptyValues.clear(); + memoTrace.values.clear(); + } + memoTrace.active = record; + if (memoForNodes) { + // as setNode called it before the memo + if (!callLuaGlobal("node_function", true)) luaState["node_function"](); + } else if (!callLuaGlobal("way_function", false)) { + kaguya::LuaFunction way_function = luaState["way_function"]; + kaguya::LuaRef ret = way_function(); + assert(!ret); + } + memoTrace.active = false; + if (record) memoInsert(); +} + +// The walk reads the way's tags through memoSlots (filled once per way from the tags as added, +// see memo_keys.h), which answers as currentTags->getKey/getValueFromKey would. +void OsmLuaProcessing::runWayFunction() { + if (!memoEnabled) { runWayFunctionInLua(false); return; } + if (memoNodes.empty()) memoNodes.emplace_back(); + const bool canRecord = memoNodes.size() < MEMO_MAX_NODES; + for (size_t id = memoKeys.valueOnly.size(); id < memoKeys.size(); id++) + memoKeys.valueOnly.push_back(memoValueTag(memoKeys.name(id).data(), memoKeys.name(id).size())); + // IsClosed() as Lua sees it: false for a node + const bool closed = IsClosed(), hasRelations = !relationList.empty(); + char sig[MemoSigCache::SIG_MAX]; + const size_t sigLen = memoSlots.fill(memoKeys, currentTags->added(), uint8_t(closed) | uint8_t(hasRelations) << 1, sig, sizeof(sig)); + const uint64_t sigHash = sigLen ? MemoSigCache::hashOf(sig, sigLen) : 0; + uint32_t last = 0; + uint32_t n = sigLen ? memoWalkCache.find(sigHash, sig, sigLen) : UINT32_MAX; + if (n == UINT32_MAX) { + // The reads that aren't tags, as rawNextRelation, rawMercatorArea, rawFindInRelation record + // them. A walk that made one isn't a function of the signature: it isn't cached. + struct Extra { + OsmLuaProcessing &p; + bool uncached = false; + int relation = -1; // as relationSubscript + std::string result; + std::string_view nextRelation() { + if (!p.memoRelations) return "1"; // such a way runs through Lua: no reads after this one + uncached = true; + if (size_t(++relation) >= p.relationList.size()) return "0"; + result = "1" + p.osmStore.scannedRelations.getRole(p.relationList[relation].second); + return result; + } + std::string_view area() { + uncached = true; + result = p.memoAreaResult(p.MercatorArea()); + return result; + } + std::string_view findInRelation(uint32_t key) { + uncached = true; + if (relation < 0 || size_t(relation) >= p.relationList.size()) return "\xff"; // not recorded: no child + result = p.osmStore.scannedRelations.get_relation_tag(p.relationList[relation].first, p.memoKeys.name(key)); + return result; + } + } extra{ *this }; + n = memoWalk(memoNodes.data(), memoSlots, closed, hasRelations, [this](uint32_t k, std::string_view r) { + const MemoHotNode &node = memoNodes[k]; + return node.more == UINT32_MAX ? UINT32_MAX : memoMore[node.more].child(r); + }, extra, last); + if (sigLen && !extra.uncached && n != UINT32_MAX && (memoNodes[n].kind == MemoNode::Leaf || memoNodes[n].kind == MemoNode::Lua)) + memoWalkCache.put(sigHash, sig, sigLen, n); + } + if (n == UINT32_MAX && memoNodes[last].overflow) { memoLua++; runWayFunctionInLua(false); return; } + if (n != UINT32_MAX && memoNodes[n].kind == MemoNode::Leaf) { + MemoMore &leaf = memoMore[memoNodes[n].more]; + if (leaf.uses != UINT32_MAX) leaf.uses++; + memoHits++; memoReplay(leaf.ops); return; + } + if (n != UINT32_MAX && memoNodes[n].kind == MemoNode::Lua) { + MemoMore &end = memoMoreOf(n); + if (end.uses != UINT32_MAX) end.uses++; + memoLua++; runWayFunctionInLua(false); return; + } + // no child for the result read, or Unset: record, if the tree has room + memoRecorded++; + if (!canRecord && ++memoFullMisses >= MEMO_COMPACT_MISSES) { + memoCompact(); + runWayFunctionInLua(memoNodes.size() < MEMO_MAX_NODES); + return; + } + runWayFunctionInLua(canRecord); +} + +// The tree is full, and objects keep taking paths it hasn't got: keep only the paths that objects +// have taken since the last time (Leaf and Lua nodes used, and the reads to them), so that the +// paths of the objects to come can be recorded. The tree then is as if those paths had been +// recorded alone (in the same order). +void OsmLuaProcessing::memoCompact() { + const size_t size = memoNodes.size(); + std::vector remap(size, UINT32_MAX); + auto keptChild = [&](uint32_t c) { return c != UINT32_MAX && remap[c] != UINT32_MAX; }; + // A child has a higher index than its parent: going backwards, the children are decided + std::vector keep(size, 0); + for (size_t i = size; i-- > 0; ) { + const MemoHotNode &node = memoNodes[i]; + bool k = i == 0; + if (node.kind == MemoNode::Leaf || node.kind == MemoNode::Lua) k = k || (node.more != UINT32_MAX && memoMore[node.more].uses > 0); + else if (node.kind != MemoNode::Unset) { + for (int q = 0; q < 4 && !k; q++) k = node.quick[q] != UINT32_MAX && keep[node.quick[q]]; + if (!k && node.more != UINT32_MAX) + for (const auto &c : memoMore[node.more].children) if (keep[c.second]) { k = true; break; } + } + keep[i] = k; + } + uint32_t next = 0; + for (size_t i = 0; i < size; i++) if (keep[i]) remap[i] = next++; + std::vector nodes; + std::vector more; + nodes.reserve(next); + for (size_t i = 0; i < size; i++) { + if (!keep[i]) continue; + const MemoHotNode &old = memoNodes[i]; + MemoHotNode node; + node.kind = old.kind; node.valueOnly = old.valueOnly; node.overflow = old.overflow; node.key = old.key; + node.children = 0; + for (int q = 0; q < 4; q++) + if (keptChild(old.quick[q])) { node.quick[q] = remap[old.quick[q]]; node.children++; } + if (old.more != UINT32_MAX) { + MemoMore &om = memoMore[old.more]; + MemoMore m; + for (const auto &c : om.children) + if (keptChild(c.second)) { m.addChild(c.first, remap[c.second]); node.children++; } + m.ops = std::move(om.ops); + if (!m.children.empty() || !m.ops.empty() || node.kind == MemoNode::Leaf || node.kind == MemoNode::Lua) { + node.more = more.size(); + more.push_back(std::move(m)); + } + } + // An overflowed node that lost children takes new results again (the results used often + // stay: their paths were used, so kept) + if (node.children < MEMO_MAX_CHILDREN) node.overflow = false; + nodes.push_back(node); + } + memoNodes.swap(nodes); + memoMore.swap(more); + memoWalkCache.clear(); + memoComputeJumps(memoNodes.data(), memoNodes.size()); + memoJumpsAt = memoNodes.size(); + memoFullMisses = 0; +} + +// Add the trace just recorded to the tree +void OsmLuaProcessing::memoInsert() { + if (memoTrace.ambiguous && !memoTrace.luaOnly) return; + auto nondeterministic = [&]() { + memoEnabled = false; + if (memoForNodes) std::cerr << "node_memo: node_function read differently for the same input (node " << originalOsmID << "); memo switched off for this thread" << std::endl; + else if (memoKind == MemoRelations) std::cerr << "way_memo: way_function read differently for the same input (relation " << originalOsmID << "); memo switched off for relations on this thread" << std::endl; + else std::cerr << "way_memo: way_function read differently for the same input (way " << originalOsmID << "); memo switched off for this thread" << std::endl; + }; + uint32_t n = 0; + const size_t sizeBefore = memoNodes.size(); + for (const MemoRead &r : memoTrace.reads) { + // key ids: equal ids <=> equal keys (IsClosed, NextRelation, MercatorArea: no key) + const uint32_t key = r.kind == MemoNode::Find || r.kind == MemoNode::Holds || r.kind == MemoNode::FindInRelation ? memoKeys.add(r.key) : MemoKeyTable::NONE; + if (memoNodes[n].kind == MemoNode::Unset) { memoNodes[n].kind = r.kind; memoNodes[n].key = key; memoNodes[n].valueOnly = r.valueOnly; } + else if (memoNodes[n].kind == MemoNode::Lua) return; + else if (memoNodes[n].kind != r.kind || memoNodes[n].key != key || memoNodes[n].valueOnly != r.valueOnly) { nondeterministic(); return; } + uint32_t next = memoChild(n, r.result); + if (next == UINT32_MAX) { + if (memoNodes[n].children >= MEMO_MAX_CHILDREN) { + memoNodes[n].overflow = true; + return; + } + next = memoNodes.size(); + memoNodes.emplace_back(); + memoAddChild(n, r.result, next); + } + n = next; + } + MemoHotNode &end = memoNodes[n]; + if (end.kind == MemoNode::Unset) { + if (memoTrace.luaOnly) end.kind = MemoNode::Lua; + else { + end.kind = MemoNode::Leaf; + std::vector &ops = memoMoreOf(n).ops; + ops = memoTrace.ops; + for (MemoOp &op : ops) if (!op.valueKey.empty()) op.valueKeyId = memoKeys.add(op.valueKey); + } + } else if (end.kind != MemoNode::Lua && (memoTrace.luaOnly || end.kind != MemoNode::Leaf)) { + nondeterministic(); + } + // The jumps of the nodes just added (their children are newer: none yet), and those of all + // nodes when the tree has doubled since the last time (geometrically: O(nodes) in all) and once + // more when it is full, so that older nodes' jumps reach into the paths added since + if (memoNodes.size() >= memoJumpsAt * 2 + 64 || (memoNodes.size() >= MEMO_MAX_NODES && memoJumpsAt < memoNodes.size())) { + memoComputeJumps(memoNodes.data(), memoNodes.size()); + memoJumpsAt = memoNodes.size(); + } else if (memoNodes.size() > sizeBefore) memoComputeJumps(memoNodes.data(), memoNodes.size(), sizeBefore); +} + +// Make the calls way_function made +// +// The calls of one leaf are the same every time, but for the values of way_memo_values tags +// (valueKey) and whether a Layer/Centroid call adds an output (that depends on the geometry). So the +// layer index, key index and pair of an attribute op without a valueKey are worked out once and +// kept in the op (the pair index for the same pair never changes). An output gets the attribute +// ops after its layer op up to the next layer op (its segment), and those of following layer ops +// that add no output. If its segment has no valueKey ops, and it gets no others, its attribute +// set is always the same, and so its AttributeIndex (one per distinct set): the first time it is +// kept in the layer op (segmentSet), and later replays skip the segment's attribute ops and the +// set lookup. Should a later layer op add no output but have attribute ops, the skipped ops are +// made after all (as they were, with outputKeys as Layer left it), and then the later ones. +static inline bool memoIsLayerOp(OsmLuaProcessing::MemoOp::Kind k) { return k == OsmLuaProcessing::MemoOp::Layer || k == OsmLuaProcessing::MemoOp::Centroid; } +static inline bool memoIsAttributeOp(OsmLuaProcessing::MemoOp::Kind k) { return k >= OsmLuaProcessing::MemoOp::Attr && k <= OsmLuaProcessing::MemoOp::FastInteger; } + +void OsmLuaProcessing::memoReplay(std::vector &ops) { + bool skipping = false; // the attribute ops are those of outputs.back()'s segment, whose AttributeIndex is known + for (size_t i = 0; i < ops.size(); i++) { + MemoOp &op = ops[i]; + if (memoIsLayerOp(op.kind)) { + if (op.layerIndex < 0) { + const auto layer = layers.layerMap.find(op.key); + if (layer == layers.layerMap.end()) { Layer(op.key, op.flag); continue; } // throws + op.layerIndex = layer->second; + } + if (!(op.segment & MemoOp::SegmentKnown)) { + uint8_t s = MemoOp::SegmentKnown | MemoOp::SegmentConstant; + size_t j = i + 1; + for (; j < ops.size() && !memoIsLayerOp(ops[j].kind); j++) + if (memoIsAttributeOp(ops[j].kind)) { + s |= MemoOp::SegmentAttributes; + if (!ops[j].valueKey.empty()) s &= ~MemoOp::SegmentConstant; + } + op.segment = s; + op.segmentOps = j - i - 1; + op.segmentId = ++memoSegmentIds; + } + const size_t before = outputs.size(); + if (op.kind == MemoOp::Layer) layerAt(op.layerIndex, op.flag); + else layerAsCentroidAt(op.layerIndex, static_cast(static_cast(op.number)), {}); + if (outputs.size() > before) { + outputPreset.resize(before, UINT32_MAX); outputLearn.resize(before, nullptr); outputSegment.resize(before, UINT32_MAX); + const bool constant = op.segment & MemoOp::SegmentConstant; + AttributeIndex preset = constant ? op.segmentSet : UINT32_MAX; + bool learn = constant && preset == UINT32_MAX; + if (!constant && (op.segment & MemoOp::SegmentAttributes) && op.segmentId != 0) { + // its set is a function of the segment and the values of its way_memo_values tags + char sig[MemoSigCache::SIG_MAX]; + const size_t len = memoSegmentSig(op, sig); + if (len) { + preset = memoSegmentSets.find(MemoSigCache::hashOf(sig, len), sig, len); + learn = preset == UINT32_MAX; + } + } + skipping = preset != UINT32_MAX; + outputPreset.push_back(preset); + outputLearn.push_back(learn ? &op : nullptr); + outputSegment.push_back(i); + } else { + // This segment's attribute ops go to the output before, if any + if ((op.segment & MemoOp::SegmentAttributes) && !outputs.empty() && outputPreset.size() == outputs.size()) { + if (outputPreset.back() != UINT32_MAX) memoSegmentAttributes(ops, outputSegment.back()); + outputPreset.back() = UINT32_MAX; + outputLearn.back() = nullptr; + } + skipping = false; + } + continue; + } + if (memoIsAttributeOp(op.kind)) { + if (skipping) continue; + memoAttribute(op, memoOpValue(op)); + continue; + } + switch (op.kind) { + case MemoOp::MinZoom: MinZoom(op.number); break; + case MemoOp::ZOrder: ZOrder(op.number); break; + case MemoOp::Score: Score(op.number); break; + case MemoOp::ModifyId: ModifyId(static_cast(op.number)); break; + case MemoOp::MinZoomByArea: MinZoomByArea(op.number, op.numbers); break; + default: break; + } + } +} + +// The skipped attribute ops of the segment of layer op `layerOp`, onto outputs.back(), as made then +// (Layer had cleared outputKeys), and outputKeys as the layer op since has left it +void OsmLuaProcessing::memoSegmentAttributes(std::vector &ops, size_t layerOp) { + outputKeys.clear(); + for (size_t j = layerOp + 1; j < ops.size() && !memoIsLayerOp(ops[j].kind); j++) + if (memoIsAttributeOp(ops[j].kind)) memoAttribute(ops[j], memoOpValue(ops[j])); + outputKeys.clear(); +} + +// The value of an attribute op: its string, or for a valueKey op the current object's value of the +// tag, as currentTags->getKey/getValueFromKey (the leaf's value keys are in memoKeys since before +// the walk). It is copied if the attribute store keeps it. +protozero::data_view OsmLuaProcessing::memoOpValue(const MemoOp &op) const { + if (op.valueKey.empty()) return protozero::data_view(op.str.data(), op.str.size()); + const std::string_view v = memoSlots.get(op.valueKeyId); + return v.empty() ? protozero::data_view("", 0) : protozero::data_view(v.data(), v.size()); +} + +// The signature of the attribute set of an output of layer op `layerOp`, for a segment with valueKey +// ops: the segment's id, then the value of each of its valueKey ops (its length, its bytes). The +// other attribute ops are the same each time, so equal signatures mean equal sets. 0: too long. +size_t OsmLuaProcessing::memoSegmentSig(const MemoOp &layerOp, char *sig) const { + size_t len = 0; + memcpy(sig, &layerOp.segmentId, sizeof(layerOp.segmentId)); len += sizeof(layerOp.segmentId); + const MemoOp *ops = &layerOp + 1; + for (uint32_t k = 0; k < layerOp.segmentOps; k++) { + const MemoOp &op = ops[k]; + if (!memoIsAttributeOp(op.kind) || op.valueKey.empty()) continue; + const std::string_view v = memoSlots.get(op.valueKeyId); + if (v.size() >= 256 || len + 1 + v.size() > MemoSigCache::SIG_MAX) return 0; + sig[len++] = char(v.size()); + memcpy(sig + len, v.data(), v.size()); + len += v.size(); + } + return len; +} + +// An attribute op of the replay: as Attribute*()/attributeFast(), with the key index, the pair +// (no valueKey) and the vector_layers bookkeeping kept in the op +void OsmLuaProcessing::memoAttribute(MemoOp &op, const protozero::data_view &str) { + if (outputs.size()==0) { ProcessingError("Can't add Attribute if no Layer set"); return; } + if (op.keyIndex < 0) op.keyIndex = attributeStore.keyStore.key2index(op.key); + removeAttributeIfNeeded(op.keyIndex); + AttributeSet &set = outputs.back().second; + uint type = 0; + switch (op.kind) { + case MemoOp::Attr: case MemoOp::FastStr: + type = 0; + if (!op.valueKey.empty()) { attributeStore.addAttributeWithKeyIndex(set, op.keyIndex, op.key, str, op.minzoom); break; } + if (op.pairIndex == 0) op.pairIndex = attributeStore.pairIndexWithKeyIndex(op.keyIndex, op.key, str, op.minzoom); + set.addPair(op.pairIndex); + break; + case MemoOp::AttrNumeric: case MemoOp::FastNumeric: + type = 1; + if (op.valueLength) { attributeStore.addAttributeWithKeyIndex(set, op.keyIndex, static_cast(str.size()), op.minzoom); break; } + if (op.pairIndex == 0) op.pairIndex = attributeStore.pairIndexWithKeyIndex(op.keyIndex, op.number, op.minzoom); + set.addPair(op.pairIndex); + break; + case MemoOp::AttrBoolean: case MemoOp::FastBoolean: + type = 2; + if (op.pairIndex == 0) op.pairIndex = attributeStore.pairIndexWithKeyIndex(op.keyIndex, op.flag, op.minzoom); + set.addPair(op.pairIndex); + break; + default: // AttrInteger, FastInteger + type = 3; + if (op.valueLength) { attributeStore.addAttributeWithKeyIndex(set, op.keyIndex, static_cast(str.size()), op.minzoom); break; } + if (op.pairIndex == 0) op.pairIndex = attributeStore.pairIndexWithKeyIndex(op.keyIndex, static_cast(op.number), op.minzoom); + set.addPair(op.pairIndex); + break; + } + const uint_least8_t layer = outputs.back().first.layer; + if (op.seenLayer != layer) { + setVectorLayerMetadata(layer, op.key, type); + op.seenLayer = layer; + } +} + +void OsmLuaProcessing::memoStringAttribute(MemoOp &&op) { + if (!memoTrace.values.empty() && !op.str.empty()) { + // Which way_memo_values tags has this value? Exactly one, and this attribute one of its: a placeholder + const std::pair *source = nullptr; + int matches = 0; + for (const auto &v : memoTrace.values) if (v.second == op.str) { source = &v; matches++; } + if (matches > 0) { + const auto it = memoValueTags.find(source->first); + const bool declared = matches == 1 && std::find(it->second.begin(), it->second.end(), op.key) != it->second.end(); + if (!declared) { memoTrace.ambiguous = true; return; } + op.valueKey = source->first; + op.str.clear(); + memoOp(std::move(op)); + return; + } + } + // An empty declared attribute: a copy of a tag it is declared for, read empty (as for every + // object that replays this, so the constant is right) + if (op.str.empty()) + for (const auto &e : memoTrace.emptyValues) { + const auto it = memoValueTags.find(e); + if (it != memoValueTags.end() && std::find(it->second.begin(), it->second.end(), op.key) != it->second.end()) { memoOp(std::move(op)); return; } + } + // A declared attribute that isn't a plain copy: the profile does more with the value than declared + for (const auto &t : memoValueTags) + if (std::find(t.second.begin(), t.second.end(), op.key) != t.second.end()) { memoLuaOnly(); return; } + memoOp(std::move(op)); +} + +// { tag = { "attribute", ..., attribute = "length", ... }, ... }: the attributes the tag's value is +// copied into, and (lengths: attribute -> tag) those that are set to its length +void OsmLuaProcessing::memoNumberAttribute(MemoOp &&op) { + if (!memoValueLengths.empty()) { + const auto it = memoValueLengths.find(op.key); + if (it != memoValueLengths.end()) { + // declared as the length of a tag: a placeholder if it is that (else the profile does more than declared) + for (const auto &v : memoTrace.values) + if (v.first == it->second && static_cast(v.second.size()) == op.number) { + op.valueKey = v.first; + op.valueLength = true; + op.number = 0; + memoOp(std::move(op)); + return; + } + // the length of the tag read empty (as for every object that replays this): a constant + if (op.number == 0 && std::find(memoTrace.emptyValues.begin(), memoTrace.emptyValues.end(), it->second) != memoTrace.emptyValues.end()) { + memoOp(std::move(op)); + return; + } + memoLuaOnly(); + return; + } + } + memoOp(std::move(op)); +} + +// The comparisons of the area with the memo_area numbers: per number '0' (<), '1' (>), '2' (==), '3' (NaN) +std::string OsmLuaProcessing::memoAreaResult(double area) const { + std::string r(memoArea.size(), '3'); + for (size_t i = 0; i < memoArea.size(); i++) + r[i] = area < memoArea[i] ? '0' : area > memoArea[i] ? '1' : area == memoArea[i] ? '2' : '3'; + return r; +} + +// The length of the table at idx (the declarations are plain tables: no __len). luaL_len is Lua 5.2+; +// Lua 5.1 and LuaJIT have lua_objlen. +static lua_Integer memoTableLength(lua_State *L, int idx) { +#if LUA_VERSION_NUM >= 502 + return luaL_len(L, idx); +#else + return static_cast(lua_objlen(L, idx)); +#endif +} + +static std::unordered_map> readMemoValueTags(lua_State *L, const char *global, std::unordered_map &lengths) { + std::unordered_map> tags; + lua_getglobal(L, global); + if (lua_istable(L, -1)) { + lua_pushnil(L); + while (lua_next(L, -2) != 0) { + if (lua_type(L, -2) == LUA_TSTRING && lua_istable(L, -1)) { + const std::string tag = lua_tostring(L, -2); + std::vector &attrs = tags[tag]; + const lua_Integer n = memoTableLength(L, -1); + for (lua_Integer i = 1; i <= n; i++) { + lua_rawgeti(L, -1, i); + if (lua_type(L, -1) == LUA_TSTRING) attrs.push_back(lua_tostring(L, -1)); + lua_pop(L, 1); + } + lua_pushnil(L); + while (lua_next(L, -2) != 0) { + if (lua_type(L, -2) == LUA_TSTRING && lua_type(L, -1) == LUA_TSTRING && std::string(lua_tostring(L, -1)) == "length") { + lua_pushvalue(L, -2); // lua_tostring on a copy: lua_next needs the key unchanged + lengths[lua_tostring(L, -1)] = tag; + lua_pop(L, 1); + } + lua_pop(L, 1); + } + } + lua_pop(L, 1); + } + } + lua_pop(L, 1); + return tags; +} + +// { number, ... }: the numbers way_function compares MercatorArea() with. An integer that a double +// can't hold exactly would compare differently in Lua: then none (MercatorArea() runs through Lua). +static std::vector readMemoArea(lua_State *L, const char *global) { + std::vector area; + lua_getglobal(L, global); + if (lua_istable(L, -1)) { + const lua_Integer n = memoTableLength(L, -1); + for (lua_Integer i = 1; i <= n; i++) { + lua_rawgeti(L, -1, i); + bool ok = lua_type(L, -1) == LUA_TNUMBER; +#if LUA_VERSION_NUM >= 503 + if (ok && lua_isinteger(L, -1)) { + const lua_Integer v = lua_tointeger(L, -1); + ok = v >= -(lua_Integer(1) << 53) && v <= (lua_Integer(1) << 53); + } +#endif + if (ok) area.push_back(lua_tonumber(L, -1)); + else { std::cerr << global << ": entry " << i << " isn't a number a double holds exactly; ignored (MercatorArea() runs through Lua)" << std::endl; area.clear(); lua_pop(L, 1); break; } + lua_pop(L, 1); + } + } + lua_pop(L, 1); + return area; +} diff --git a/src/tag_map.cpp b/src/tag_map.cpp index cc63958f..19518318 100644 --- a/src/tag_map.cpp +++ b/src/tag_map.cpp @@ -7,6 +7,9 @@ TagMap::TagMap() { } void TagMap::reset() { + tags.clear(); + if (!built) return; + built = false; // Only clear shards that were used since the last reset. for (unsigned i = 0, mask = usedKeyShards; mask; i++, mask >>= 1) if (mask & 1) @@ -18,10 +21,6 @@ void TagMap::reset() { usedValueShards = 0; } -bool TagMap::empty() const { - return usedKeyShards == 0; -} - static inline std::size_t hashString(const char* str, size_t size) { // This is a pretty crappy hash function in terms of bit // avalanching and distribution of output values. @@ -38,7 +37,7 @@ static inline std::size_t hashString(const char* str, size_t size) { return hash; } -uint32_t TagMap::ensureKey(const protozero::data_view& key) { +uint32_t TagMap::ensureKey(const protozero::data_view& key) const { const uint16_t shard = hashString(key.data(), key.size()) & (SHARDS - 1); std::vector& bucket = keys[shard]; const uint32_t size = key.size(); @@ -51,7 +50,7 @@ uint32_t TagMap::ensureKey(const protozero::data_view& key) { return shard << 16 | (bucket.size() - 1); } -uint32_t TagMap::ensureValue(const protozero::data_view& value) { +uint32_t TagMap::ensureValue(const protozero::data_view& value) const { const uint16_t shard = hashString(value.data(), value.size()) & (SHARDS - 1); std::vector& bucket = values[shard]; for (size_t i = 0; i < bucket.size(); i++) @@ -63,7 +62,13 @@ uint32_t TagMap::ensureValue(const protozero::data_view& value) { return shard << 16 | (bucket.size() - 1); } -void TagMap::addTag(const protozero::data_view& key, const protozero::data_view& value) { +void TagMap::build() const { + built = true; + for (const Tag& tag : tags) + insert(tag.key, tag.value); +} + +void TagMap::insert(const protozero::data_view& key, const protozero::data_view& value) const { uint32_t valueLoc = ensureValue(value); uint32_t keyLoc = ensureKey(key); @@ -72,6 +77,7 @@ void TagMap::addTag(const protozero::data_view& key, const protozero::data_view& int64_t TagMap::getKey(const char* key, size_t size) const { // Return -1 if key not found, else return its keyLoc. + ensureBuilt(); const uint16_t shard = hashString(key, size) & (SHARDS - 1); const std::vector& bucket = keys[shard]; for (size_t i = 0; i < bucket.size(); i++) { @@ -87,6 +93,7 @@ int64_t TagMap::getKey(const char* key, size_t size) const { int64_t TagMap::getValue(const char* value, size_t size) const { // Return -1 if value not found, else return its valueLoc. + ensureBuilt(); const uint16_t shard = hashString(value, size) & (SHARDS - 1); const std::vector& bucket = values[shard]; for (size_t i = 0; i < bucket.size(); i++) { @@ -101,16 +108,19 @@ int64_t TagMap::getValue(const char* value, size_t size) const { } const protozero::data_view* TagMap::getValueFromKey(uint32_t keyLoc) const { + ensureBuilt(); const uint32_t valueLoc = keys[keyLoc >> 16][keyLoc & 0xFFFF].valueLoc; return &values[valueLoc >> 16][valueLoc & 0xFFFF]; } const protozero::data_view* TagMap::getValue(uint32_t valueLoc) const { + ensureBuilt(); return &values[valueLoc >> 16][valueLoc & 0xFFFF]; } boost::container::flat_map TagMap::exportToBoostMap() const { boost::container::flat_map rv; + ensureBuilt(); for (size_t i = 0; i < keys.size(); i++) { for (size_t j = 0; j < keys[i].size(); j++) { @@ -124,6 +134,7 @@ boost::container::flat_map TagMap::exportToBoostMap() } TagMap::Iterator TagMap::begin() const { + ensureBuilt(); size_t shard = 0; while(keys.size() > shard && keys[shard].size() == 0) shard++; diff --git a/src/tilemaker.cpp b/src/tilemaker.cpp index f118840d..30e8d93c 100644 --- a/src/tilemaker.cpp +++ b/src/tilemaker.cpp @@ -510,6 +510,8 @@ int main(const int argc, const char* argv[]) { // Cluster tiles: breadth-first for z0..z5, depth-first for z6 sortTileCoordinates(config.baseZoom, options.threadNum, tileCoordinates); + OsmLuaProcessing::printMemoStats(); + // Shared pieces of large polygons, for the SHARED_PIECE_ZOOM tiles below which a tile is written { const unsigned pieceZoom = TileDataSource::SHARED_PIECE_ZOOM;