diff --git a/Makefile b/Makefile index 57227d4c..b03576c2 100644 --- a/Makefile +++ b/Makefile @@ -156,13 +156,16 @@ test: \ test_append_vector \ test_attribute_store \ test_config_validator \ + test_correct \ test_deque_map \ test_geom \ test_helpers \ test_options_parser \ + test_ordered_batches \ test_pbf_reader \ test_pooled_string \ test_relation_roles \ + test_shp_mem_tiles \ test_significant_tags \ test_sorted_node_store \ test_sorted_way_store \ @@ -193,6 +196,10 @@ src/config_schema.h: resources/config-schema.json src/config_validator.o: src/config_schema.h +test_correct: \ + test/correct.test.o + $(CXX) $(CXXFLAGS) -o test.correct $^ $(INC) $(LIB) $(LDFLAGS) && ./test.correct + test_deque_map: \ test/deque_map.test.o $(CXX) $(CXXFLAGS) -o test.deque_map $^ $(INC) $(LIB) $(LDFLAGS) && ./test.deque_map @@ -223,6 +230,10 @@ test_options_parser: \ test/options_parser.test.o $(CXX) $(CXXFLAGS) -o test.options_parser $^ $(INC) $(LIB) $(LDFLAGS) && ./test.options_parser +test_ordered_batches: \ + test/ordered_batches.test.o + $(CXX) $(CXXFLAGS) -o test.ordered_batches $^ $(INC) $(LIB) $(LDFLAGS) && ./test.ordered_batches + test_osm_store: \ test/osm_store.test.o $(CXX) $(CXXFLAGS) -o test.osm_store $^ $(INC) $(LIB) $(LDFLAGS) && ./test.osm_store @@ -262,6 +273,22 @@ test_sorted_way_store: \ test/sorted_way_store.test.o $(CXX) $(CXXFLAGS) -o test.sorted_way_store $^ $(INC) $(LIB) $(LDFLAGS) && ./test.sorted_way_store +test_shp_mem_tiles: \ + src/attribute_store.o \ + src/coordinates.o \ + src/coordinates_geom.o \ + src/declutter.o \ + src/geom.o \ + src/mmap_allocator.o \ + src/output_object.o \ + src/pooled_string.o \ + src/shp_mem_tiles.o \ + src/tile_coordinates_set.o \ + src/tile_data.o \ + src/tile_sorting.o \ + test/shp_mem_tiles.test.o + $(CXX) $(CXXFLAGS) -o test.shp_mem_tiles $^ $(INC) $(LIB) $(LDFLAGS) && ./test.shp_mem_tiles + test_tile_coordinates_set: \ src/tile_coordinates_set.o \ test/tile_coordinates_set.test.o diff --git a/include/append_vector.h b/include/append_vector.h index 28b29a8b..8a66add9 100644 --- a/include/append_vector.h +++ b/include/append_vector.h @@ -162,6 +162,10 @@ namespace AppendVectorNS { return el; } + const T& operator [](int idx) const { + return vecs[idx / APPEND_VECTOR_SIZE][idx % APPEND_VECTOR_SIZE]; + } + Iterator begin() { return Iterator(*this, 0, 0); } diff --git a/include/attribute_store.h b/include/attribute_store.h index 295c495c..849b3889 100644 --- a/include/attribute_store.h +++ b/include/attribute_store.h @@ -9,6 +9,8 @@ #include #include #include +#include +#include #include #include #include "pooled_string.h" @@ -30,11 +32,16 @@ class AttributeKeyStore { uint16_t key2index(const std::string& key); const std::string& getKey(uint16_t index) const; const std::string& getKeyUnsafe(uint16_t index) const; - void finalize() { finalized = true; } + // Position of the key in alphabetical order. Unlike the key index, which is + // handed out in first-use order (i.e. depends on thread scheduling), it is + // the same in every run. Only valid after finalize(). + uint16_t getRankUnsafe(uint16_t index) const { return ranks[index]; } + void finalize(); std::atomic keys2indexSize; private: bool finalized; + std::vector ranks; mutable std::mutex keys2indexMutex; // NB: we use a deque, not a vector, because a deque never invalidates // pointers to its members as long as you only push_back @@ -439,6 +446,24 @@ struct AttributeStore { } attrSet->forEachPair(fn); } + // As forEachPairIndexUnsafe, but in an order that is the same in every run + // (by key name), not by pair index, which depends on thread scheduling. + template + void forEachPairIndexCanonicalUnsafe(AttributeIndex index, F&& fn) const { + boost::container::small_vector order; + forEachPairIndexUnsafe(index, [&](uint32_t pairIndex) { order.push_back(pairIndex); }); + if (order.size() > 1) + std::sort(order.begin(), order.end(), [this](uint32_t a, uint32_t b) { + return comparePairsUnsafe(a, b) < 0; + }); + for (const uint32_t pairIndex : order) + fn(pairIndex); + } + // Order and hash attribute pairs/sets by content (key names and values), not by + // index, so that they are the same in every run. Only valid after finalize(). + int comparePairsUnsafe(uint32_t a, uint32_t b) const; + int compareContentUnsafe(AttributeIndex a, AttributeIndex b) const; + uint64_t contentHashUnsafe(AttributeIndex index) const; void reset(); // used for testing size_t size() const; void reportSize() const; diff --git a/include/clip_cache.h b/include/clip_cache.h index 66a400ab..e650671a 100644 --- a/include/clip_cache.h +++ b/include/clip_cache.h @@ -16,6 +16,9 @@ class TileBbox; extern thread_local uint64_t clipCacheBatchId; // Set when the current batch can't reuse clips (a single tile), so add() is skipped. extern thread_local bool clipCacheNoStore; +// Set while a shared piece of a large polygon is computed (TileDataSource::precomputeSharedPieces): +// the batch cache is then neither read nor written. +extern thread_local bool clipCacheBypass; template class ClipCache { @@ -36,6 +39,8 @@ class ClipCache { } Entry getEntry(uint zoom, TileCoordinate x, TileCoordinate y, NodeID objectID) const { + if (clipCacheBypass) + return Entry(); // Look for a previously clipped version at z-1, z-2, ... (nearest ancestor first) LocalStore& s = local(zoom, TileCoordinates(x, y)); Entry rv; @@ -59,7 +64,7 @@ class ClipCache { void add(const TileBbox& bbox, const NodeID objectID, const T& output, bool valid = false) { // The point of caching is to reuse the clip, so caching at the terminal zoom is // pointless. - if (bbox.zoom == baseZoom || clipCacheNoStore) + if (bbox.zoom == baseZoom || clipCacheNoStore || clipCacheBypass) return; // Levels beyond MAX_LEVELS could never be looked up by a child tile anyway diff --git a/include/coordinates_geom.h b/include/coordinates_geom.h index f233ad3e..acf47b9f 100644 --- a/include/coordinates_geom.h +++ b/include/coordinates_geom.h @@ -28,6 +28,7 @@ class TileBbox { std::pair scaleLatpLon(double latp, double lon) const; void scaleRing(Ring &dst, Ring const &src) const; Ring scaleRing(Ring const &src) const; + void scaleRingNoBacktrack(Ring &dst, Ring const &src) const; void scaleGeometry(MultiPolygon &dst, MultiPolygon const &src) const; MultiPolygon scaleGeometry(MultiPolygon const &src) const; std::pair floorLatpLon(double latp, double lon) const; diff --git a/include/geometry/correct.hpp b/include/geometry/correct.hpp index fba96c69..b26c74af 100644 --- a/include/geometry/correct.hpp +++ b/include/geometry/correct.hpp @@ -10,6 +10,8 @@ * ---------------------------------------------------------------------------- */ +#include +#include #include #include #include @@ -23,13 +25,18 @@ namespace geometry { namespace impl { -template -static inline void result_combine(C &result, T &&new_element) +// boxes[i] is the envelope of result[i]. Polygons whose envelopes are disjoint +// cannot intersect, so the box test skips most of the costly intersects() calls +// (otherwise quadratic in the number of rings, e.g. a lake with 14000 islands). +template +static inline void result_combine(C &result, std::vector &boxes, T &&new_element) { result.push_back(std::forward(new_element)); + boxes.push_back(boost::geometry::return_envelope(result.back())); for(std::size_t i = 0; i < result.size() - 1; ) { - if(!boost::geometry::intersects(result[i], result.back())) { + if(boost::geometry::disjoint(boxes[i], boxes.back()) || + !boost::geometry::intersects(result[i], result.back())) { ++i; continue; } @@ -43,8 +50,10 @@ static inline void result_combine(C &result, T &&new_element) } result.back() = std::move(union_result[0]); + boxes.back() = boost::geometry::return_envelope(result.back()); result.erase(result.begin() + i); - } + boxes.erase(boxes.begin() + i); + } } struct pseudo_vertice_key @@ -199,27 +208,40 @@ static inline std::vector dissolve_generate_rings( // Generate all polygons by tracing all the intersections // Perform union to combine all polygons into single polygon again - while(!start_keys.empty()) { - ring_t new_ring; - - // Store point in generated polygon - auto push_point = [&new_ring](auto const &p) { - if(new_ring.empty() || boost::geometry::comparable_distance(new_ring.back(), p) > 0) - new_ring.push_back(p); - }; + // Position of each point in the ring being traced. Its points are distinct until + // it closes, so a point already seen closes it: looking that up here instead of + // scanning the whole ring for every new point keeps tracing linear, not quadratic. + struct point_hash { + std::size_t operator()(std::pair const &p) const { + return std::hash()(p.first) * 31 + std::hash()(p.second); + } + }; + std::unordered_map, std::size_t, point_hash> seen; - auto is_closed = [](ring_t &ring) { - if(ring.size() < 2) return false; - for(std::size_t i = 0; i < ring.size() - 1; ++i) { - if(boost::geometry::comparable_distance(ring[i], ring.back()) == 0) { - ring.erase(ring.begin(), ring.begin() + i); - return true; - } + while(!start_keys.empty()) { + ring_t new_ring; + seen.clear(); + bool closed = false; + + // Store point in generated polygon; when it repeats an earlier point, the + // ring is closed and the points before that one are dropped + auto push_point = [&new_ring, &seen, &closed](auto const &p) { + if(!new_ring.empty() && !(boost::geometry::comparable_distance(new_ring.back(), p) > 0)) + return; + new_ring.push_back(p); + // + 0.0 turns -0.0 into 0.0, which compares equal to it + std::pair key(boost::geometry::get<0>(p) + 0.0, boost::geometry::get<1>(p) + 0.0); + auto found = seen.find(key); + if(found == seen.end()) { + seen.emplace(key, new_ring.size() - 1); + } else { + new_ring.erase(new_ring.begin(), new_ring.begin() + found->second); + closed = true; } - - return false; }; + auto is_closed = [&closed](ring_t &) { return closed; }; + auto start_iter = pseudo_vertices.find(*start_keys.begin()); auto i = start_iter; @@ -302,13 +324,15 @@ template< > struct combine_non_zero_winding { - inline void operator()(multi_polygon_t &combined_outers, multi_polygon_t &combined_inners, polygon_t &poly) + using box_t = boost::geometry::model::box; + + inline void operator()(multi_polygon_t &combined_outers, std::vector &outer_boxes, multi_polygon_t &combined_inners, std::vector &inner_boxes, polygon_t &poly) { if(boost::geometry::area(poly) > 0) - result_combine(combined_outers, std::move(poly)); + result_combine(combined_outers, outer_boxes, std::move(poly)); else { std::reverse(poly.outer().begin(), poly.outer().end()); - result_combine(combined_inners, std::move(poly)); + result_combine(combined_inners, inner_boxes, std::move(poly)); } } }; @@ -320,7 +344,9 @@ template< > struct combine_odd_even { - inline void operator()(multi_polygon_t &combined_outers, multi_polygon_t &combined_inners, polygon_t &poly) + using box_t = boost::geometry::model::box; + + inline void operator()(multi_polygon_t &combined_outers, std::vector &, multi_polygon_t &combined_inners, std::vector &, polygon_t &poly) { if(boost::geometry::area(poly) < 0) std::reverse(poly.outer().begin(), poly.outer().end()); @@ -346,11 +372,12 @@ static inline void correct(polygon_t const &input, multi_polygon_t &output, doub // Calculate all outers and combine them if possible multi_polygon_t combined_outers; multi_polygon_t combined_inners; + std::vector> outer_boxes, inner_boxes; for(auto &ring: outer_rings) { polygon_t poly; poly.outer() = std::move(ring); - combine(combined_outers, combined_inners, poly); + combine(combined_outers, outer_boxes, combined_inners, inner_boxes, poly); } // Calculate all inners and combine them if possible @@ -362,8 +389,8 @@ static inline void correct(polygon_t const &input, multi_polygon_t &output, doub correct(poly, new_inners, remove_spike_min_area, combine); for(auto &poly: new_inners) { - result_combine(combined_inners, std::move(poly)); - } + result_combine(combined_inners, inner_boxes, std::move(poly)); + } } // Cut out all inners from all the outers @@ -379,13 +406,18 @@ template< > static inline void correct(multi_polygon_t const &input, multi_polygon_t &output, double remove_spike_min_area, combine_function_t combine) { + using box_t = boost::geometry::model::box; + std::vector output_boxes; + for(auto const &polygon: output) + output_boxes.push_back(boost::geometry::return_envelope(polygon)); + for(auto const &polygon: input) { multi_polygon_t new_polygons; correct(polygon, new_polygons, remove_spike_min_area, combine); - for(auto &new_polygon: new_polygons) - result_combine(output, std::move(new_polygon)); + for(auto &new_polygon: new_polygons) + result_combine(output, output_boxes, std::move(new_polygon)); } } diff --git a/include/leased_store.h b/include/leased_store.h index 0349d64d..019b110d 100644 --- a/include/leased_store.h +++ b/include/leased_store.h @@ -2,6 +2,7 @@ #define _LEASED_STORE_H #include "tile_data.h" +#include // LeasedStore implements "leases" for generated geometries. // @@ -45,23 +46,26 @@ inline std::vector>& g template class LeasedStore { - std::vector>> leases; + std::vector>> leases; // source, its leaseNumber, lease public: ~LeasedStore() { + // Only to the sources still alive (see registerLeaseSource) + std::lock_guard live(leaseSourcesMutex()); for (const auto& lease : leases) { - auto source = lease.first; + auto source = std::get<0>(lease); + if (!leaseSourceAlive(source, std::get<1>(lease))) continue; std::lock_guard lock(source->storeMutex); std::vector>& availableLeases = getAvailableLeases(source); - availableLeases.push_back(lease.second); + availableLeases.push_back(std::get<2>(lease)); } } std::pair get(TileDataSource* source) { for (const auto& lease : leases) { - if (lease.first == source) - return lease.second; + if (std::get<0>(lease) == source && std::get<1>(lease) == source->leaseNumber) + return std::get<2>(lease); } std::lock_guard lock(source->storeMutex); @@ -74,7 +78,7 @@ class LeasedStore { std::pair entry = availableLeases.back(); availableLeases.pop_back(); - leases.push_back(std::make_pair(source, entry)); + leases.push_back(std::make_tuple(source, source->leaseNumber, entry)); return entry; } diff --git a/include/mbtiles.h b/include/mbtiles.h index ca671b07..8181fafa 100644 --- a/include/mbtiles.h +++ b/include/mbtiles.h @@ -3,9 +3,11 @@ #define _MBTILES_H #include +#include #include #include #include "external/sqlite_modern_cpp.h" +#include "ordered_batches.h" struct PendingStatement { int zoom; @@ -27,6 +29,7 @@ class MBTiles { std::shared_ptr> pendingStatements1, pendingStatements2; std::mutex pendingStatementsMutex; + std::atomic pendingBytes; void insertOrReplace(int zoom, int x, int y, const std::string& data, bool isMerge); void flushPendingStatements(); @@ -39,6 +42,9 @@ class MBTiles { void saveTile(int zoom, int x, int y, std::string *data, bool isMerge); void closeForWriting(); + // Tiles are written in batch order, so the file doesn't depend on thread scheduling + OrderedBatches batches; + void openForReading(std::string &filename); void readBoundingBox(double &minLon, double &maxLon, double &minLat, double &maxLat); void readTileList(std::vector> &tileList); diff --git a/include/ordered_batches.h b/include/ordered_batches.h new file mode 100644 index 00000000..9531565f --- /dev/null +++ b/include/ordered_batches.h @@ -0,0 +1,157 @@ +/*! \file */ +#ifndef _ORDERED_BATCHES_H +#define _ORDERED_BATCHES_H + +#include +#include +#include +#include +#include +#include +#include +#include + +// Writes batches of output in the order they were handed out, not the order threads +// finish them, so the output file doesn't depend on thread scheduling. +// +// A thread collects a batch's items between begin() and end(). end() writes the batch +// if it's the next one, followed by any batches that were waiting for it; otherwise it +// holds the batch. Batches must be numbered 0, 1, 2, ... in the order they're handed out, +// and a thread calls waitForRoom() before it takes its next batch: that keeps what is +// held below the limit set with setMaxHeldBytes(). The batch everyone waits for is always +// being worked on by a thread that isn't waiting, so waitForRoom() always returns. +// +// writeItem is called by one thread at a time, with writeMutex() held, in batch order. +// If flushItem is given, a batch then goes through flushItem as well: also one thread at a time +// and in batch order, but without holding up writeItem. Whichever thread is free does it, so the +// two steps run side by side. A batch counts as held until it has been flushed. +template +class OrderedBatches { +public: + OrderedBatches(std::function writeItem, std::function itemBytes, + std::function flushItem = nullptr) + : writeItem(writeItem), itemBytes(itemBytes), flushItem(flushItem) { } + + void setMaxHeldBytes(uint64_t bytes) { + std::lock_guard lock(m); + maxHeldBytes = bytes; + } + + bool inBatch() const { return current().active; } + + void begin(uint64_t seq) { + Current& c = current(); + c.active = true; + c.seq = seq; + c.items.clear(); + } + + void add(T&& item) { current().items.emplace_back(std::move(item)); } + + // Hand the batch over. If it's the next one to be written and no other thread is + // writing, this thread writes it and then any held batches that follow it; meanwhile + // other threads just hand their batches over and carry on. + void end() { + Current& c = current(); + c.active = false; + uint64_t bytes = 0; + for (const auto& item : c.items) bytes += itemBytes(item); + std::unique_lock lock(m); + heldBytes += bytes; + held.emplace(c.seq, std::make_pair(bytes, std::move(c.items))); + c.items = std::vector(); + if (draining) { + flushQueued(lock); + return; + } + draining = true; + while (!held.empty() && held.begin()->first == nextSeq) { + auto batch = std::move(held.begin()->second); + held.erase(held.begin()); + lock.unlock(); + { + std::lock_guard writeLock(w); + for (auto& item : batch.second) writeItem(item); + } + lock.lock(); + nextSeq++; + if (flushItem) { + toFlush.emplace_back(std::move(batch)); + heldShrunk.notify_one(); // a waiting thread can flush it + continue; + } + heldBytes -= batch.first; + // Wake the waiting threads only once there is room. Woken for every batch + // written, they all found no room and went back to sleep, again and again: + // most of the system time of a 96-thread planet run. + if (heldBytes <= maxHeldBytes) heldShrunk.notify_all(); + } + draining = false; + flushQueued(lock); + } + + void waitForRoom() { + std::unique_lock lock(m); + while (heldBytes > maxHeldBytes) { + if (!toFlush.empty() && !flushing) flushQueued(lock); + else heldShrunk.wait(lock); + } + } + + bool allWritten() { + std::lock_guard lock(m); + return held.empty() && toFlush.empty() && !flushing; + } + + // Held while items are written; lock it to write anything outside a batch + std::mutex& writeMutex() { return w; } + +private: + struct Current { + bool active = false; + uint64_t seq = 0; + std::vector items; + }; + // Flush the batches waiting for flushItem, unless another thread already does; called and + // returns with m held + void flushQueued(std::unique_lock &lock) { + if (!flushItem || flushing) return; + flushing = true; + while (!toFlush.empty()) { + auto batch = std::move(toFlush.front()); + toFlush.pop_front(); + lock.unlock(); + for (auto& item : batch.second) flushItem(item); + batch.second = std::vector(); + lock.lock(); + heldBytes -= batch.first; + if (heldBytes <= maxHeldBytes) heldShrunk.notify_all(); + } + flushing = false; + // batches queued while finishing the last one are taken by the next thread that comes by; + // wake one in case all others wait for room + heldShrunk.notify_one(); + } + + // The batch this thread is working on (one per thread and item type) + static Current& current() { + static thread_local Current c; + return c; + } + + std::function writeItem; + std::function itemBytes; + std::function flushItem; + uint64_t maxHeldBytes = 1ull << 30; + std::mutex m; // guards the fields below + std::mutex w; // held while writing + std::condition_variable heldShrunk; + bool draining = false; + uint64_t nextSeq = 0; + uint64_t heldBytes = 0; + std::map>> held; // seq -> (bytes, items) + std::deque>> toFlush; // written in order, waiting for flushItem + bool flushing = false; // a thread is flushing +}; + +#endif //_ORDERED_BATCHES_H diff --git a/include/osm_mem_tiles.h b/include/osm_mem_tiles.h index 1966bc4b..5ab90fc1 100644 --- a/include/osm_mem_tiles.h +++ b/include/osm_mem_tiles.h @@ -57,6 +57,8 @@ class OsmMemTiles : public TileDataSource { void populateLinestring(Linestring& ls, NodeID objectID) const; Linestring& getOrBuildLinestring(NodeID objectID) const; void populateMultiPolygon(MultiPolygon& dst, NodeID objectID) override; + // Polygons from ways are built from the way store, only relations are in the multipolygon store + bool storesMultiPolygon(NodeID objectID) const override { return objectID < OSM_THRESHOLD; } // Way-store polygons are only emitted by OsmLuaProcessing::Layer when // CorrectGeometry found them Valid (or Reversed: valid after reversing the ring) // on the same lon/latp coordinates. diff --git a/include/osm_store.h b/include/osm_store.h index 4f50cd5f..1f9915ec 100644 --- a/include/osm_store.h +++ b/include/osm_store.h @@ -154,22 +154,31 @@ class RelationScanStore { RelationScanStore(): relationsForWays(128), relationsForNodes(128), relationTags(128), mutex(128) {} + // Keep each member's parent relations ordered by relation id: relations are scanned on + // several threads, so appending would make the order NextRelation() returns depend on + // thread scheduling. Entries of the same relation keep their member order. + static void insertByRelation(std::vector>& list, RelationID relid, uint16_t roleId) { + const auto it = std::upper_bound(list.begin(), list.end(), relid, + [](RelationID id, const std::pair& entry) { return id < entry.first; }); + list.emplace(it, relid, roleId); + } + void relation_contains_way(RelationID relid, WayID wayid, std::string role) { uint16_t roleId = relationRoles.getOrAddRole(role); const size_t shard = wayid % mutex.size(); std::lock_guard lock(mutex[shard]); - relationsForWays[shard][wayid].emplace_back(std::make_pair(relid, roleId)); + insertByRelation(relationsForWays[shard][wayid], relid, roleId); } void relation_contains_node(RelationID relid, NodeID nodeId, std::string role) { uint16_t roleId = relationRoles.getOrAddRole(role); const size_t shard = nodeId % mutex.size(); std::lock_guard lock(mutex[shard]); - relationsForNodes[shard][nodeId].emplace_back(std::make_pair(relid, roleId)); + insertByRelation(relationsForNodes[shard][nodeId], relid, roleId); } void relation_contains_relation(RelationID relid, RelationID relationId, std::string role) { uint16_t roleId = relationRoles.getOrAddRole(role); std::lock_guard lock(mutex[0]); - relationsForRelations[relationId].emplace_back(std::make_pair(relid, roleId)); + insertByRelation(relationsForRelations[relationId], relid, roleId); } void store_relation_tags(RelationID relid, const tag_map_t &tags) { const size_t shard = relid % mutex.size(); diff --git a/include/pmtiles.h b/include/pmtiles.h index 7e8040c8..fb2e6682 100644 --- a/include/pmtiles.h +++ b/include/pmtiles.h @@ -3,7 +3,12 @@ #define _PMTILES_H #include +#include +#include +#include +#include #include "external/pmtiles.hpp" +#include "ordered_batches.h" struct TileOffset { uint64_t offset : 40; @@ -28,6 +33,14 @@ struct TileOffset { // PMTiles v3 tile type for MapLibre Tiles (not yet in external/pmtiles.hpp) #define TILETYPE_MLT 0x6 +// A tile waiting to be written: tiny tiles keep their raw data, so that a repeat +// can reuse the earlier copy, and are only compressed if they're written +struct PendingTile { + uint64_t tileId; + bool isTiny; + std::string data; // after writeTile: the bytes flushTile appends to the file (empty if a repeat) +}; + class PMTiles { public: @@ -42,10 +55,14 @@ class PMTiles { void saveTile(int zoom, int x, int y, std::string &data); void close(std::string &metadata); + // Tiles are written in batch order, so the file doesn't depend on thread scheduling + OrderedBatches batches; + private: std::ofstream outputStream; - std::mutex fileMutex; // guards file writes, numTilesWritten - std::mutex indexMutex; // guards access to sparseIndex, denseIndex, tinyCache, numTilesAddressed + std::vector outputBuffer; // a large stream buffer: one thread writes all tiles + uint64_t tileDataEnd = 0; // bytes of tile data written so far (tellp() costs a flush and a seek per tile) + std::mutex fileMutex; // guards leaf directory writes in close() uint64_t leafStart = 0; uint64_t numTilesWritten = 0; uint64_t numTilesAddressed = 0; @@ -54,6 +71,8 @@ class PMTiles { std::vector denseIndex; std::unordered_map tinyCache; + void writeTile(PendingTile &tile); // with batches.writeMutex() held: places the tile + void flushTile(PendingTile &tile); // in the same order, by one thread at a time: writes its bytes void appendWithRLE(std::vector &entries, pmtiles::entryv3 &entry); void appendTileEntry(uint64_t tileId, TileOffset offset, std::vector &rootEntries, std::vector &entries); void flushEntries(std::vector &rootEntries, std::vector &entries); diff --git a/include/polylabel.h b/include/polylabel.h index 6ce44f16..30de7a6d 100644 --- a/include/polylabel.h +++ b/include/polylabel.h @@ -329,13 +329,13 @@ Cell getCentroidCell(const Polygon& polygon) { return Cell(area == 0 ? ring.at(0) : Point { ox + cx / area, oy + cy / area }, 0, polygon); } -// Upper bound on pointToPolygonDist(p, polygon), from the edge nearest to `near.c`. The full scan +// Upper bound on pointToPolygonDist(p, polygon), from the edge nearest to `nearCell.c`. The full scan // computes min over all edges of getSegDistSq(p, a, b) with the same arguments, so its result is // <= this edge's value; the margin covers FP contraction differences between call sites. -inline double edgeDistBound(const Point& p, const Cell& near, const Polygon& polygon) { - const auto& ring = near.kr < 0 ? polygon.outer() : polygon.inners()[near.kr]; - const Point& a = ring[near.ki]; - const Point& b = ring[near.ki == 0 ? ring.size() - 1 : near.ki - 1]; +inline double edgeDistBound(const Point& p, const Cell& nearCell, const Polygon& polygon) { + const auto& ring = nearCell.kr < 0 ? polygon.outer() : polygon.inners()[nearCell.kr]; + const Point& a = ring[nearCell.ki]; + const Point& b = ring[nearCell.ki == 0 ? ring.size() - 1 : nearCell.ki - 1]; return std::sqrt(getSegDistSq(p, a, b)) * (1 + 1e-12) + 1e-11; } diff --git a/include/shp_mem_tiles.h b/include/shp_mem_tiles.h index a5cf4f3f..0bab561c 100644 --- a/include/shp_mem_tiles.h +++ b/include/shp_mem_tiles.h @@ -27,7 +27,8 @@ class ShpMemTiles : public TileDataSource const std::string& name, uint minzoom, int32_t score, - AttributeIndex attrIdx + AttributeIndex attrIdx, + bool knownValid = false // polygon: caller has checked it is valid ); std::vector QueryMatchingGeometries( diff --git a/include/stable_hash.h b/include/stable_hash.h new file mode 100644 index 00000000..e1af029f --- /dev/null +++ b/include/stable_hash.h @@ -0,0 +1,42 @@ +/*! \file */ +#ifndef _STABLE_HASH_H +#define _STABLE_HASH_H + +#include +#include +#include + +// Hash functions that give the same result on every platform, compiler and +// library version (unlike std::hash or boost::hash), for orderings that the +// output depends on. + +inline uint64_t stableMix(uint64_t x) { + // splitmix64 finaliser + x += 0x9e3779b97f4a7c15ull; + x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9ull; + x = (x ^ (x >> 27)) * 0x94d049bb133111ebull; + return x ^ (x >> 31); +} + +inline uint64_t stableCombine(uint64_t seed, uint64_t value) { + return stableMix(seed ^ stableMix(value)); +} + +inline uint64_t stableCombine(uint64_t seed, double value) { + if (value == 0) value = 0; // -0.0 and 0.0 compare equal, so hash them equally + uint64_t bits; + std::memcpy(&bits, &value, sizeof(bits)); + return stableCombine(seed, bits); +} + +inline uint64_t stableCombine(uint64_t seed, const char* data, std::size_t length) { + // FNV-1a over the bytes + uint64_t h = 0xcbf29ce484222325ull; + for (std::size_t i = 0; i < length; i++) { + h ^= static_cast(data[i]); + h *= 0x100000001b3ull; + } + return stableCombine(stableCombine(seed, static_cast(length)), h); +} + +#endif //_STABLE_HASH_H diff --git a/include/tile_coordinates_set.h b/include/tile_coordinates_set.h index 3d45c50e..21f14649 100644 --- a/include/tile_coordinates_set.h +++ b/include/tile_coordinates_set.h @@ -14,6 +14,8 @@ class TileCoordinatesSet { virtual void set(TileCoordinate x, TileCoordinate y) = 0; virtual size_t size() const = 0; virtual size_t zoom() const = 0; + // Set every tile with minX <= x <= maxX and minY <= y <= maxY. + virtual void setBox(TileCoordinate minX, TileCoordinate maxX, TileCoordinate minY, TileCoordinate maxY); // Visit every set tile in x-major, y-minor order. virtual void forEachSet(const std::function& f) const; }; @@ -30,7 +32,10 @@ class PreciseTileCoordinatesSet : public TileCoordinatesSet { size_t size() const override; size_t zoom() const override; void set(TileCoordinate x, TileCoordinate y) override; + void setBox(TileCoordinate minX, TileCoordinate maxX, TileCoordinate minY, TileCoordinate maxY) override; void forEachSet(const std::function& f) const override; + // Add the tiles set in other, a set of the same zoom. + void merge(const PreciseTileCoordinatesSet& other); private: unsigned int zoom_; @@ -50,6 +55,7 @@ class LossyTileCoordinatesSet : public TileCoordinatesSet { size_t size() const override; size_t zoom() const override; void set(TileCoordinate x, TileCoordinate y) override; + void forEachSet(const std::function& f) const override; private: unsigned int zoom_; diff --git a/include/tile_data.h b/include/tile_data.h index 3e8d20e9..0adf6d95 100644 --- a/include/tile_data.h +++ b/include/tile_data.h @@ -2,10 +2,20 @@ #ifndef _TILE_DATA_H #define _TILE_DATA_H +#include +#include #include +#include #include #include +#include +#include #include +#include +#include +#include +#include +#include #include "append_vector.h" #include "clip_cache.h" #include "mmap_allocator.h" @@ -14,6 +24,14 @@ typedef std::vector SourceList; +// The TileDataSources alive, for the store leases a thread keeps until it ends (leased_store.h): the +// main thread gives its leases back at exit, after the sources are gone. A source is also told apart +// by its number, as a new source can be at the address of a destroyed one. +uint64_t registerLeaseSource(const TileDataSource *source); +void unregisterLeaseSource(const TileDataSource *source); +std::mutex &leaseSourcesMutex(); +bool leaseSourceAlive(const TileDataSource *source, uint64_t number); // with leaseSourcesMutex() held + class TileBbox; template void sortOutputObjects( @@ -31,77 +49,134 @@ template void finalizeObjects( typename std::vector>::iterator end, typename std::vector>& lowZoom ) { - size_t z6OffsetDivisor = indexZoom >= CLUSTER_ZOOM ? (1 << (indexZoom - CLUSTER_ZOOM)) : 1; #ifdef CLOCK_MONOTONIC timespec startTs, endTs; clock_gettime(CLOCK_MONOTONIC, &startTs); #endif - - int i = -1; - for (auto it = begin; it != end; it++) { - i++; - if (it->size() > 0 || i % 50 == 0 || i == 4095) { - std::cout << "\r" << name << ": finalizing z6 tile " << (i + 1) << "/" << CLUSTER_ZOOM_AREA; - + const size_t shards = end - begin; + std::mutex progressMutex; + size_t done = 0; + auto reportDone = [&](size_t n) { + std::lock_guard lock(progressMutex); + done += n; + std::cout << "\r" << name << ": finalizing z6 tile " << done << "/" << CLUSTER_ZOOM_AREA; #ifdef CLOCK_MONOTONIC - clock_gettime(CLOCK_MONOTONIC, &endTs); - uint64_t elapsedNs = 1e9 * (endTs.tv_sec - startTs.tv_sec) + endTs.tv_nsec - startTs.tv_nsec; - std::cout << " (" << std::to_string((uint32_t)(elapsedNs / 1e6)) << " ms)"; + clock_gettime(CLOCK_MONOTONIC, &endTs); + uint64_t elapsedNs = 1e9 * (endTs.tv_sec - startTs.tv_sec) + endTs.tv_nsec - startTs.tv_nsec; + std::cout << " (" << std::to_string((uint32_t)(elapsedNs / 1e6)) << " ms)"; #endif - std::cout << std::flush; - } - if (it->size() == 0) - continue; + std::cout << std::flush; + }; + auto finalizeShard = [&](size_t i, size_t sortThreads) { + auto it = begin + i; // We track a separate copy of low zoom objects to avoid scanning large // lists of objects that may be on slow disk storage. for (auto objectIt = it->begin(); objectIt != it->end(); objectIt++) if (objectIt->oo.minZoom < CLUSTER_ZOOM) lowZoom[i].push_back(*objectIt); - sortOutputObjects(indexZoom, threadNum, it->begin(), it->end()); + sortOutputObjects(indexZoom, sortThreads, it->begin(), it->end()); + }; + + // Shards are independent. A shard with more than 1/threadNum of all objects + // (e.g. one of the few populated z6 tiles of a small extract) is sorted on its + // own using all threads. The others are sorted in parallel, one thread each, + // largest first. + size_t total = 0, empty = 0; + for (auto it = begin; it != end; it++) total += it->size(); + std::vector big, small; + for (size_t i = 0; i < shards; i++) { + const size_t n = (begin + i)->size(); + if (n == 0) empty++; + else if (threadNum <= 1 || n * threadNum > total) big.push_back(i); + else small.push_back(i); + } + std::sort(small.begin(), small.end(), [&](size_t a, size_t b) { return (begin + a)->size() > (begin + b)->size(); }); + reportDone(empty); + + for (size_t i : big) { + finalizeShard(i, threadNum); + reportDone(1); + } + + if (!small.empty()) { + // Each sort copies its shard into a buffer, so cap the objects being + // sorted at once (a single shard may always run). + const size_t budget = std::max(total / 8, (begin + small.front())->size()); + size_t inFlight = 0; + std::mutex budgetMutex; + std::condition_variable budgetFreed; + boost::asio::thread_pool pool(threadNum); + for (size_t i : small) { + boost::asio::post(pool, [&, i]() { + const size_t n = (begin + i)->size(); + { + std::unique_lock lock(budgetMutex); + budgetFreed.wait(lock, [&]() { return inFlight == 0 || inFlight + n <= budget; }); + inFlight += n; + } + finalizeShard(i, 1); + { + std::lock_guard lock(budgetMutex); + inFlight -= n; + } + budgetFreed.notify_all(); + reportDone(1); + }); + } + pool.join(); } std::cout << std::endl; } -template void collectTilesWithObjectsAtZoomTemplate( +// Set the tiles holding the objects of z6 tile i (shard), and their parent tiles. +template void collectTilesWithObjectsInShardTemplate( const unsigned int& indexZoom, - const typename std::vector>::iterator objects, - const size_t size, + AppendVectorNS::AppendVector& shard, + const size_t i, std::vector>& zooms ) { size_t maxZoom = zooms.size() - 1; uint16_t z6OffsetDivisor = indexZoom >= CLUSTER_ZOOM ? (1 << (indexZoom - CLUSTER_ZOOM)) : 1; int64_t lastX = -1; int64_t lastY = -1; - for (size_t i = 0; i < size; i++) { - const size_t z6x = i / CLUSTER_ZOOM_WIDTH; - const size_t z6y = i % CLUSTER_ZOOM_WIDTH; - - for (size_t j = 0; j < objects[i].size(); j++) { - // Compute the x, y at the base zoom level - TileCoordinate baseX = z6x * z6OffsetDivisor + objects[i][j].x; - TileCoordinate baseY = z6y * z6OffsetDivisor + objects[i][j].y; - - // Translate the x, y at the requested zoom level - TileCoordinate x = baseX / (1 << (indexZoom - maxZoom)); - TileCoordinate y = baseY / (1 << (indexZoom - maxZoom)); - - if (lastX != x || lastY != y) { - lastX = x; - lastY = y; - - for (int zoom = maxZoom; zoom >= 0; zoom--) { - zooms[zoom]->set(x, y); - x /= 2; - y /= 2; - } + const size_t z6x = i / CLUSTER_ZOOM_WIDTH; + const size_t z6y = i % CLUSTER_ZOOM_WIDTH; + + for (size_t j = 0; j < shard.size(); j++) { + // Compute the x, y at the base zoom level + TileCoordinate baseX = z6x * z6OffsetDivisor + shard[j].x; + TileCoordinate baseY = z6y * z6OffsetDivisor + shard[j].y; + + // Translate the x, y at the requested zoom level + TileCoordinate x = baseX / (1 << (indexZoom - maxZoom)); + TileCoordinate y = baseY / (1 << (indexZoom - maxZoom)); + + if (lastX != x || lastY != y) { + lastX = x; + lastY = y; + + for (int zoom = maxZoom; zoom >= 0; zoom--) { + zooms[zoom]->set(x, y); + x /= 2; + y /= 2; } } } } +template void collectTilesWithObjectsAtZoomTemplate( + const unsigned int& indexZoom, + const typename std::vector>::iterator objects, + const size_t size, + std::vector>& zooms +) { + for (size_t i = 0; i < size; i++) + collectTilesWithObjectsInShardTemplate(indexZoom, objects[i], i, zooms); +} + template OutputObjectID outputObjectWithId(const OO& input) { return OutputObjectID({ input.oo, 0 }); @@ -245,7 +320,7 @@ template void collectObjectsForTileTemplate( class TileDataSource { public: // Store for generated geometries - using point_store_t = std::vector; + using point_store_t = AppendVectorNS::AppendVector; using linestring_t = boost::geometry::model::linestring; using linestring_store_t = std::vector; @@ -308,8 +383,12 @@ class TileDataSource { public: TileDataSource(size_t threadNum, unsigned int indexZoom, bool includeID); + ~TileDataSource(); + const uint64_t leaseNumber = registerLeaseSource(this); // see registerLeaseSource void collectTilesWithObjectsAtZoom(std::vector>& zooms); + // The same for the objects of one z6 tile (0 <= shard < CLUSTER_ZOOM_AREA) + void collectTilesWithObjectsInShard(size_t shard, std::vector>& zooms); void collectTilesWithLargeObjectsAtZoom(std::vector>& zooms); @@ -348,11 +427,40 @@ class TileDataSource { std::vector getObjectsForTile( const std::vector& sortOrders, + const AttributeStore& attributeStore, unsigned int zoom, TileCoordinates coordinates ); + void orderStoredGeometryTies(std::vector& data) const; virtual Geometry buildWayGeometry(OutputGeometryType const geomType, NodeID const objectID, const TileBbox &bbox); + + // Large polygons are clipped once per SHARED_PIECE_ZOOM tile before the tiles are written, + // and deeper tiles clip from that piece instead of the whole polygon (see tile_data.cpp). + static constexpr unsigned SHARED_PIECE_ZOOM = 8; + // present(x, y): whether any tile below SHARED_PIECE_ZOOM tile x/y will be written + void precomputeSharedPieces(const std::function &present, unsigned threadNum); + // Compute pieces until none is left to start; the tile threads call this before their first tile + void computeSharedPieces(); + // Drop the shared pieces inside a z6 tile, once no tile in it is still to be written + void releaseSharedPieces(TileCoordinate z6x, TileCoordinate z6y); + // Whether objectID's polygon is kept in the multipolygon store (retrieveMultiPolygon) + virtual bool storesMultiPolygon(NodeID objectID) const { return true; } + struct SharedPiece { + std::shared_ptr geom; // null: no usable piece, clip the whole polygon + bool valid = false; + }; + // Filled by precomputeSharedPieces, then only read while the tiles are written (no lock). + // releaseSharedPieces resets the pieces of a z6 tile no thread reads any more. + std::unordered_set pieceObjects; + std::map, std::shared_future>> sharedPieces; + struct PieceJob { + std::tuple key; + uint64_t points; + std::shared_ptr>> promise; + }; + std::vector pieceJobs; // largest polygon first + std::atomic nextPieceJob { 0 }; virtual LatpLon buildNodeGeometry(NodeID const objectID, const TileBbox &bbox) const; void open() { @@ -398,8 +506,8 @@ class TileDataSource { const auto& shardId = getShard(id); const auto& shard = pointStores[shardId]; const auto offset = getId(id); - if (offset > shard.size()) throw std::out_of_range("Could not find generated node with id " + std::to_string(id) + ", shard " + std::to_string(shardId) + ", offset=" + std::to_string(offset)); - return shard.at(offset); + if (offset >= shard.size()) throw std::out_of_range("Could not find generated node with id " + std::to_string(id) + ", shard " + std::to_string(shardId) + ", offset=" + std::to_string(offset)); + return shard[offset]; } NodeID storeLinestring(const Linestring& src); @@ -436,7 +544,8 @@ class TileDataSource { void populateTilesAtZoom( const std::vector& sources, - std::vector>& zooms + std::vector>& zooms, + size_t threadNum = 1 ); #endif //_TILE_DATA_H diff --git a/resources/process-debug.lua b/resources/process-debug.lua index 4ea5e629..5fc399f1 100644 --- a/resources/process-debug.lua +++ b/resources/process-debug.lua @@ -155,6 +155,10 @@ poiTags = { aerialway = Set { "station" }, sport = Set { "american_football", "archery", "athletics", "australian_football", "badminton", "baseball", "basketball", "beachvolleyball", "billiards", "bmx", "boules", "bowls", "boxing", "canadian_football", "canoe", "chess", "climbing", "climbing_adventure", "cricket", "cricket_nets", "croquet", "curling", "cycling", "disc_golf", "diving", "dog_racing", "equestrian", "fatsal", "field_hockey", "free_flying", "gaelic_games", "golf", "gymnastics", "handball", "hockey", "horse_racing", "horseshoes", "ice_hockey", "ice_stock", "judo", "karting", "korfball", "long_jump", "model_aerodrome", "motocross", "motor", "multi", "netball", "orienteering", "paddle_tennis", "paintball", "paragliding", "pelota", "racquet", "rc_car", "rowing", "rugby", "rugby_league", "rugby_union", "running", "sailing", "scuba_diving", "shooting", "shooting_range", "skateboard", "skating", "skiing", "soccer", "surfing", "swimming", "table_soccer", "table_tennis", "team_handball", "tennis", "toboggan", "volleyball", "water_ski", "yoga" }, tourism = Set { "alpine_hut", "aquarium", "artwork", "attraction", "bed_and_breakfast", "camp_site", "caravan_site", "chalet", "gallery", "guest_house", "hostel", "hotel", "information", "motel", "museum", "picnic_site", "theme_park", "viewpoint", "zoo" }, waterway = Set { "dock" } } +-- poiTags keys in alphabetical order: GetPOIRank tries them in this order, and the first key an object has decides its class +poiTagOrder = {} +for k in pairs(poiTags) do poiTagOrder[#poiTagOrder+1] = k end +table.sort(poiTagOrder) -- POI "class" values: based on https://github.com/openmaptiles/openmaptiles/blob/master/layers/poi/poi.yaml poiClasses = { townhall="town_hall", public_building="town_hall", courthouse="town_hall", community_centre="town_hall", @@ -504,7 +508,8 @@ function GetPOIRank(obj) local k,list,v,class,rank -- Can we find the tag? - for k,list in pairs(poiTags) do + for _,k in ipairs(poiTagOrder) do + list = poiTags[k] if list[Find(k)] then v = Find(k) -- k/v are the OSM tag pair class = poiClasses[v] or v diff --git a/resources/process-openmaptiles.lua b/resources/process-openmaptiles.lua index 74892add..7591e60f 100644 --- a/resources/process-openmaptiles.lua +++ b/resources/process-openmaptiles.lua @@ -271,6 +271,10 @@ poiTags = { aerialway = Set { "station" }, sport = Set { "american_football", "archery", "athletics", "australian_football", "badminton", "baseball", "basketball", "beachvolleyball", "billiards", "bmx", "boules", "bowls", "boxing", "canadian_football", "canoe", "chess", "climbing", "climbing_adventure", "cricket", "cricket_nets", "croquet", "curling", "cycling", "disc_golf", "diving", "dog_racing", "equestrian", "fatsal", "field_hockey", "free_flying", "gaelic_games", "golf", "gymnastics", "handball", "hockey", "horse_racing", "horseshoes", "ice_hockey", "ice_stock", "judo", "karting", "korfball", "long_jump", "model_aerodrome", "motocross", "motor", "multi", "netball", "orienteering", "paddle_tennis", "paintball", "paragliding", "pelota", "racquet", "rc_car", "rowing", "rugby", "rugby_league", "rugby_union", "running", "sailing", "scuba_diving", "shooting", "shooting_range", "skateboard", "skating", "skiing", "soccer", "surfing", "swimming", "table_soccer", "table_tennis", "team_handball", "tennis", "toboggan", "volleyball", "water_ski", "yoga" }, tourism = Set { "alpine_hut", "aquarium", "artwork", "attraction", "bed_and_breakfast", "camp_site", "caravan_site", "chalet", "gallery", "guest_house", "hostel", "hotel", "information", "motel", "museum", "picnic_site", "theme_park", "viewpoint", "zoo" }, waterway = Set { "dock" } } +-- poiTags keys in alphabetical order: GetPOIRank tries them in this order, and the first key an object has decides its class +poiTagOrder = {} +for k in pairs(poiTags) do poiTagOrder[#poiTagOrder+1] = k end +table.sort(poiTagOrder) -- POI "class" values: based on https://github.com/openmaptiles/openmaptiles/blob/master/layers/poi/poi.yaml poiClasses = { townhall="town_hall", public_building="town_hall", courthouse="town_hall", community_centre="town_hall", @@ -912,7 +916,8 @@ function GetPOIRank() local k,list,v,class,rank -- Can we find the tag? - for k,list in pairs(poiTags) do + for _,k in ipairs(poiTagOrder) do + list = poiTags[k] if list[Find(k)] then v = Find(k) -- k/v are the OSM tag pair class = poiClasses[v] or poiTagsAsClass[k] or v diff --git a/src/attribute_store.cpp b/src/attribute_store.cpp index 6e661c8d..f2a3d77d 100644 --- a/src/attribute_store.cpp +++ b/src/attribute_store.cpp @@ -1,4 +1,5 @@ #include "attribute_store.h" +#include "stable_hash.h" #include #include @@ -55,6 +56,15 @@ const std::string& AttributeKeyStore::getKeyUnsafe(uint16_t index) const { return keys[index]; } +void AttributeKeyStore::finalize() { + finalized = true; + // keys2index is ordered by key string + ranks.assign(keys.size(), 0); + uint16_t rank = 0; + for (const auto& it : keys2index) + ranks[it.second] = ++rank; +} + // AttributePair void AttributePair::ensureStringIsOwned() { // Before we store an AttributePair in our long-term storage, we need @@ -557,6 +567,52 @@ void AttributeStore::reset() { pairIndexGeneration++; } +int AttributeStore::comparePairsUnsafe(uint32_t a, uint32_t b) const { + if (a == b) return 0; + const AttributePair& p = pairStore.getPairUnsafe(a); + const AttributePair& q = pairStore.getPairUnsafe(b); + const uint16_t pRank = keyStore.getRankUnsafe(p.keyIndex), qRank = keyStore.getRankUnsafe(q.keyIndex); + if (pRank != qRank) return pRank < qRank ? -1 : 1; + if (p.minzoom != q.minzoom) return p.minzoom < q.minzoom ? -1 : 1; + if (p.valueType != q.valueType) return p.valueType < q.valueType ? -1 : 1; + if (p.hasStringValue()) { + if (p.pooledString() < q.pooledString()) return -1; + if (q.pooledString() < p.pooledString()) return 1; + return 0; + } + if (p.doubleValue_ < q.doubleValue_) return -1; + if (q.doubleValue_ < p.doubleValue_) return 1; + return 0; +} + +int AttributeStore::compareContentUnsafe(AttributeIndex a, AttributeIndex b) const { + if (a == b) return 0; + boost::container::small_vector x, y; + forEachPairIndexCanonicalUnsafe(a, [&](uint32_t pairIndex) { x.push_back(pairIndex); }); + forEachPairIndexCanonicalUnsafe(b, [&](uint32_t pairIndex) { y.push_back(pairIndex); }); + for (size_t i = 0; i < x.size() && i < y.size(); i++) { + const int c = comparePairsUnsafe(x[i], y[i]); + if (c != 0) return c; + } + if (x.size() != y.size()) return x.size() < y.size() ? -1 : 1; + return 0; +} + +uint64_t AttributeStore::contentHashUnsafe(AttributeIndex index) const { + // Sum of the pair hashes, so the (index dependent) pair order doesn't matter + uint64_t rv = 0; + forEachPairUnsafe(index, [&](const AttributePair& p) { + uint64_t h = stableCombine(keyStore.getRankUnsafe(p.keyIndex), static_cast(p.minzoom)); + h = stableCombine(h, static_cast(p.valueType)); + if (p.hasStringValue()) + h = stableCombine(h, p.pooledString().data(), p.pooledString().size()); + else + h = stableCombine(h, p.doubleValue_); + rv += h; + }); + return rv; +} + void AttributeStore::finalize() { finalized = true; keyStore.finalize(); diff --git a/src/coordinates_geom.cpp b/src/coordinates_geom.cpp index 16887856..d6c35d86 100644 --- a/src/coordinates_geom.cpp +++ b/src/coordinates_geom.cpp @@ -50,6 +50,24 @@ void TileBbox::scaleRing(Ring &points, Ring const &src) const { } } +// Scaling that only drops points repeating their immediate predecessor. Used as +// a fallback for rings that scaleRing() collapses below 4 points: its +// backtracking window is positional, so a duplicated FIRST vertex shifts every +// later vertex by one, the closing vertex then matches that duplicate at j==4, +// and resize() truncates the whole ring. Without the backtracking there is +// nothing to mis-align, and the ring survives with its shape intact. +void TileBbox::scaleRingNoBacktrack(Ring &points, Ring const &src) const { + points.clear(); + points.reserve(src.size()); + for(auto const &i: src) { + auto scaled = scaleLatpLon(i.y(), i.x()); + if (!points.empty() && + points.back().x()==scaled.first && points.back().y()==scaled.second) + continue; + points.push_back(Point(scaled.first,scaled.second)); + } +} + Ring TileBbox::scaleRing(Ring const &src) const { Ring points; scaleRing(points, src); @@ -66,8 +84,15 @@ void TileBbox::scaleGeometry(MultiPolygon &dst, MultiPolygon const &src) const { // Copy the outer ring scaleRing(p.outer(), poly.outer()); - if (p.outer().size()<4) - continue; + if (p.outer().size()<4) { + // A collapsed outer ring means the whole feature disappears from the + // tile, at any size - so before giving up, retry without the + // backtracking that can truncate a ring to two points. Every ring that + // already survives is left untouched. + scaleRingNoBacktrack(p.outer(), poly.outer()); + if (p.outer().size()<4) + continue; + } // Copy the inner rings if (p.inners().size() < poly.inners().size()) diff --git a/src/geom.cpp b/src/geom.cpp index 43d7d079..a3221c2b 100644 --- a/src/geom.cpp +++ b/src/geom.cpp @@ -178,13 +178,17 @@ struct PrecheckBox { }; struct PrecheckScratch { std::vector segs; - std::vector boxes, holeBoxes; + std::vector boxes; std::vector> pairs; + std::vector bandStart, bandFill, bandSegs, crossed; void trim() { if (segs.capacity() > kPrecheckRetained) std::vector().swap(segs); if (boxes.capacity() > kPrecheckRetained) std::vector().swap(boxes); - if (holeBoxes.capacity() > kPrecheckRetained) std::vector().swap(holeBoxes); if (pairs.capacity() > kPrecheckRetained) std::vector>().swap(pairs); + if (bandStart.capacity() > kPrecheckRetained) std::vector().swap(bandStart); + if (bandFill.capacity() > kPrecheckRetained) std::vector().swap(bandFill); + if (bandSegs.capacity() > kPrecheckRetained) std::vector().swap(bandSegs); + if (crossed.capacity() > kPrecheckRetained) std::vector().swap(crossed); } }; thread_local PrecheckScratch precheckScratch; @@ -331,20 +335,27 @@ bool segmentsClear(std::vector &segs, double tol) { return true; } +// Whether the ray from h to +x crosses segment a->b: 1 = yes, -1 = no, 0 = unsure. +// Vertices are classed as above (y > hy) or not with exact comparisons (the +// half-open rule), so a ray through a vertex is counted consistently and the +// crossing parity is exact; only the side of h is decided with a tolerance. +inline int rayClearlyCrosses(const Point &h, const Point &a, const Point &b, double tol) { + const double hx = h.x(), hy = h.y(); + if ((a.y() > hy) == (b.y() > hy)) return -1; + if (std::max(a.x(), b.x()) < hx) return -1; + if (std::min(a.x(), b.x()) > hx) return 1; + const int o = clearOrient(a, b, h, tol); + if (o == 0) return 0; + return (b.y() > a.y()) == (o > 0) ? 1 : -1; +} + // Crossing-number test: 1 = h clearly inside r, -1 = clearly outside, 0 = unsure. int pointClearlyInRing(const Point &h, const Ring &r, double tol) { bool inside = false; - const double hx = h.x(), hy = h.y(); for (std::size_t i = 0; i + 1 < r.size(); i++) { - const Point &a = r[i], &b = r[i + 1]; - if ((a.y() > hy + tol && b.y() > hy + tol) || (a.y() < hy - tol && b.y() < hy - tol)) continue; - if (std::fabs(a.y() - hy) <= tol || std::fabs(b.y() - hy) <= tol) return 0; - // a and b are clearly on opposite sides of the horizontal line through h - if (std::max(a.x(), b.x()) < hx - tol) continue; - if (std::min(a.x(), b.x()) > hx + tol) { inside = !inside; continue; } - const int o = clearOrient(a, b, h, tol); - if (o == 0) return 0; - if ((b.y() > a.y()) == (o > 0)) inside = !inside; // the ray to +x crosses a->b + const int c = rayClearlyCrosses(h, r[i], r[i + 1], tol); + if (c == 0) return 0; + if (c > 0) inside = !inside; } return inside ? 1 : -1; } @@ -360,12 +371,91 @@ int ringClearlyInRing(const Ring &inner, const Ring &r, double tol) { return 0; } +// Holes lie inside the outer ring (ring 0) and outside each other. `segs` holds the +// segments of all rings of p, known not to touch. They are bucketed into horizontal +// bands, so that a ray from a vertex of each hole is tested against the segments of +// that vertex's band only, counting crossings per ring (odd = inside that ring). +bool holesClearlyPlaced(const Polygon &p, PrecheckScratch &scratch, double tol) { + const auto &inners = p.inners(); + const std::vector &segs = scratch.segs; + if (inners.empty()) return true; + if (segs.size() >= std::numeric_limits::max()) return false; + + double minY = std::numeric_limits::infinity(), maxY = -minY; + for (const PrecheckSeg &s : segs) { minY = std::min(minY, s.minY); maxY = std::max(maxY, s.maxY); } + minY -= tol; maxY += tol; + // About 4 segments per band; fewer bands while tall segments would make the index much bigger than segs + std::size_t bands = std::max(1, segs.size() / 4); + double scale = 0; + auto bandOf = [&](double y) { + const double b = (y - minY) * scale; + return b <= 0 ? std::size_t(0) : std::min(bands - 1, static_cast(b)); + }; + std::size_t entries; + for (;;) { + scale = bands / (maxY - minY); + entries = 0; + for (const PrecheckSeg &s : segs) entries += bandOf(s.maxY + tol) - bandOf(s.minY - tol) + 1; + if (entries <= 4 * segs.size() || bands == 1) break; + bands = (bands + 1) / 2; + } + if (entries >= std::numeric_limits::max()) return false; + + // Segments of band b: bandSegs[bandStart[b] .. bandStart[b+1]) + std::vector &start = scratch.bandStart, &fill = scratch.bandFill, &items = scratch.bandSegs; + start.assign(bands + 1, 0); + for (const PrecheckSeg &s : segs) + for (std::size_t b = bandOf(s.minY - tol), hi = bandOf(s.maxY + tol); b <= hi; b++) start[b + 1]++; + for (std::size_t b = 0; b < bands; b++) start[b + 1] += start[b]; + fill.assign(start.begin(), start.end() - 1); + items.resize(entries); + for (std::size_t k = 0; k < segs.size(); k++) + for (std::size_t b = bandOf(segs[k].minY - tol), hi = bandOf(segs[k].maxY + tol); b <= hi; b++) + items[fill[b]++] = static_cast(k); + + // 1 = h inside the outer ring and no other hole, -1 = not, 0 = unsure + std::vector &crossed = scratch.crossed; + auto locate = [&](const Point &h, uint32_t ownRing) { + crossed.clear(); + const std::size_t b = bandOf(h.y()); + for (uint32_t k = start[b]; k < start[b + 1]; k++) { + const PrecheckSeg &s = segs[items[k]]; + if (s.ring == ownRing) continue; + const int c = rayClearlyCrosses(h, s.a[0], s.a[1], tol); + if (c == 0) return 0; + if (c > 0) crossed.push_back(s.ring); + } + std::sort(crossed.begin(), crossed.end()); + bool inOuter = false; + for (std::size_t i = 0; i < crossed.size(); ) { + std::size_t j = i; + while (j < crossed.size() && crossed[j] == crossed[i]) j++; + if ((j - i) % 2 == 1) { + if (crossed[i] != 0) return -1; // inside another hole + inOuter = true; + } + i = j; + } + return inOuter ? 1 : -1; + }; + + // A ring known not to touch the others lies wherever any of its vertices does + for (std::size_t i = 0; i < inners.size(); i++) { + const Ring &hole = inners[i]; + const std::size_t m = hole.size() - 1; + const std::size_t tries = std::min(m, 8); + int loc = 0; + for (std::size_t k = 0; k < tries && loc == 0; k++) + loc = locate(hole[k * m / tries], static_cast(i + 1)); + if (loc != 1) return false; + } + return true; +} + bool polygonClear(const Polygon &p, PrecheckScratch &scratch, PrecheckBox &outerBox) { const Ring &outer = p.outer(); const auto &inners = p.inners(); if (inners.size() >= std::numeric_limits::max()) return false; - // Every hole is located by walking the outer ring - if (inners.size() * outer.size() > kPrecheckMaxWork) return false; double maxAbs = 0; if (!ringFinite(outer, maxAbs)) return false; for (const auto &inner : inners) @@ -374,29 +464,16 @@ bool polygonClear(const Polygon &p, PrecheckScratch &scratch, PrecheckBox &outer std::vector &segs = scratch.segs; segs.clear(); - std::vector &holeBoxes = scratch.holeBoxes; - holeBoxes.resize(inners.size()); if (!ringClear(outer, false, tol, 0, segs, outerBox)) return false; - for (std::size_t i = 0; i < inners.size(); i++) { - if (!ringClear(inners[i], true, tol, static_cast(i + 1), segs, holeBoxes[i])) return false; - holeBoxes[i].id = static_cast(i); - } + PrecheckBox holeBox; + for (std::size_t i = 0; i < inners.size(); i++) + if (!ringClear(inners[i], true, tol, static_cast(i + 1), segs, holeBox)) return false; // No two rings touch, and no ring touches itself. if (!segmentsClear(segs, tol)) return false; // Holes lie inside the outer ring and are not nested in each other. - for (std::size_t i = 0; i < inners.size(); i++) { - if (boxesApart(holeBoxes[i], outerBox, tol)) return false; - if (ringClearlyInRing(inners[i], outer, tol) != 1) return false; - } - std::size_t work = 0; - return forOverlappingBoxes(holeBoxes, tol, [&](uint32_t i, uint32_t j) { - work += inners[i].size() + inners[j].size(); - return work <= kPrecheckMaxWork && - ringClearlyInRing(inners[i], inners[j], tol) == -1 && - ringClearlyInRing(inners[j], inners[i], tol) == -1; - }); + return holesClearlyPlaced(p, scratch, tol); } // a's outer ring, known not to touch any ring of b, lies outside b's interior: @@ -839,6 +916,11 @@ struct RectClipper { bool inside(const Point &p) const { return p.x() >= x0 && p.x() <= x1 && p.y() >= y0 && p.y() <= y1; } + // Both end points beyond the same edge: the segment can't reach the box + bool beyondOneEdge(const Point &a, const Point &b) const { + return (a.x() < x0 && b.x() < x0) || (a.x() > x1 && b.x() > x1) || + (a.y() < y0 && b.y() < y0) || (a.y() > y1 && b.y() > y1); + } bool strictlyInside(const Point &p) const { return p.x() > x0 && p.x() < x1 && p.y() > y0 && p.y() < y1; } @@ -965,7 +1047,9 @@ RingClip clipRing(const RectClipper &rc, const Ring &ring, std::vector t0; + // Most segments of a large polygon lie far from a small tile's box: reject + // those with four comparisons before the clip computation + const bool hit = !rc.beyondOneEdge(a, b) && rc.clipSegment(a, b, t0, t1, pa, pb) && t1 > t0; if (!hit || (building && t0 > 0)) { if (building && !finish()) return RingClip::Fail; if (!hit) continue; diff --git a/src/mbtiles.cpp b/src/mbtiles.cpp index c056d3a6..1d2e653f 100644 --- a/src/mbtiles.cpp +++ b/src/mbtiles.cpp @@ -2,15 +2,30 @@ #include "helpers.h" #include #include +#include using namespace sqlite; using namespace std; MBTiles::MBTiles(): pendingStatements1(std::make_shared>()), - pendingStatements2(std::make_shared>()) + pendingStatements2(std::make_shared>()), + pendingBytes(0), + batches([this](PendingStatement& stmt) { + const std::lock_guard lock(m); + insertOrReplace(stmt.zoom, stmt.x, stmt.y, stmt.data, stmt.isMerge); + }, + [](const PendingStatement& stmt) { return sizeof(PendingStatement) + stmt.data.size(); }) {} +// At most this much is buffered while SQLite is busy; beyond it, workers wait +// for the writer instead (a planet run otherwise buffered ~15M tiles, over 3 GB). +static const size_t MAX_PENDING_BYTES = 256 << 20; + +static size_t pendingCost(const std::string& data) { + return sizeof(PendingStatement) + data.size(); +} + MBTiles::~MBTiles() { if (db && inTransaction) db << "COMMIT;"; // commit all the changes if open } @@ -71,6 +86,7 @@ void MBTiles::flushPendingStatements() { while(!pendingStatements2->empty()) { const PendingStatement& stmt = pendingStatements2->back(); insertOrReplace(stmt.zoom, stmt.x, stmt.y, stmt.data, stmt.isMerge); + pendingBytes -= pendingCost(stmt.data); pendingStatements2->pop_back(); } @@ -80,19 +96,37 @@ void MBTiles::flushPendingStatements() { } void MBTiles::saveTile(int zoom, int x, int y, string *data, bool isMerge) { + if (batches.inBatch()) { + batches.add({zoom, x, y, *data, isMerge}); + return; + } + // If the lock is available, write directly to SQLite. if (m.try_lock()) { insertOrReplace(zoom, x, y, *data, isMerge); flushPendingStatements(); m.unlock(); - } else { + return; + } + + { // Else buffer the write for later, copying its binary blob. const std::lock_guard lock(pendingStatementsMutex); - pendingStatements1->push_back({zoom, x, y, *data, isMerge}); + if (pendingBytes < MAX_PENDING_BYTES) { + pendingBytes += pendingCost(*data); + pendingStatements1->push_back({zoom, x, y, *data, isMerge}); + return; + } } + + // The buffer is full: wait for SQLite, then help drain the buffer. + const std::lock_guard lock(m); + insertOrReplace(zoom, x, y, *data, isMerge); + flushPendingStatements(); } void MBTiles::closeForWriting() { + if (!batches.allWritten()) throw std::runtime_error("MBTiles: a batch of tiles was never written - please file an issue"); flushPendingStatements(); preparedStatements[0].used(true); preparedStatements[1].used(true); diff --git a/src/mmap_allocator.cpp b/src/mmap_allocator.cpp index 65818873..33ced307 100644 --- a/src/mmap_allocator.cpp +++ b/src/mmap_allocator.cpp @@ -1,9 +1,13 @@ #include "mmap_allocator.h" +#include +#include #include #include #include #include #include +#include +#include #include #include @@ -94,13 +98,15 @@ static mmap_dir_t mmap_dir; std::vector< mmap_shm_ptr > mmap_shm_regions; size_t mmap_shm::mmap_file_size = 0; -std::mutex mmap_allocator_mutex; +// Never destroyed (nor the two lists below): thread_regions_t's destructor uses them, and the +// main thread's thread-locals are destroyed after the statics on some platforms (macOS) +std::mutex &mmap_allocator_mutex = *new std::mutex; // Regions of threads that have exited, for reuse by new threads. Every thread // pool (one per .shp/GeoJSON layer, per .pbf read phase) starts new threads, and // a new region (or 1GB store file) for each of them would never be released. -std::vector free_mmap_files; -std::vector free_mmap_shm_regions; +std::vector &free_mmap_files = *new std::vector(); +std::vector &free_mmap_shm_regions = *new std::vector(); // The regions the current thread allocates from struct thread_regions_t @@ -191,6 +197,31 @@ bool mmap_dir_t::is_open() { if(!mmap_dir_filename.empty()) { try { + // The files of exited threads, kept for reuse: let them go first, so that they are + // deleted below (free_mmap_files is never destroyed) + { + std::lock_guard lock(mmap_allocator_mutex); + free_mmap_files.clear(); + } + // Unmapping and deleting a big store (planet: ~140 GB in ~300 files) + // takes half a minute on one thread, so release the files from several. + const size_t threads = std::min(files.size(), std::min(16u, std::max(1u, std::thread::hardware_concurrency()))); + if (threads > 1) { + std::atomic next(0); + std::vector workers; + for (size_t t = 0; t < threads; t++) { + try { + workers.emplace_back([this, &next]() { + for (size_t i; (i = next++) < files.size(); ) + files[i].reset(); + }); + } catch (const std::system_error &) { + break; // fewer threads; files.clear() below releases the rest + } + } + for (auto &worker : workers) + worker.join(); + } files.clear(); if(mmap_dir_created) { diff --git a/src/osm_store.cpp b/src/osm_store.cpp index 5ec7a81c..3bb086f2 100644 --- a/src/osm_store.cpp +++ b/src/osm_store.cpp @@ -6,6 +6,7 @@ #include #include +#include #include "node_store.h" #include "way_store.h" @@ -16,6 +17,42 @@ static inline bool isClosed(const std::vector& way) { return !way.empty() && way.front() == way.back(); } +// geom::within(inner, outer) for many inner rings against one big outer ring. +// Boost analyses the outer ring afresh on every call (a reservoir with 2500 +// islands and a 600,000-point shore takes 47 s); here its segments are indexed +// once. An inner ring that shares no point with the outer ring lies wholly +// inside or wholly outside it, so one of its vertices decides; when the rings +// touch or cross, Boost decides as before. +class RingWithinIndex { + using Segment = bg::model::segment; + using Value = std::pair; + const Ring &outer; + bg::index::rtree> segments; + + static std::vector segmentBoxes(const Ring &ring) { + std::vector values; + for (std::size_t i = 0; i + 1 < ring.size(); i++) + values.emplace_back(bg::return_envelope(Segment(ring[i], ring[i + 1])), i); + return values; + } + +public: + RingWithinIndex(const Ring &outer) : outer(outer), segments(segmentBoxes(outer)) { } + + bool within(const Ring &inner) const { + if (inner.size() < 4) return geom::within(inner, outer); + for (std::size_t j = 0; j + 1 < inner.size(); j++) { + const Segment s(inner[j], inner[j + 1]); + for (auto it = segments.qbegin(bg::index::intersects(bg::return_envelope(s))); it != segments.qend(); ++it) { + const std::size_t i = it->second; + if (bg::intersects(s, Segment(outer[i], outer[i + 1]))) + return geom::within(inner, outer); + } + } + return geom::within(inner.front(), outer); + } +}; + UsedObjects::UsedObjects(Status status): status(status), mutex(256), ids(256 * 1024) { } @@ -76,17 +113,31 @@ MultiPolygon OSMStore::wayListMultiPolygon(WayVec::const_iterator outerBegin, Wa // add all inners and outers to the multipolygon std::vector filledInners; + std::vector innerBoxes; for (auto it = inners.begin(); it != inners.end(); ++it) { Ring inner; fillPoints(inner, it->begin(), it->end()); + innerBoxes.emplace_back(geom::return_envelope(inner)); filledInners.emplace_back(inner); } bool onlyOneOuter = outers.size()==1; for (auto ot = outers.begin(); ot != outers.end(); ot++) { Polygon poly; fillPoints(poly.outer(), ot->begin(), ot->end()); - for (auto it = filledInners.begin(); it != filledInners.end(); ++it) { - if (onlyOneOuter || geom::within(*it, poly.outer())) { poly.inners().emplace_back(*it); } + const Box outerBox = geom::return_envelope(poly.outer()); + // Index big outer rings that many inner rings must be tested against + std::unique_ptr index; + if (!onlyOneOuter && filledInners.size() >= 16 && poly.outer().size() >= 4096) + index.reset(new RingWithinIndex(poly.outer())); + for (std::size_t i = 0; i < filledInners.size(); i++) { + const Ring &inner = filledInners[i]; + // A ring within the outer has its bounding box within the outer's, + // so the (costly) within test is only needed when it does + if (onlyOneOuter || + ((inner.empty() || geom::covered_by(innerBoxes[i], outerBox)) && + (index ? index->within(inner) : geom::within(inner, poly.outer())))) { + poly.inners().emplace_back(inner); + } } mp.emplace_back(move(poly)); } diff --git a/src/output_object.cpp b/src/output_object.cpp index 3d5703ca..bc4c1855 100644 --- a/src/output_object.cpp +++ b/src/output_object.cpp @@ -42,8 +42,8 @@ static void writeAttributesTo( FeatureBuilder& fbuilder, char zoom ) { - attributeStore.forEachPairUnsafe(attributes, [&](const AttributePair& pair) { - const AttributePair* it = &pair; + attributeStore.forEachPairIndexCanonicalUnsafe(attributes, [&](uint32_t pairIndex) { + const AttributePair* it = &attributeStore.pairStore.getPairUnsafe(pairIndex); if (it->minzoom > zoom) return; // TODO: consider taking a data view that is stable @@ -179,7 +179,7 @@ const uint32_t* MvtAttributeMemo::resolve(const AttributeStore& attributeStore, if (pairs.used >= MEMO_MAX_PAIRS) pairs.clear(); const uint32_t begin = entries.size(); - attributeStore.forEachPairIndexUnsafe(index, [&](uint32_t pairIndex) { + attributeStore.forEachPairIndexCanonicalUnsafe(index, [&](uint32_t pairIndex) { uint32_t key, value; if (const Table::Slot* s = pairs.find(pairIndex)) { key = s->a; diff --git a/src/pmtiles.cpp b/src/pmtiles.cpp index c9561b59..dceb5af4 100644 --- a/src/pmtiles.cpp +++ b/src/pmtiles.cpp @@ -10,11 +10,19 @@ #include "helpers.h" TileOffset::TileOffset() { } -PMTiles::PMTiles() { } +// Placing tiles (offset, repeats, index) and writing their bytes are two steps, each in batch order +// and one thread at a time, but run by different threads side by side: one thread doing both was +// the bottleneck of the tile phase at 96 threads. +PMTiles::PMTiles() : + batches([this](PendingTile &tile) { writeTile(tile); }, + [](const PendingTile &tile) { return tile.data.size(); }, + [this](PendingTile &tile) { flushTile(tile); }) { } PMTiles::~PMTiles() { } void PMTiles::open(std::string &filename) { std::cout << "Creating pmtiles at " << filename << std::endl; + outputBuffer.resize(4 << 20); + outputStream.rdbuf()->pubsetbuf(outputBuffer.data(), outputBuffer.size()); outputStream.open(filename, std::ios::out | std::ios::trunc | std::ios::binary); // dummy header/root directory for now - we'll write it all later char header[HEADER_ROOT] = "PMTiles"; @@ -24,6 +32,7 @@ void PMTiles::open(std::string &filename) { // Finish writing the .pmtiles file void PMTiles::close(std::string &metadata) { std::cout << "\nClosing pmtiles file" << std::flush; + if (!batches.allWritten()) { throw std::runtime_error("PMTiles: a batch of tiles was never written - please file an issue"); } // add all tiles to directories, writing leaf directories as we go std::vector rootEntries; @@ -129,34 +138,50 @@ void PMTiles::flushEntries(std::vector &rootEntries, std::vect rootEntries.emplace_back(rootEntry); } -// Write a tile to file and store it in the index -// - if the tile is small and has already been written, reuse that instead -// - we're a bit fussy about mutexs because compress_string is expensive +// Save a tile: compress it here (compress_string is expensive), but leave tiny tiles +// raw until they're written, since most of them (e.g. sea) repeat an earlier one void PMTiles::saveTile(int zoom, int x, int y, std::string &data) { - TileOffset offset; - bool isNew = false; - uint64_t tileId = pmtiles::zxy_to_tileid(zoom,x,y); + PendingTile tile; + tile.tileId = pmtiles::zxy_to_tileid(zoom,x,y); + tile.isTiny = data.size() lock(batches.writeMutex()); + writeTile(tile); + flushTile(tile); +} - // if it's a tiny tile (e.g. sea), see if we've written it already - std::unique_lock indexLock1(indexMutex); - if (data.size()second; + tile.data = std::string(); // already in the file } else { - indexLock1.unlock(); - std::string compressed = compress_string(data, Z_DEFAULT_COMPRESSION, true); - std::lock_guard lock(fileMutex); - // write to file - offset = TileOffset(static_cast(outputStream.tellp()) - HEADER_ROOT, compressed.size()); - outputStream.write(compressed.c_str(), compressed.size()); + std::string compressed = tile.isTiny ? compress_string(tile.data, Z_DEFAULT_COMPRESSION, true) : std::move(tile.data); + offset = TileOffset(tileDataEnd, compressed.size()); + tileDataEnd += compressed.size(); numTilesWritten++; - isNew = true; + // add to cache if tiny + if (tile.isTiny) { + if (tinyCache.size()>TINY_MAX_SIZE) tinyCache.clear(); + tinyCache.insert({ tile.data, offset }); + } + tile.data = std::move(compressed); // flushTile writes it } - + // store in index - std::lock_guard indexLock2(indexMutex); + uint64_t tileId = tile.tileId; numTilesAddressed++; if (isSparse) { sparseIndex[tileId] = offset; @@ -164,10 +189,4 @@ void PMTiles::saveTile(int zoom, int x, int y, std::string &data) { if (tileId >= denseIndex.size()) denseIndex.resize(tileId+10000, { 0, 0xffffff }); denseIndex[tileId] = offset; } - - // add to cache if tiny - if (isNew && data.size()TINY_MAX_SIZE) tinyCache.clear(); - tinyCache.insert({ data, offset }); - } } diff --git a/src/shp_mem_tiles.cpp b/src/shp_mem_tiles.cpp index e5c69e52..fd673e65 100644 --- a/src/shp_mem_tiles.cpp +++ b/src/shp_mem_tiles.cpp @@ -78,14 +78,58 @@ bool ShpMemTiles::mayIntersect(const std::string& layerName, const Box& box) con uint32_t y1 = latp2tiley(latp1, spatialIndexZoom); uint32_t y2 = latp2tiley(latp2, spatialIndexZoom); - for (int x = std::min(x1, x2); x <= std::min((1u << spatialIndexZoom) - 1u, std::max(x1, x2)); x++) { - for (int y = std::min(y1, y2); y <= std::min((1u << spatialIndexZoom) - 1u, std::max(y1, y2)); y++) { - uint32_t z6x = x / (1u << (spatialIndexZoom - CLUSTER_ZOOM)); - uint32_t z6y = y / (1u << (spatialIndexZoom - CLUSTER_ZOOM)); + const uint32_t xMin = std::min(x1, x2), xMax = std::min((1u << spatialIndexZoom) - 1u, std::max(x1, x2)); + const uint32_t yMin = std::min(y1, y2), yMax = std::min((1u << spatialIndexZoom) - 1u, std::max(y1, y2)); + const uint32_t divisor = 1u << (spatialIndexZoom - CLUSTER_ZOOM); + + // First pass, without any geometry checks: is a tile in the box already known to + // intersect, and how many tiles still need the exact check below? + size_t unchecked = 0; + for (uint32_t x = xMin; x <= xMax; x++) { + for (uint32_t y = yMin; y <= yMax; y++) { + uint32_t z6x = x / divisor, z6y = y / divisor; + auto& bitvec = sparseLayerVector[z6x * CLUSTER_ZOOM_WIDTH + z6y]; + uint64_t index = 2u * ((x - z6x * divisor) * divisor + (y - z6y * divisor)); + if (!bitvec.empty() && bitvec[index]) { + if (bitvec[index + 1]) return true; + unchecked++; + } + } + } + if (unchecked == 0) return false; + + // Each tile's exact check intersects the tile with every candidate geometry, so its + // cost grows with the size of those geometries. A large query box that lies inside + // the bounding box of a huge geometry without touching it (say, an offshore marine + // reserve inside a country's bounding box) would check every one of thousands of + // tiles, and find nothing. Instead, check the query box once: if no geometry + // intersects it, none intersects any tile in it. Otherwise the caller's exact check + // decides. + const size_t maxTileChecks = 16; + if (unchecked > maxTileChecks) { + Box queryBox = box; + return !QueryMatchingGeometries( + layerName, + true, + queryBox, + [&](const RTree &rtree) { // indexQuery + vector results; + rtree.query(geom::index::intersects(queryBox), back_inserter(results)); + return results; + }, + [&](OutputObject const &oo) { // checkQuery + return geom::intersects(queryBox, retrieveMultiPolygon(oo.objectID)); + } + ).empty(); + } + + for (uint32_t x = xMin; x <= xMax; x++) { + for (uint32_t y = yMin; y <= yMax; y++) { + uint32_t z6x = x / divisor; + uint32_t z6y = y / divisor; auto& bitvec = sparseLayerVector[z6x * CLUSTER_ZOOM_WIDTH + z6y]; - uint32_t divisor = 1u << (spatialIndexZoom - CLUSTER_ZOOM); uint64_t index = 2u * ((x - z6x * divisor) * divisor + (y - z6y * divisor)); if (!bitvec.empty() && bitvec[index]) { @@ -137,7 +181,8 @@ void ShpMemTiles::StoreGeometry( const std::string& name, uint minzoom, int32_t score, - AttributeIndex attrIdx + AttributeIndex attrIdx, + bool knownValid ) { geom::model::box box; @@ -175,9 +220,9 @@ void ShpMemTiles::StoreGeometry( case POLYGON_: { - // Checked once here, so tiles can clip valid polygons on the fast path + // Checked once here (unless the caller has), so tiles can clip valid polygons on the fast path const MultiPolygon &mp = boost::get(geometry); - NodeID oid = storeMultiPolygon(mp, multiPolygonDefinitelyValid(mp) || geom::is_valid(mp)); + NodeID oid = storeMultiPolygon(mp, knownValid || multiPolygonDefinitelyValid(mp) || geom::is_valid(mp)); oo = std::make_shared(geomType, layerNum, oid, attrIdx, minzoom); std::vector oolist { *oo }; addGeometryToIndex(boost::get(geometry), oolist, 0); diff --git a/src/shp_processor.cpp b/src/shp_processor.cpp index f3b3c4f2..b32291b2 100644 --- a/src/shp_processor.cpp +++ b/src/shp_processor.cpp @@ -366,8 +366,8 @@ void ShpProcessor::processShapeGeometry(SHPObject* shape, AttributeIndex attrIdx // Make valid if needs be string reason; - const bool precheckValid = multiPolygonDefinitelyValid(multi); - if (!precheckValid && !geom::is_valid(multi, reason)) { + const bool valid = multiPolygonDefinitelyValid(multi) || geom::is_valid(multi, reason); + if (!valid) { if (verbose) cerr << "Shapefile entity " << shape->nShapeId << " type " << shapeType << " is invalid. Parts:" << shape->nParts << ". Reason:" << reason; make_valid(multi); if (verbose) { @@ -376,11 +376,17 @@ void ShpProcessor::processShapeGeometry(SHPObject* shape, AttributeIndex attrIdx cerr << endl; } } - // clip to bounding box - MultiPolygon out; - geom::intersection(multi, clippingBox, out); - if (boost::size(out)>0) { - shpMemTiles.StoreGeometry(layerNum, layer.name, POLYGON_, out, layer.indexed, hasName, name, minzoom, score, attrIdx); + // clip to bounding box (a valid polygon wholly inside it is stored as it is) + Box envelope; + geom::envelope(multi, envelope); + if (valid && geom::covered_by(envelope, clippingBox)) { + shpMemTiles.StoreGeometry(layerNum, layer.name, POLYGON_, multi, layer.indexed, hasName, name, minzoom, score, attrIdx, true); + } else { + MultiPolygon out; + geom::intersection(multi, clippingBox, out); + if (boost::size(out)>0) { + shpMemTiles.StoreGeometry(layerNum, layer.name, POLYGON_, out, layer.indexed, hasName, name, minzoom, score, attrIdx); + } } } else { diff --git a/src/tile_coordinates_set.cpp b/src/tile_coordinates_set.cpp index e6d6e78e..7a25a1ba 100644 --- a/src/tile_coordinates_set.cpp +++ b/src/tile_coordinates_set.cpp @@ -40,6 +40,12 @@ void TileCoordinatesSet::forEachSet(const std::function> 12] |= (uint64_t)1 << ((w >> 6) & 63); } +// Fills each row of the box word by word, rather than tile by tile. +void PreciseTileCoordinatesSet::setBox(TileCoordinate minX, TileCoordinate maxX, TileCoordinate minY, TileCoordinate maxY) { + const uint64_t dim = 1ull << zoom_; + if (minX > maxX || minY > maxY || minX >= dim || minY >= dim) + return; + maxX = std::min(maxX, dim - 1); + maxY = std::min(maxY, dim - 1); + for (uint64_t x = minX; x <= maxX; x++) { + const uint64_t first = x * dim + minY, last = x * dim + maxY; + const uint64_t w0 = first >> 6, w1 = last >> 6; + for (uint64_t w = w0; w <= w1; w++) { + uint64_t mask = ~0ull; + if (w == w0) mask &= ~0ull << (first & 63); + if (w == w1) mask &= ~0ull >> (63 - (last & 63)); + tiles[w] |= mask; + } + for (uint64_t block = w0 >> 6; block <= (w1 >> 6); block++) + summary[block >> 6] |= (uint64_t)1 << (block & 63); + } +} + +void PreciseTileCoordinatesSet::merge(const PreciseTileCoordinatesSet& other) { + if (other.zoom_ != zoom_) + throw std::invalid_argument("PreciseTileCoordinatesSet::merge: zooms differ"); + forEachMarkedWord(other.summary, numWords, [&](size_t w) { tiles[w] |= other.tiles[w]; }); + for (size_t s = 0; s < summary.size(); s++) + summary[s] |= other.summary[s]; +} + void PreciseTileCoordinatesSet::forEachSet(const std::function& f) const { // loc = x * 2^zoom + y, so ascending loc order is x-major, y-minor. // Only blocks marked in the summary can hold set bits; visiting them in @@ -134,3 +169,24 @@ void LossyTileCoordinatesSet::set(TileCoordinate x, TileCoordinate y) { throw std::logic_error("LossyTileCoordinatesSet::set() is not implemented; LossyTileCoordinatesSet is read-only"); } +void LossyTileCoordinatesSet::forEachSet(const std::function& f) const { + // Expand the underlying set instead of testing all 4^zoom tiles (2^34 at z17). + // Its tiles arrive x-major, y-minor; one column of them becomes `scale` + // columns here, which keeps this set's order x-major, y-minor too. + std::vector column; + TileCoordinate columnX = 0; + auto flush = [&]() { + for (TileCoordinate dx = 0; dx < scale; dx++) + for (TileCoordinate y : column) + for (TileCoordinate dy = 0; dy < scale; dy++) + f(columnX * scale + dx, y * scale + dy); + column.clear(); + }; + tiles.forEachSet([&](TileCoordinate x, TileCoordinate y) { + if (!column.empty() && x != columnX) flush(); + columnX = x; + column.push_back(y); + }); + if (!column.empty()) flush(); +} + diff --git a/src/tile_data.cpp b/src/tile_data.cpp index 89b367e0..866b853a 100644 --- a/src/tile_data.cpp +++ b/src/tile_data.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -6,6 +7,8 @@ #include "tile_data.h" #include "coordinates_geom.h" #include "leased_store.h" +#include +#include "stable_hash.h" #include using namespace std; @@ -46,6 +49,34 @@ thread_local LeasedStore linestringStore; thread_local LeasedStore multilinestringStore; thread_local LeasedStore multipolygonStore; +// Never destroyed: the thread-locals that use them may be destroyed after the statics (macOS) +static std::unordered_map &leaseSources() { + static std::unordered_map &sources = *new std::unordered_map(); + return sources; +} +std::mutex &leaseSourcesMutex() { + static std::mutex &m = *new std::mutex; + return m; +} +uint64_t registerLeaseSource(const TileDataSource *source) { + static uint64_t last = 0; + std::lock_guard lock(leaseSourcesMutex()); + const uint64_t number = ++last; + leaseSources()[source] = number; + return number; +} +void unregisterLeaseSource(const TileDataSource *source) { + std::lock_guard lock(leaseSourcesMutex()); + leaseSources().erase(source); +} +bool leaseSourceAlive(const TileDataSource *source, uint64_t number) { + const auto it = leaseSources().find(source); + return it != leaseSources().end() && it->second == number; +} +TileDataSource::~TileDataSource() { + unregisterLeaseSource(this); +} + TileDataSource::TileDataSource(size_t threadNum, unsigned int indexZoom, bool includeID) : includeID(includeID), @@ -88,6 +119,116 @@ TileDataSource::TileDataSource(size_t threadNum, unsigned int indexZoom, bool in thread_local std::vector>* tlsPendingSmallIndexObjects = nullptr; thread_local uint64_t clipCacheBatchId = 0; thread_local bool clipCacheNoStore = false; +thread_local bool clipCacheBypass = false; +thread_local bool clipNeededRepair = false; // set when a POLYGON_ clip went down the repair path +thread_local bool clipResultValid = false; // whether the last POLYGON_ clip is known to be valid + +// Shared pieces. Every tile batch that needs a large polygon (often 100,000+ points) used to copy +// it from the store and clip it from scratch, at every zoom. Instead, polygons with at least +// SHARED_PIECE_MIN_POINTS points are clipped once per SHARED_PIECE_ZOOM tile before the tiles are +// written, and tiles below clip from that piece. A piece only depends on the polygon and the tile, +// never on the thread or the order, so the output stays the same for any thread count. +// +// The pieces are known up front and computed by all threads, largest polygon first, before they +// start on tiles: computed on demand, a slow piece (up to ~15 s on the planet) held up every +// thread whose tiles needed it, a third of the tile phase on a 96-thread planet run. A thread +// that runs out of pieces to start goes on to tiles; only a tile that needs a piece still being +// computed waits for it. The map itself doesn't change, so the tiles look pieces up without a lock. +#define SHARED_PIECE_MIN_POINTS 10000 + +void TileDataSource::precomputeSharedPieces(const std::function &present, unsigned threadNum) { + const unsigned zoom = SHARED_PIECE_ZOOM; + if (zoom > indexZoom) return; + const unsigned shift = indexZoom - zoom; + // Candidates: polygons in the large-object index (those spanning 16+ index tiles), with the + // index tiles they cover + std::map boxes; + auto add = [&](const Box &box, const OutputObject &oo) { + if (oo.geomType != POLYGON_ || !storesMultiPolygon(oo.objectID)) return; + auto it = boxes.find(oo.objectID); + if (it == boxes.end()) boxes.emplace(oo.objectID, box); + else boost::geometry::expand(it->second, box); + }; + for (auto const &r : boxRtree) add(r.first, r.second); + for (auto const &r : boxRtreeWithIds) add(r.first, r.second.oo); + if (boxes.empty()) return; + const std::vector> candidates(boxes.begin(), boxes.end()); + + // The pieces each candidate needs: tiles below which a tile is written, that its envelope reaches + using Job = PieceJob; + std::vector jobs; + std::mutex jobsMutex; + std::atomic next(0); + auto findJobs = [&]() { + std::vector local; + for (size_t i = next++; i < candidates.size(); i = next++) { + const NodeID objectID = candidates[i].first; + uint64_t points = 0; + Box envelope; + boost::geometry::assign_inverse(envelope); + for (auto const &p : retrieveMultiPolygon(objectID)) { + points += p.outer().size(); + for (auto const &inner : p.inners()) points += inner.size(); + for (auto const &pt : p.outer()) boost::geometry::expand(envelope, pt); + } + if (points < SHARED_PIECE_MIN_POINTS) continue; + const Box &box = candidates[i].second; + const TileCoordinate x0 = (TileCoordinate)box.min_corner().x() >> shift, x1 = (TileCoordinate)box.max_corner().x() >> shift; + const TileCoordinate y0 = (TileCoordinate)box.min_corner().y() >> shift, y1 = (TileCoordinate)box.max_corner().y() >> shift; + for (TileCoordinate x = x0; x <= x1; x++) + for (TileCoordinate y = y0; y <= y1; y++) + if (present(x, y) && boost::geometry::intersects(envelope, TileBbox(TileCoordinates(x, y), zoom, false, false).clippingBox)) + local.push_back(Job { std::make_tuple(objectID, x, y), points, nullptr }); + } + std::lock_guard lock(jobsMutex); + jobs.insert(jobs.end(), local.begin(), local.end()); + }; + { + boost::asio::thread_pool pool(threadNum); + for (unsigned t = 0; t < threadNum; t++) boost::asio::post(pool, findJobs); + pool.join(); + } + std::sort(jobs.begin(), jobs.end(), [](const Job &a, const Job &b) { return a.points != b.points ? a.points > b.points : a.key < b.key; }); + for (Job &job : jobs) { + pieceObjects.insert(std::get<0>(job.key)); + job.promise = std::make_shared>>(); + sharedPieces.emplace(job.key, job.promise->get_future().share()); + } + pieceJobs = std::move(jobs); + nextPieceJob = 0; +} + +void TileDataSource::computeSharedPieces() { + const unsigned zoom = SHARED_PIECE_ZOOM; + for (size_t i = nextPieceJob++; i < pieceJobs.size(); i = nextPieceJob++) { + const auto &key = pieceJobs[i].key; + auto piece = std::make_shared(); + try { + clipCacheBypass = true; + clipNeededRepair = false; + Geometry g = buildWayGeometry(POLYGON_, std::get<0>(key), TileBbox(TileCoordinates(std::get<1>(key), std::get<2>(key)), zoom, false, false)); + // A piece that needed the repair path can be wrong (once a lake came out empty), and + // every tile below it would inherit that: leave those tiles to clip the whole polygon. + if (!clipNeededRepair) { + piece->geom = std::make_shared(std::move(boost::get(g))); + // buildWayGeometry has just checked it (for a huge piece, is_valid takes seconds) + piece->valid = clipResultValid; + } + } catch (std::exception &) { + // The tiles clip the whole polygon, and report the problem there + } + clipCacheBypass = false; + pieceJobs[i].promise->set_value(piece); + pieceJobs[i].promise.reset(); // the map's future keeps the piece, until its z6 tile is released + } +} + +void TileDataSource::releaseSharedPieces(TileCoordinate z6x, TileCoordinate z6y) { + const unsigned shift = SHARED_PIECE_ZOOM - CLUSTER_ZOOM; + for (auto &e : sharedPieces) + if ((std::get<1>(e.first) >> shift) == z6x && (std::get<2>(e.first) >> shift) == z6y) + e.second = std::shared_future>(); +} void TileDataSource::finalize(size_t threadNum) { uint64_t finalized = 0; @@ -166,6 +307,13 @@ void TileDataSource::collectTilesWithObjectsAtZoom(std::vector(indexZoom, objectsWithIds.begin(), objectsWithIds.size(), zooms); } +void TileDataSource::collectTilesWithObjectsInShard(size_t shard, std::vector>& zooms) { + if (shard < objects.size()) + collectTilesWithObjectsInShardTemplate(indexZoom, objects[shard], shard, zooms); + if (shard < objectsWithIds.size()) + collectTilesWithObjectsInShardTemplate(indexZoom, objectsWithIds[shard], shard, zooms); +} + void addCoveredTilesToOutput(const uint indexZoom, std::vector>& zooms, const Box& box) { size_t maxZoom = zooms.size() - 1; // std::cout << "addCoveredTilesToOutput maxZoom=" << maxZoom << ", indexZoom - maxZoom = " << (indexZoom - maxZoom) << std::endl; @@ -174,16 +322,11 @@ void addCoveredTilesToOutput(const uint indexZoom, std::vector= 0; zoom--) { - zooms[zoom]->set(zx, zy); - zx /= 2; - zy /= 2; - } - } + // The parents of the tiles in the box are the tiles in the box scaled down + for (int zoom = maxZoom; zoom >= 0; zoom--) { + zooms[zoom]->setBox(minx, maxx, miny, maxy); + minx /= 2; maxx /= 2; + miny /= 2; maxy /= 2; } } @@ -329,6 +472,18 @@ Geometry TileDataSource::buildWayGeometry(OutputGeometryType const geomType, auto cachedEntry = multiPolygonClipCache.getEntry(bbox.zoom, bbox.index.x, bbox.index.y, objectID); std::shared_ptr cachedClip = cachedEntry.geom; bool inputKnownValid = cachedClip == nullptr ? polygonKnownValid(objectID) : cachedEntry.valid; + if (cachedClip == nullptr && bbox.zoom > SHARED_PIECE_ZOOM && pieceObjects.count(objectID)) { + // A large polygon: clip from its shared piece + const unsigned shift = bbox.zoom - SHARED_PIECE_ZOOM; + auto it = sharedPieces.find(std::make_tuple(objectID, bbox.index.x >> shift, bbox.index.y >> shift)); + if (it != sharedPieces.end() && it->second.valid()) { + const std::shared_ptr &piece = it->second.get(); // waits if still being computed + if (piece && piece->geom) { + cachedClip = piece->geom; + inputKnownValid = piece->valid; + } + } + } MultiPolygon uncached; @@ -385,7 +540,13 @@ Geometry TileDataSource::buildWayGeometry(OutputGeometryType const geomType, } MultiPolygon mp; - if (cachedClip == nullptr) + // rect_clip below needs the input as it was before fast_clip. For a large valid + // polygon, keep it rather than load it from the store again for every tile. + std::size_t inputPoints = 0; + if (cachedClip == nullptr && inputKnownValid) + for (auto const &p : uncached) inputPoints += geom::num_points(p); + const bool keepUncached = inputPoints > 1000; + if (cachedClip == nullptr && !keepUncached) mp = std::move(uncached); else geom::assign(mp, input); @@ -394,6 +555,7 @@ Geometry TileDataSource::buildWayGeometry(OutputGeometryType const geomType, // fast_clip was the identity on a geometry that already passed // is_valid, so correct() is a no-op and it is still valid. multiPolygonClipCache.add(bbox, objectID, mp, true); + clipResultValid = true; return mp; } geom::correct(mp); @@ -405,17 +567,19 @@ Geometry TileDataSource::buildWayGeometry(OutputGeometryType const geomType, // instead, which is much cheaper than the is_valid/intersection/repair // path below; rect_clip bails out on anything it can't handle. MultiPolygon original; - if (cachedClip == nullptr) populateMultiPolygon(original, objectID); + if (cachedClip == nullptr && !keepUncached) populateMultiPolygon(original, objectID); MultiPolygon clipped; - if (rect_clip(cachedClip == nullptr ? original : input, box, clipped) && + if (rect_clip(cachedClip == nullptr && !keepUncached ? original : input, box, clipped) && (multiPolygonDefinitelyValid(clipped) || geom::is_valid(clipped))) { multiPolygonClipCache.add(bbox, objectID, clipped, true); + clipResultValid = true; return clipped; } } if (!valid) valid = geom::is_valid(mp,failure); if (!valid) { + clipNeededRepair = true; if (verbose) { // Build the whole line first and emit it with a single stream write: // tilemaker runs multi-threaded and chained operator<< calls are not @@ -451,6 +615,7 @@ Geometry TileDataSource::buildWayGeometry(OutputGeometryType const geomType, repairMultiPolygon(output, objectID, &outputValid); // Children of this tile can then take the fast paths for valid input multiPolygonClipCache.add(bbox, objectID, output, outputValid); + clipResultValid = outputValid; return output; } else if (!valid) { // occasionally also wrong_topological_dimension, disconnected_interior: @@ -461,6 +626,7 @@ Geometry TileDataSource::buildWayGeometry(OutputGeometryType const geomType, } multiPolygonClipCache.add(bbox, objectID, mp, valid); + clipResultValid = valid; return mp; } @@ -498,35 +664,172 @@ void TileDataSource::reportSize() const { void populateTilesAtZoom( const std::vector& sources, - std::vector>& zooms + std::vector>& zooms, + size_t threadNum ) { if (zooms.size() > 15) throw std::out_of_range("populateTilesAtZoom: expected at most z14 zooms (15), but found " + std::to_string(zooms.size()) + " vectors"); - for(size_t i=0; icollectTilesWithObjectsAtZoom(zooms); - sources[i]->collectTilesWithLargeObjectsAtZoom(zooms); + std::vector precise; + for (const auto& zoom : zooms) + precise.push_back(dynamic_cast(zoom.get())); + const bool allPrecise = std::find(precise.begin(), precise.end(), nullptr) == precise.end(); + + if (threadNum <= 1 || !allPrecise) { + for (size_t i = 0; i < sources.size(); i++) + sources[i]->collectTilesWithObjectsAtZoom(zooms); + } else { + // Objects are kept per z6 tile. Each worker takes whole columns of z6 + // tiles (so that it touches only its part of the bitmaps), sets bits in + // bitmaps of its own, and these are merged at the end. + std::atomic nextColumn(0); + std::vector>> local(threadNum); + boost::asio::thread_pool pool(threadNum); + for (size_t t = 0; t < threadNum; t++) { + boost::asio::post(pool, [&, t]() { + auto& mine = local[t]; + for (size_t zoom = 0; zoom < zooms.size(); zoom++) + mine.push_back(std::make_shared(zoom)); + for (size_t column; (column = nextColumn++) < CLUSTER_ZOOM_WIDTH; ) + for (size_t row = 0; row < CLUSTER_ZOOM_WIDTH; row++) + for (const auto source : sources) + source->collectTilesWithObjectsInShard(column * CLUSTER_ZOOM_WIDTH + row, mine); + }); + } + pool.join(); + for (const auto& mine : local) + for (size_t zoom = 0; zoom < mine.size(); zoom++) + precise[zoom]->merge(static_cast(*mine[zoom])); } + + for (size_t i = 0; i < sources.size(); i++) + sources[i]->collectTilesWithLargeObjectsAtZoom(zooms); } void sortOutputObjectIDs( const std::vector& sortOrders, + const AttributeStore& attributeStore, std::vector& data ); std::vector TileDataSource::getObjectsForTile( const std::vector& sortOrders, + const AttributeStore& attributeStore, unsigned int zoom, TileCoordinates coordinates ) { std::vector data; collectObjectsForTile(zoom, coordinates, data); collectLargeObjectsForTile(zoom, coordinates, data); - sortOutputObjectIDs(sortOrders, data); + sortOutputObjectIDs(sortOrders, attributeStore, data); data.erase(unique(data.begin(), data.end()), data.end()); + orderStoredGeometryTies(data); return data; } +namespace { +struct GeometryTieKey { + double x, y; // first point + uint64_t points; // number of points + uint64_t hash; // of all points, only computed when the above are equal + OutputObjectID o; +}; + +// Adds a ring/linestring to the key: first point and point count, or (withHash) the hash of all points +template +void addToTieKey(GeometryTieKey& key, const R& ring, bool withHash) { + if (withHash) { + key.hash = stableCombine(key.hash, static_cast(ring.size())); + for (const auto& p : ring) { + key.hash = stableCombine(key.hash, static_cast(p.x())); + key.hash = stableCombine(key.hash, static_cast(p.y())); + } + return; + } + if (key.points == 0 && !ring.empty()) { + key.x = ring.front().x(); + key.y = ring.front().y(); + } + key.points += ring.size(); +} + +void geometryTieKey(const TileDataSource& source, GeometryTieKey& key, bool withHash) { + const NodeID objectID = key.o.oo.objectID; + switch (key.o.oo.geomType) { + case POINT_: { + const Point& p = source.retrievePoint(objectID); + addToTieKey(key, std::array{p}, withHash); + break; + } + case LINESTRING_: + addToTieKey(key, source.retrieveLinestring(objectID), withHash); + break; + case MULTILINESTRING_: + for (const auto& ls : source.retrieveMultiLinestring(objectID)) + addToTieKey(key, ls, withHash); + break; + case POLYGON_: + for (const auto& poly : source.retrieveMultiPolygon(objectID)) { + addToTieKey(key, poly.outer(), withHash); + for (const auto& inner : poly.inners()) + addToTieKey(key, inner, withHash); + } + break; + } +} +} + +// Geometries stored at runtime (relations, shapefiles, ...) have an objectID that +// depends on which thread stored them, so the sort ties objects that share layer, +// z_order, type, attributes and id in an arbitrary order. Order such runs by +// geometry instead. OSM ways and nodes carry their OSM id in the objectID, and +// sort after stored geometries, so they never need this. +void TileDataSource::orderStoredGeometryTies(std::vector& data) const { + const NodeID storedLimit = 1ull << TILE_DATA_ID_SIZE; + auto tied = [](const OutputObjectID& a, const OutputObjectID& b) { + return a.oo.layer == b.oo.layer && a.oo.z_order == b.oo.z_order && a.oo.geomType == b.oo.geomType && + a.oo.attributes == b.oo.attributes && a.id == b.id; + }; + + std::vector keys; + size_t i = 0; + while (i < data.size()) { + size_t j = i + 1; + if (data[i].oo.objectID < storedLimit) + while (j < data.size() && data[j].oo.objectID < storedLimit && tied(data[i], data[j])) j++; + if (j - i < 2) { i = j; continue; } + + keys.clear(); + for (size_t k = i; k < j; k++) { + keys.push_back(GeometryTieKey{0, 0, 0, 0, data[k]}); + geometryTieKey(*this, keys.back(), false); + } + auto byPoint = [](const GeometryTieKey& a, const GeometryTieKey& b) { + if (a.x != b.x) return a.x < b.x; + if (a.y != b.y) return a.y < b.y; + return a.points < b.points; + }; + std::sort(keys.begin(), keys.end(), byPoint); + + // Still tied: hash all points (identical geometries may stay in any order) + for (size_t a = 0; a < keys.size(); ) { + size_t b = a + 1; + while (b < keys.size() && !byPoint(keys[a], keys[b])) b++; + if (b - a > 1) { + for (size_t k = a; k < b; k++) geometryTieKey(*this, keys[k], true); + std::sort(keys.begin() + a, keys.begin() + b, [](const GeometryTieKey& p, const GeometryTieKey& q) { + if (p.hash != q.hash) return p.hash < q.hash; + return p.o.oo.objectID < q.o.oo.objectID; + }); + } + a = b; + } + + for (size_t k = i; k < j; k++) data[k] = keys[k - i].o; + i = j; + } +} + // ------------------------------------ // Add geometries to tile/large indices @@ -660,7 +963,7 @@ NodeID TileDataSource::storePoint(const Point& input) { const auto& store = pointStore.get(this); NodeID offset = store.second->size(); - store.second->emplace_back(input); + store.second->push_back(input); NodeID rv = (store.first << (TILE_DATA_ID_SIZE - shardBits)) + offset; return rv; } diff --git a/src/tile_sorting.cpp b/src/tile_sorting.cpp index 64588163..3bb9442f 100644 --- a/src/tile_sorting.cpp +++ b/src/tile_sorting.cpp @@ -208,26 +208,75 @@ template void sortOutputObjects( typename AppendVectorNS::AppendVector::Iterator end ); +namespace { +struct AttributeKeyCacheEntry { + AttributeIndex index; + uint64_t key; +}; +// Direct-mapped, per thread: neighbouring tiles share most attribute sets +const size_t ATTRIBUTE_KEY_CACHE_SIZE = 1 << 12; +thread_local std::vector tlsAttributeKeys; +thread_local const AttributeStore* tlsAttributeKeysStore = nullptr; + +struct KeyedOutputObjectID { + uint64_t attributeKey; + OutputObjectID o; +}; + +uint64_t attributeKey(const AttributeStore& attributeStore, AttributeIndex index) { + if (tlsAttributeKeysStore != &attributeStore) { + tlsAttributeKeys.assign(ATTRIBUTE_KEY_CACHE_SIZE, AttributeKeyCacheEntry{~0u, 0}); + tlsAttributeKeysStore = &attributeStore; + } + uint32_t slot = index; + slot ^= slot >> 16; slot *= 0x7feb352dU; slot ^= slot >> 15; + AttributeKeyCacheEntry& entry = tlsAttributeKeys[slot & (ATTRIBUTE_KEY_CACHE_SIZE - 1)]; + if (entry.index != index) + entry = AttributeKeyCacheEntry{index, attributeStore.contentHashUnsafe(index)}; + return entry.key; +} +} + void sortOutputObjectIDs( const std::vector& sortOrders, + const AttributeStore& attributeStore, std::vector& data ) { - // Lexicographic comparison, with the order of: layer, geomType, attributes, and objectID. + // Lexicographic comparison, with the order of: layer, geomType, attributes, id and objectID. // Note that attributes is preferred to objectID. // It is to arrange objects with the identical attributes continuously. // Such objects will be merged into one object, to reduce the size of output. - boost::sort::pdqsort(data.begin(), data.end(), [&sortOrders](const OutputObjectID& x, const OutputObjectID& y) -> bool { - if (x.oo.layer < y.oo.layer) return true; - if (x.oo.layer > y.oo.layer) return false; - if (x.oo.z_order < y.oo.z_order) return sortOrders[x.oo.layer]; - if (x.oo.z_order > y.oo.z_order) return !sortOrders[x.oo.layer]; - if (x.oo.geomType < y.oo.geomType) return true; - if (x.oo.geomType > y.oo.geomType) return false; - if (x.oo.attributes < y.oo.attributes) return true; - if (x.oo.attributes > y.oo.attributes) return false; - if (x.oo.objectID < y.oo.objectID) return true; - return false; + // + // Attribute sets are compared by content, not by AttributeIndex: the index is + // handed out in insertion order, which depends on thread scheduling. The same + // goes for the objectID of stored geometries, so the OSM id goes first; + // TileDataSource::orderStoredGeometryTies handles the remaining ties. + std::vector keyed; + keyed.reserve(data.size()); + for (const OutputObjectID& o : data) + keyed.push_back(KeyedOutputObjectID{attributeKey(attributeStore, o.oo.attributes), o}); + + boost::sort::pdqsort(keyed.begin(), keyed.end(), [&sortOrders, &attributeStore](const KeyedOutputObjectID& a, const KeyedOutputObjectID& b) -> bool { + const OutputObject& x = a.o.oo; + const OutputObject& y = b.o.oo; + if (x.layer < y.layer) return true; + if (x.layer > y.layer) return false; + if (x.z_order < y.z_order) return sortOrders[x.layer]; + if (x.z_order > y.z_order) return !sortOrders[x.layer]; + if (x.geomType < y.geomType) return true; + if (x.geomType > y.geomType) return false; + if (x.attributes != y.attributes) { + if (a.attributeKey != b.attributeKey) return a.attributeKey < b.attributeKey; + const int c = attributeStore.compareContentUnsafe(x.attributes, y.attributes); + if (c != 0) return c < 0; + return x.attributes < y.attributes; + } + if (a.o.id != b.o.id) return a.o.id < b.o.id; + return x.objectID < y.objectID; }); + + for (size_t i = 0; i < keyed.size(); i++) + data[i] = keyed[i].o; } void sortTileCoordinates( @@ -286,5 +335,7 @@ void sortTileCoordinates( return false; }, - threadNum); + // block_indirect_sort slows down badly with many threads: on the planet (346M tiles) it took + // 200-300 s at 96 threads instead of about 7 s at 16, with the threads spinning on its internal lock. + std::min(threadNum, 16)); } \ No newline at end of file diff --git a/src/tilemaker.cpp b/src/tilemaker.cpp index 7a7ae1e5..f118840d 100644 --- a/src/tilemaker.cpp +++ b/src/tilemaker.cpp @@ -97,11 +97,19 @@ int main(const int argc, const char* argv[]) { #ifdef __GLIBC__ // glibc raises its mmap threshold dynamically (up to 32/64 MB) once a large // mmapped chunk is freed; multi-MB buffers then land in per-thread arenas - // that are never trimmed, inflating peak RSS. Setting the trim threshold - // explicitly also disables that dynamic adjustment (the mmap threshold - // stays at its 128 KB default), unless the user tunes malloc via the environment. - if (!getenv("MALLOC_TRIM_THRESHOLD_") && !getenv("MALLOC_MMAP_THRESHOLD_") && !getenv("GLIBC_TUNABLES")) { - mallopt(M_TRIM_THRESHOLD, 4 << 20); + // that are never trimmed, inflating peak RSS. Setting malloc parameters + // explicitly turns that dynamic adjustment off, so both are set here: + // - a 16 MB top pad makes the heap grow and shrink in large steps; with many + // threads allocating, the default 128 KB steps cost many brk() calls and + // page faults + // - a fixed 8 MB mmap threshold: otherwise it would stay at 128 KB, and every + // larger buffer would be a fresh mmap with zeroed pages. Boost's validity + // check on a huge polygon grows such buffers over and over (one relation + // with 1.9M points took 3x as long). + // Users who tune malloc via the environment keep their settings. + if (!getenv("MALLOC_TRIM_THRESHOLD_") && !getenv("MALLOC_MMAP_THRESHOLD_") && !getenv("MALLOC_TOP_PAD_") && !getenv("GLIBC_TUNABLES")) { + mallopt(M_TOP_PAD, 16 << 20); + mallopt(M_MMAP_THRESHOLD, 8 << 20); } #endif @@ -446,7 +454,7 @@ int main(const int argc, const char* argv[]) { clock_gettime(CLOCK_MONOTONIC, &start); #endif std::cout << "collecting tiles" << std::flush; - populateTilesAtZoom(sources, zoomResults); + populateTilesAtZoom(sources, zoomResults, options.threadNum); #ifdef CLOCK_MONOTONIC clock_gettime(CLOCK_MONOTONIC, &end); uint64_t tileNs = 1e9 * (end.tv_sec - start.tv_sec) + end.tv_nsec - start.tv_nsec; @@ -502,10 +510,44 @@ int main(const int argc, const char* argv[]) { // Cluster tiles: breadth-first for z0..z5, depth-first for z6 sortTileCoordinates(config.baseZoom, options.threadNum, tileCoordinates); + // Shared pieces of large polygons, for the SHARED_PIECE_ZOOM tiles below which a tile is written + { + const unsigned pieceZoom = TileDataSource::SHARED_PIECE_ZOOM; + std::vector piecePresent((size_t)1 << (2 * pieceZoom), false); + for (auto const &t : tileCoordinates) + if (t.first > pieceZoom) + piecePresent[((size_t)(t.second.x >> (t.first - pieceZoom)) << pieceZoom) + (t.second.y >> (t.first - pieceZoom))] = true; + auto present = [&](TileCoordinate x, TileCoordinate y) { + return x < (1u << pieceZoom) && y < (1u << pieceZoom) && piecePresent[((size_t)x << pieceZoom) + y]; + }; + for (auto source : sources) source->precomputeSharedPieces(present, options.threadNum); + } + std::atomic batchCounter(0); const uint cutZoom = std::max(12, config.startZoom); - std::size_t batchSize = 0; - for(std::size_t startIndex = 0; startIndex < tileCoordinates.size(); startIndex += batchSize) { + // Batches are handed out as threads become free rather than all posted up + // front: a planet has ~14M of them, and their queued handlers held GBs. + std::mutex batchMutex; + std::size_t nextBatchStart = 0; + uint64_t nextBatchSeq = 0; + // Shared pieces (TileDataSource::precomputeSharedPieces) lie inside one z6 tile. Tiles from z6 on + // are written z6 tile by z6 tile, so once the hand-out has moved past a z6 tile and none of its + // batches is still running, no tile needs its pieces any more. + std::vector z6Running(CLUSTER_ZOOM_WIDTH * CLUSTER_ZOOM_WIDTH, 0); + int64_t z6HandedOut = -1; + auto z6Of = [&](std::size_t i) -> int64_t { + const unsigned z = tileCoordinates[i].first; + if (z < CLUSTER_ZOOM) return -1; + return (int64_t)(tileCoordinates[i].second.x >> (z - CLUSTER_ZOOM)) * CLUSTER_ZOOM_WIDTH + (tileCoordinates[i].second.y >> (z - CLUSTER_ZOOM)); + }; + auto releaseZ6 = [&](int64_t b) { + for (auto source : sources) source->releaseSharedPieces(b / CLUSTER_ZOOM_WIDTH, b % CLUSTER_ZOOM_WIDTH); + }; + static thread_local std::vector batchZ6; // z6 tiles of the batch this thread is working on + auto nextBatch = [&](std::size_t &startIndex, std::size_t &batchSize, uint64_t &seq) { + const std::lock_guard lock(batchMutex); + startIndex = nextBatchStart; + seq = nextBatchSeq++; // Compute how many tiles should be assigned to this batch -- // higher-zoom tiles are cheaper to compute, lower-zoom tiles more expensive. batchSize = 0; @@ -529,65 +571,106 @@ int main(const int argc, const char* argv[]) { batchSize++; } + nextBatchStart += batchSize; + batchZ6.clear(); + for (std::size_t i = startIndex; i < startIndex + batchSize; i++) { + const int64_t b = z6Of(i); + if (b >= 0 && (batchZ6.empty() || batchZ6.back() != b)) batchZ6.push_back(b); + } + for (int64_t b : batchZ6) z6Running[b]++; + if (!batchZ6.empty() && batchZ6.back() != z6HandedOut) { + if (z6HandedOut >= 0 && z6Running[z6HandedOut] == 0) releaseZ6(z6HandedOut); + z6HandedOut = batchZ6.back(); + } + return batchSize > 0; + }; - boost::asio::post(pool, [=, &tileCoordinates, &pool, &sharedData, &sources, &attributeStore, &io_mutex, &tilesWritten, &lastTilesWritten, &batchCounter]() { - clipCacheBatchId = ++batchCounter; - // A single-tile batch has no descendants to reuse its clips. - clipCacheNoStore = batchSize == 1; - std::vector tileTimings; - std::size_t endIndex = std::min(tileCoordinates.size(), startIndex + batchSize); - for(std::size_t i = startIndex; i < endIndex; ++i) { - unsigned int zoom = tileCoordinates[i].first; - TileCoordinates coords = tileCoordinates[i].second; + // .pmtiles and .mbtiles are written in batch order, so the file doesn't depend on thread scheduling + const bool toPMTiles = options.outputMode == OptionsParser::OutputMode::PMTiles; + const bool toMBTiles = options.outputMode == OptionsParser::OutputMode::MBTiles; + // While an earlier batch is still being worked on, each thread may run ahead by about + // 64 MB of finished tiles. With less, threads often wait for a slow batch (mostly a + // dense z12 area), above all when two threads share a core and each runs at half speed. + const uint64_t maxHeldBytes = (64ull << 20) * options.threadNum; + if (toPMTiles) sharedData.pmtiles.batches.setMaxHeldBytes(maxHeldBytes); + if (toMBTiles) sharedData.mbtiles.batches.setMaxHeldBytes(maxHeldBytes); + for (unsigned int t = 0; t < options.threadNum; t++) { + boost::asio::post(pool, [&]() { + std::size_t startIndex, batchSize; + uint64_t seq; + // Shared pieces first (see TileDataSource::precomputeSharedPieces) + for (auto source : sources) source->computeSharedPieces(); + while (true) { + if (toPMTiles) sharedData.pmtiles.batches.waitForRoom(); + if (toMBTiles) sharedData.mbtiles.batches.waitForRoom(); + if (!nextBatch(startIndex, batchSize, seq)) break; + if (toPMTiles) sharedData.pmtiles.batches.begin(seq); + if (toMBTiles) sharedData.mbtiles.batches.begin(seq); + clipCacheBatchId = ++batchCounter; + // A single-tile batch has no descendants to reuse its clips. + clipCacheNoStore = batchSize == 1; + std::vector tileTimings; + std::size_t endIndex = std::min(tileCoordinates.size(), startIndex + batchSize); + for(std::size_t i = startIndex; i < endIndex; ++i) { + unsigned int zoom = tileCoordinates[i].first; + TileCoordinates coords = tileCoordinates[i].second; #ifdef CLOCK_MONOTONIC - timespec start, end; - if (options.logTileTimings) - clock_gettime(CLOCK_MONOTONIC, &start); + timespec start, end; + if (options.logTileTimings) + clock_gettime(CLOCK_MONOTONIC, &start); #endif - std::vector> data; - for (auto source : sources) { - data.emplace_back(source->getObjectsForTile(sortOrders, zoom, coords)); - } - outputProc(sharedData, sources, attributeStore, data, coords, zoom); + std::vector> data; + for (auto source : sources) { + data.emplace_back(source->getObjectsForTile(sortOrders, attributeStore, zoom, coords)); + } + outputProc(sharedData, sources, attributeStore, data, coords, zoom); #ifdef CLOCK_MONOTONIC - if (options.logTileTimings) { - clock_gettime(CLOCK_MONOTONIC, &end); - uint64_t tileNs = 1e9 * (end.tv_sec - start.tv_sec) + end.tv_nsec - start.tv_nsec; - std::string output = "z" + std::to_string(zoom) + "/" + std::to_string(coords.x) + "/" + std::to_string(coords.y) + " took " + std::to_string(tileNs/1e6) + " ms"; - tileTimings.push_back(output); - } + if (options.logTileTimings) { + clock_gettime(CLOCK_MONOTONIC, &end); + uint64_t tileNs = 1e9 * (end.tv_sec - start.tv_sec) + end.tv_nsec - start.tv_nsec; + std::string output = "z" + std::to_string(zoom) + "/" + std::to_string(coords.x) + "/" + std::to_string(coords.y) + " took " + std::to_string(tileNs/1e6) + " ms"; + tileTimings.push_back(output); + } #endif - } + } - if (options.logTileTimings) { - const std::lock_guard lock(io_mutex); - std::cout << std::endl; - for (const auto& output : tileTimings) - std::cout << output << std::endl; - } + if (options.logTileTimings) { + const std::lock_guard lock(io_mutex); + std::cout << std::endl; + for (const auto& output : tileTimings) + std::cout << output << std::endl; + } - tilesWritten += (endIndex - startIndex); - - if (io_mutex.try_lock()) { - uint64_t written = tilesWritten.load(); - - if (written >= lastTilesWritten + tileCoordinates.size() / 100 || ISATTY) { - lastTilesWritten = written; - // Show progress grouped by z6 (or lower) - size_t z = tileCoordinates[startIndex].first; - size_t x = tileCoordinates[startIndex].second.x; - size_t y = tileCoordinates[startIndex].second.y; - if (z > CLUSTER_ZOOM) { - x = x / (1 << (z - CLUSTER_ZOOM)); - y = y / (1 << (z - CLUSTER_ZOOM)); - z = CLUSTER_ZOOM; + { + const std::lock_guard lock(batchMutex); + for (int64_t b : batchZ6) + if (--z6Running[b] == 0 && b != z6HandedOut) releaseZ6(b); + } + if (toPMTiles) sharedData.pmtiles.batches.end(); + if (toMBTiles) sharedData.mbtiles.batches.end(); + tilesWritten += (endIndex - startIndex); + + if (io_mutex.try_lock()) { + uint64_t written = tilesWritten.load(); + + if (written >= lastTilesWritten + tileCoordinates.size() / 100 || ISATTY) { + lastTilesWritten = written; + // Show progress grouped by z6 (or lower) + size_t z = tileCoordinates[startIndex].first; + size_t x = tileCoordinates[startIndex].second.x; + size_t y = tileCoordinates[startIndex].second.y; + if (z > CLUSTER_ZOOM) { + x = x / (1 << (z - CLUSTER_ZOOM)); + y = y / (1 << (z - CLUSTER_ZOOM)); + z = CLUSTER_ZOOM; + } + cout << "z" << z << "/" << x << "/" << y << ", writing tile " << written << " of " << tileCoordinates.size() << " \r" << std::flush; } - cout << "z" << z << "/" << x << "/" << y << ", writing tile " << written << " of " << tileCoordinates.size() << " \r" << std::flush; + io_mutex.unlock(); } - io_mutex.unlock(); } }); } diff --git a/test/append_vector.test.cpp b/test/append_vector.test.cpp index b3104428..f0d5d704 100644 --- a/test/append_vector.test.cpp +++ b/test/append_vector.test.cpp @@ -21,6 +21,12 @@ MU_TEST(test_append_vector) { mu_check(vec[25] == 25); + const AppendVector& constVec = vec; + mu_check(constVec[25] == 25); + mu_check(constVec[8191] == 8191); + mu_check(constVec[8192] == 8192); + mu_check(constVec[9999] == 9999); + const AppendVector::Iterator& it = vec.begin(); mu_check(*it == 0); mu_check(*(it + 1) == 1); diff --git a/test/correct.test.cpp b/test/correct.test.cpp new file mode 100644 index 00000000..7178d173 --- /dev/null +++ b/test/correct.test.cpp @@ -0,0 +1,72 @@ +#include +#include +#include +#include "external/minunit.h" +#include "geometry/correct.hpp" + +typedef boost::geometry::model::d2::point_xy point_t; +typedef boost::geometry::model::polygon polygon_t; +typedef boost::geometry::model::multi_polygon multi_polygon_t; + +static bool near(double a, double b) { return std::abs(a - b) < 1e-9; } + +MU_TEST(test_correct_merges_overlapping_holes) { + // Two overlapping holes are merged into one; a disjoint hole is kept as is + polygon_t input; + boost::geometry::read_wkt( + "POLYGON((0 0,0 10,10 10,10 0,0 0)," + "(1 1,3 1,3 3,1 3,1 1),(2 2,4 2,4 4,2 4,2 2),(6 6,7 6,7 7,6 7,6 6))", input); + multi_polygon_t output; + geometry::correct(input, output); + + mu_check(output.size() == 1); + mu_check(output[0].inners().size() == 2); + mu_check(near(boost::geometry::area(output), 100 - 7 - 1)); + mu_check(boost::geometry::is_valid(output)); +} + +MU_TEST(test_correct_merges_overlapping_outers) { + multi_polygon_t input; + boost::geometry::read_wkt( + "MULTIPOLYGON(((0 0,0 2,2 2,2 0,0 0)),((1 1,1 3,3 3,3 1,1 1)),((5 5,5 6,6 6,6 5,5 5)))", input); + multi_polygon_t output; + geometry::correct(input, output); + + mu_check(output.size() == 2); + mu_check(near(boost::geometry::area(output), 4 + 4 - 1 + 1)); + mu_check(boost::geometry::is_valid(output)); +} + +MU_TEST(test_correct_many_disjoint_holes) { + // 400 disjoint holes, e.g. a lake with many islands: all are kept + std::string wkt = "POLYGON((0 0,0 100,100 100,100 0,0 0)"; + for (int x = 0; x < 20; x++) + for (int y = 0; y < 20; y++) { + double x0 = 1 + x * 5, y0 = 1 + y * 5; + wkt += ",(" + std::to_string(x0) + " " + std::to_string(y0) + "," + std::to_string(x0 + 2) + " " + std::to_string(y0) + "," + + std::to_string(x0 + 2) + " " + std::to_string(y0 + 2) + "," + std::to_string(x0) + " " + std::to_string(y0 + 2) + "," + + std::to_string(x0) + " " + std::to_string(y0) + ")"; + } + wkt += ")"; + polygon_t input; + boost::geometry::read_wkt(wkt, input); + multi_polygon_t output; + geometry::correct(input, output); + + mu_check(output.size() == 1); + mu_check(output[0].inners().size() == 400); + mu_check(near(boost::geometry::area(output), 100 * 100 - 400 * 4)); + mu_check(boost::geometry::is_valid(output)); +} + +MU_TEST_SUITE(test_suite_correct) { + MU_RUN_TEST(test_correct_merges_overlapping_holes); + MU_RUN_TEST(test_correct_merges_overlapping_outers); + MU_RUN_TEST(test_correct_many_disjoint_holes); +} + +int main() { + MU_RUN_SUITE(test_suite_correct); + MU_REPORT(); + return MU_EXIT_CODE; +} diff --git a/test/geom.test.cpp b/test/geom.test.cpp index e33b5990..2a31d97d 100644 --- a/test/geom.test.cpp +++ b/test/geom.test.cpp @@ -116,6 +116,44 @@ MU_TEST(test_precheck_many_holes) { checkPrecheck(mp, false); } +static Ring squareRing(double x, double y, double size) { + Ring r; + geom::append(r, Point(x, y)); geom::append(r, Point(x + size, y)); + geom::append(r, Point(x + size, y + size)); geom::append(r, Point(x, y + size)); + geom::append(r, Point(x, y)); + return r; +} + +MU_TEST(test_precheck_large_polygon) { + // A 40000-vertex outer ring (a circle) with 3600 holes: far more than walking the + // outer ring for every hole would allow (the old 2^26 work cap) + Polygon p; + const int n = 40000; + for (int i = 0; i <= n; i++) { + const double a = 2 * M_PI * (i % n) / n; + geom::append(p.outer(), Point(100 + 100 * std::cos(a), 100 + 100 * std::sin(a))); + } + for (int i = 0; i < 60; i++) + for (int j = 0; j < 60; j++) + p.inners().push_back(squareRing(40 + i * 2, 40 + j * 2, 1)); + MultiPolygon mp; + mp.push_back(p); + geom::correct(mp); + checkPrecheck(mp, true); + + // A hole outside the outer ring + MultiPolygon outside = mp; + outside[0].inners().push_back(squareRing(300, 100, 1)); + geom::correct(outside); + checkPrecheck(outside, false); + + // A hole nested in a hole on the other side of the polygon + MultiPolygon nested = mp; + nested[0].inners().insert(nested[0].inners().begin(), squareRing(40.25, 40.25, 0.5)); + geom::correct(nested); + checkPrecheck(nested, false); +} + MU_TEST_SUITE(test_suite_geom) { MU_RUN_TEST(test_rect_clip_concave); MU_RUN_TEST(test_rect_clip_holes); @@ -124,6 +162,7 @@ MU_TEST_SUITE(test_suite_geom) { MU_RUN_TEST(test_rect_clip_touch_from_inside); MU_RUN_TEST(test_precheck_overlapping_bboxes); MU_RUN_TEST(test_precheck_many_holes); + MU_RUN_TEST(test_precheck_large_polygon); } int main() { diff --git a/test/ordered_batches.test.cpp b/test/ordered_batches.test.cpp new file mode 100644 index 00000000..b9ff9599 --- /dev/null +++ b/test/ordered_batches.test.cpp @@ -0,0 +1,145 @@ +#include +#include +#include +#include +#include +#include +#include "external/minunit.h" +#include "ordered_batches.h" + +MU_TEST(test_ordered_batches_single_thread) { + std::vector written; + OrderedBatches batches([&](int& item) { written.push_back(item); }, [](const int&) { return 1; }); + + // batch 1 finishes first, so it's held until batch 0 is written + batches.begin(1); + mu_check(batches.inBatch()); + batches.add(10); + batches.add(11); + batches.end(); + mu_check(!batches.inBatch()); + mu_check(written.empty()); + mu_check(!batches.allWritten()); + + batches.begin(0); + batches.add(0); + batches.end(); + mu_check(written == std::vector({ 0, 10, 11 })); + mu_check(batches.allWritten()); + + // an empty batch still moves the order on + batches.begin(3); + batches.add(30); + batches.end(); + batches.begin(2); + batches.end(); + mu_check(written == std::vector({ 0, 10, 11, 30 })); + mu_check(batches.allWritten()); +} + +MU_TEST(test_ordered_batches_threads) { + // Batches are handed out in order and finish in any order; with a small limit on what + // may be held, threads also have to wait for room + const uint64_t numBatches = 2000, itemsPerBatch = 5; + std::vector written; + OrderedBatches batches([&](uint64_t& item) { written.push_back(item); }, [](const uint64_t&) { return 1; }); + batches.setMaxHeldBytes(20); + + std::mutex handOut; + uint64_t nextSeq = 0; + std::vector threads; + for (int t = 0; t < 8; t++) { + threads.emplace_back([&, t]() { + std::mt19937 rng(t); + while (true) { + batches.waitForRoom(); + uint64_t seq; + { + std::lock_guard lock(handOut); + if (nextSeq == numBatches) return; + seq = nextSeq++; + } + batches.begin(seq); + for (uint64_t i = 0; i < itemsPerBatch; i++) { + if (rng() % 4 == 0) std::this_thread::yield(); + batches.add(seq * itemsPerBatch + i); + } + batches.end(); + } + }); + } + for (auto& thread : threads) thread.join(); + + mu_check(batches.allWritten()); + mu_check(written.size() == numBatches * itemsPerBatch); + bool inOrder = true; + for (uint64_t i = 0; i < written.size(); i++) if (written[i] != i) inOrder = false; + mu_check(inOrder); +} + +MU_TEST(test_ordered_batches_flush) { + // With flushItem: every item is written and then flushed, both in order, flushes never + // overlap, and a flush can be done by a thread other than the one that wrote the batch + const uint64_t numBatches = 2000, itemsPerBatch = 5; + std::vector written, flushed; + std::atomic flushingNow(0); + std::atomic overlap(false), flushedBeforeWritten(false); + std::atomic writtenCount(0); + OrderedBatches batches( + [&](uint64_t& item) { written.push_back(item); writtenCount++; }, + [](const uint64_t&) { return 1; }, + [&](uint64_t& item) { + if (flushingNow++ != 0) overlap = true; + if (item >= writtenCount) flushedBeforeWritten = true; + flushed.push_back(item); + if (item % 7 == 0) std::this_thread::yield(); + flushingNow--; + }); + batches.setMaxHeldBytes(20); + + std::mutex handOut; + uint64_t nextSeq = 0; + std::vector threads; + for (int t = 0; t < 8; t++) { + threads.emplace_back([&, t]() { + std::mt19937 rng(t); + while (true) { + batches.waitForRoom(); + uint64_t seq; + { + std::lock_guard lock(handOut); + if (nextSeq == numBatches) return; + seq = nextSeq++; + } + batches.begin(seq); + for (uint64_t i = 0; i < itemsPerBatch; i++) { + if (rng() % 4 == 0) std::this_thread::yield(); + batches.add(seq * itemsPerBatch + i); + } + batches.end(); + } + }); + } + for (auto& thread : threads) thread.join(); + + mu_check(batches.allWritten()); + mu_check(!overlap); + mu_check(!flushedBeforeWritten); + mu_check(written.size() == numBatches * itemsPerBatch); + mu_check(flushed.size() == numBatches * itemsPerBatch); + bool inOrder = true; + for (uint64_t i = 0; i < flushed.size(); i++) if (flushed[i] != i || written[i] != i) inOrder = false; + mu_check(inOrder); +} + +MU_TEST_SUITE(test_suite_ordered_batches) { + MU_RUN_TEST(test_ordered_batches_single_thread); + MU_RUN_TEST(test_ordered_batches_threads); + MU_RUN_TEST(test_ordered_batches_flush); +} + +int main() { + MU_RUN_SUITE(test_suite_ordered_batches); + MU_REPORT(); + return MU_EXIT_CODE; +} diff --git a/test/shp_mem_tiles.test.cpp b/test/shp_mem_tiles.test.cpp new file mode 100644 index 00000000..b4629162 --- /dev/null +++ b/test/shp_mem_tiles.test.cpp @@ -0,0 +1,66 @@ +#include +#include "shp_mem_tiles.h" +#include "external/minunit.h" + +bool verbose = false; + +namespace { + MultiPolygon square(double x, double y, double size) { + MultiPolygon mp; + Polygon p; + geom::append(p.outer(), Point(x, y)); + geom::append(p.outer(), Point(x, y + size)); + geom::append(p.outer(), Point(x + size, y + size)); + geom::append(p.outer(), Point(x + size, y)); + geom::append(p.outer(), Point(x, y)); + mp.push_back(p); + return mp; + } + + Box box(double x1, double y1, double x2, double y2) { + return Box(Point(x1, y1), Point(x2, y2)); + } +} + +MU_TEST(test_may_intersect) { + Declutter declutter; + ShpMemTiles shp(1, 14, declutter); + shp.CreateNamedLayerIndex("layer"); + + // Two small squares at opposite corners: the shape's bounding box, (0,0)-(10,10), + // is almost all empty, like that of a country with far-flung islands. + MultiPolygon mp = square(0, 0, 1); + mp.push_back(square(9, 9, 1)[0]); + shp.StoreGeometry(0, "layer", POLYGON_, mp, true, true, "corners", 0, 0, 0); + + // Outside the shape's bounding box + mu_check(!shp.mayIntersect("layer", box(20, 20, 21, 21))); + + // A small box (a few z15 tiles) in the empty middle: checked tile by tile + mu_check(!shp.mayIntersect("layer", box(5, 5, 5.001, 5.001))); + // A large box (tens of thousands of z15 tiles) in the empty middle: checked at once + mu_check(!shp.mayIntersect("layer", box(3, 3, 7, 7))); + + // Small and large boxes that touch a square + mu_check(shp.mayIntersect("layer", box(0.5, 0.5, 0.501, 0.501))); + mu_check(shp.mayIntersect("layer", box(0.5, 0.5, 5, 5))); + mu_check(shp.mayIntersect("layer", box(5, 5, 9.5, 9.5))); + + // A large box that surrounds a square without containing a vertex of it + mu_check(shp.mayIntersect("layer", box(-1, -1, 2, 2))); + + // Answers stay the same once tiles have been checked and cached + mu_check(!shp.mayIntersect("layer", box(5, 5, 5.001, 5.001))); + mu_check(!shp.mayIntersect("layer", box(3, 3, 7, 7))); + mu_check(shp.mayIntersect("layer", box(0.5, 0.5, 0.501, 0.501))); +} + +MU_TEST_SUITE(test_suite_shp_mem_tiles) { + MU_RUN_TEST(test_may_intersect); +} + +int main() { + MU_RUN_SUITE(test_suite_shp_mem_tiles); + MU_REPORT(); + return MU_EXIT_CODE; +} diff --git a/test/tile_coordinates_set.test.cpp b/test/tile_coordinates_set.test.cpp index 87a054b8..973cc13c 100644 --- a/test/tile_coordinates_set.test.cpp +++ b/test/tile_coordinates_set.test.cpp @@ -45,10 +45,71 @@ MU_TEST(test_tile_coordinates_set) { mu_check(z2.test(1, 1) == true); mu_check(z2.test(2, 2) == false); } + + // forEachSet on a wrapped set visits the same tiles as a full scan, in x-major, y-minor order + { + PreciseTileCoordinatesSet z3(3); + z3.set(0, 0); z3.set(0, 5); z3.set(2, 1); z3.set(2, 7); z3.set(7, 3); + LossyTileCoordinatesSet z5(5, z3); + + std::vector> expected, visited; + for (TileCoordinate x = 0; x < 32; x++) + for (TileCoordinate y = 0; y < 32; y++) + if (z5.test(x, y)) expected.push_back({x, y}); + z5.forEachSet([&](TileCoordinate x, TileCoordinate y) { visited.push_back({x, y}); }); + + mu_check(expected.size() == 5 * 16); + mu_check(visited == expected); + } +} + +static bool sameTiles(const TileCoordinatesSet& a, const TileCoordinatesSet& b) { + std::vector> ta, tb; + a.forEachSet([&](TileCoordinate x, TileCoordinate y) { ta.emplace_back(x, y); }); + b.forEachSet([&](TileCoordinate x, TileCoordinate y) { tb.emplace_back(x, y); }); + return ta == tb && a.size() == b.size(); +} + +MU_TEST(test_tile_coordinates_set_box) { + // setBox must set exactly the tiles that set() would, including boxes that + // start and end mid-word, span several words, or are clipped at the edge + const TileCoordinate boxes[][4] = { + { 0, 0, 0, 0 }, { 3, 9, 5, 70 }, { 100, 101, 0, 1023 }, { 7, 7, 60, 200 }, + { 1000, 1023, 990, 1023 }, { 1020, 1023, 1000, 1023 } + }; + for (const auto& b : boxes) { + PreciseTileCoordinatesSet bySet(10), byBox(10); + for (TileCoordinate x = b[0]; x <= b[1]; x++) + for (TileCoordinate y = b[2]; y <= b[3]; y++) + bySet.set(x, y); + byBox.setBox(b[0], b[1], b[2], b[3]); + mu_check(sameTiles(bySet, byBox)); + } + // Partly outside the 1024x1024 grid: the part inside is set + PreciseTileCoordinatesSet edge(10); + edge.setBox(1020, 1030, 1022, 1030); + mu_check(edge.size() == 4 * 2); + mu_check(edge.test(1023, 1023)); +} + +MU_TEST(test_tile_coordinates_set_merge) { + PreciseTileCoordinatesSet a(12), b(12), both(12); + for (TileCoordinate i = 0; i < 4096; i += 37) { + a.set(i, (i * 7) % 4096); + both.set(i, (i * 7) % 4096); + } + for (TileCoordinate i = 0; i < 4096; i += 53) { + b.set((i * 3) % 4096, i); + both.set((i * 3) % 4096, i); + } + a.merge(b); + mu_check(sameTiles(a, both)); } MU_TEST_SUITE(test_suite_tile_coordinates_set) { MU_RUN_TEST(test_tile_coordinates_set); + MU_RUN_TEST(test_tile_coordinates_set_box); + MU_RUN_TEST(test_tile_coordinates_set_merge); } int main() {